From 39a1a66242a480b880032f5a6a4e31ee77414d4c Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Wed, 24 Jul 2013 13:09:07 -0700 Subject: [PATCH 01/16] Don't unload diff data unless loaded --- src/diff_file.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/diff_file.c b/src/diff_file.c index 9d06daafa..19bcf2d45 100644 --- a/src/diff_file.c +++ b/src/diff_file.c @@ -417,6 +417,9 @@ int git_diff_file_content__load(git_diff_file_content *fc) void git_diff_file_content__unload(git_diff_file_content *fc) { + if ((fc->flags & GIT_DIFF_FLAG__LOADED) == 0) + return; + if (fc->flags & GIT_DIFF_FLAG__FREE_DATA) { git__free(fc->map.data); fc->map.data = ""; From 69c66b554e1072d8b6c63366834e15310fecaea7 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Wed, 24 Jul 2013 13:09:33 -0700 Subject: [PATCH 02/16] Don't do text diff unless content will be used --- src/diff_patch.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/diff_patch.c b/src/diff_patch.c index 1b4adac03..02a45cb1a 100644 --- a/src/diff_patch.c +++ b/src/diff_patch.c @@ -230,6 +230,10 @@ static int diff_patch_generate(git_diff_patch *patch, git_diff_output *output) if ((patch->flags & GIT_DIFF_PATCH_DIFFED) != 0) return 0; + /* if we are not looking at the hunks and lines, don't do the diff */ + if (!output->hunk_cb && !output->data_cb) + return 0; + if ((patch->flags & GIT_DIFF_PATCH_LOADED) == 0 && (error = diff_patch_load(patch, output)) < 0) return error; From 18e9efc425bd617655782188056fe43ecdc673e4 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Wed, 24 Jul 2013 13:10:16 -0700 Subject: [PATCH 03/16] Don't check rename if file size difference is huge --- src/diff_tform.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/diff_tform.c b/src/diff_tform.c index ac5356a8c..ab43a3a14 100644 --- a/src/diff_tform.c +++ b/src/diff_tform.c @@ -446,6 +446,8 @@ static int similarity_calc( } blobsize = git_blob_rawsize(blob); + if (!file->size) + file->size = blobsize; if (!git__is_sizet(blobsize)) /* ? what to do ? */ blobsize = (size_t)-1; @@ -510,6 +512,13 @@ static int similarity_measure( return 0; } + /* check if file sizes too small or nowhere near each other */ + if (a_file->size > 127 && + b_file->size > 127 && + (a_file->size > (b_file->size << 4) || + b_file->size > (a_file->size << 4))) + return 0; + /* update signature cache if needed */ if (!cache[a_idx] && similarity_calc(diff, opts, a_idx, cache) < 0) return -1; From 427cc255dffc1a44f1e9a289067012370f3cc38d Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Wed, 24 Jul 2013 13:11:11 -0700 Subject: [PATCH 04/16] Use local variables in hash calc to avoid aliasing --- src/hashsig.c | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/hashsig.c b/src/hashsig.c index ab8d8b3f0..a11c4bee7 100644 --- a/src/hashsig.c +++ b/src/hashsig.c @@ -43,8 +43,8 @@ struct git_hashsig { int considered; }; -#define HEAP_LCHILD_OF(I) (((I)*2)+1) -#define HEAP_RCHILD_OF(I) (((I)*2)+2) +#define HEAP_LCHILD_OF(I) (((I)<<1)+1) +#define HEAP_RCHILD_OF(I) (((I)<<1)+2) #define HEAP_PARENT_OF(I) (((I)-1)>>1) static void hashsig_heap_init(hashsig_heap *h, hashsig_cmp cmp) @@ -152,8 +152,9 @@ static void hashsig_initial_window( hashsig_in_progress *prog) { hashsig_state state, shift_n; - int win_len; + int win_len, saw_lf = prog->saw_lf; const char *scan, *end; + char *window = &prog->window[0]; /* init until we have processed at least HASHSIG_HASH_WINDOW data */ @@ -170,7 +171,7 @@ static void hashsig_initial_window( while (scan < end && win_len < HASHSIG_HASH_WINDOW) { char ch = *scan++; - if (!hashsig_include_char(ch, sig->opt, &prog->saw_lf)) + if (!hashsig_include_char(ch, sig->opt, &saw_lf)) continue; state = (state * HASHSIG_HASH_SHIFT + ch) & HASHSIG_HASH_MASK; @@ -180,7 +181,7 @@ static void hashsig_initial_window( else shift_n = (shift_n * HASHSIG_HASH_SHIFT) & HASHSIG_HASH_MASK; - prog->window[win_len++] = ch; + window[win_len++] = ch; } /* insert initial hash if we just finished */ @@ -194,6 +195,7 @@ static void hashsig_initial_window( prog->state = state; prog->win_len = win_len; prog->shift_n = shift_n; + prog->saw_lf = saw_lf; *data = scan; } @@ -206,22 +208,26 @@ static int hashsig_add_hashes( { const char *scan = data, *end = data + size; hashsig_state state, shift_n, rmv; + int win_pos, saw_lf; + char *window = &prog->window[0]; if (prog->win_len < HASHSIG_HASH_WINDOW) hashsig_initial_window(sig, &scan, size, prog); state = prog->state; shift_n = prog->shift_n; + saw_lf = prog->saw_lf; + win_pos = prog->win_pos; /* advance window, adding new chars and removing old */ for (; scan < end; ++scan) { char ch = *scan; - if (!hashsig_include_char(ch, sig->opt, &prog->saw_lf)) + if (!hashsig_include_char(ch, sig->opt, &saw_lf)) continue; - rmv = shift_n * prog->window[prog->win_pos]; + rmv = shift_n * window[win_pos]; state = (state - rmv) & HASHSIG_HASH_MASK; state = (state * HASHSIG_HASH_SHIFT) & HASHSIG_HASH_MASK; @@ -231,11 +237,13 @@ static int hashsig_add_hashes( hashsig_heap_insert(&sig->maxs, (hashsig_t)state); sig->considered++; - prog->window[prog->win_pos] = ch; - prog->win_pos = (prog->win_pos + 1) % HASHSIG_HASH_WINDOW; + window[win_pos] = ch; + win_pos = (win_pos + 1) % HASHSIG_HASH_WINDOW; } - prog->state = state; + prog->state = state; + prog->saw_lf = saw_lf; + prog->win_pos = win_pos; return 0; } @@ -296,7 +304,7 @@ int git_hashsig_create_fromfile( const char *path, git_hashsig_option_t opts) { - char buf[4096]; + char buf[0x1000]; ssize_t buflen = 0; int error = 0, fd; hashsig_in_progress prog = HASHSIG_IN_PROGRESS_INIT; From 397357a0480b341fce5e93d3f485504f2ae94082 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Wed, 24 Jul 2013 13:12:00 -0700 Subject: [PATCH 05/16] Add rename test that used to be really slow Before the optimization commits, this test used to take about 20 seconds to run on my machine. Afterwards, there is still a couple seconds of data setup, but the actual diff and rename detection runs in a fraction of a second. --- tests-clar/diff/rename.c | 57 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests-clar/diff/rename.c b/tests-clar/diff/rename.c index 9efd9281c..ed58b7aa9 100644 --- a/tests-clar/diff/rename.c +++ b/tests-clar/diff/rename.c @@ -1122,3 +1122,60 @@ void test_diff_rename__unmodified_can_be_renamed(void) git_index_free(index); git_tree_free(tree); } + +void test_diff_rename__many_files(void) +{ + git_index *index; + git_tree *tree; + git_diff_list *diff = NULL; + diff_expects exp; + git_diff_options diffopts = GIT_DIFF_OPTIONS_INIT; + git_diff_find_options opts = GIT_DIFF_FIND_OPTIONS_INIT; + git_buf b = GIT_BUF_INIT; + int i, j; + char tmp[64]; + + cl_git_pass(git_repository_index(&index, g_repo)); + cl_git_pass( + git_revparse_single((git_object **)&tree, g_repo, "HEAD^{tree}")); + + cl_git_pass(p_rename("renames/ikeepsix.txt", "renames/ikeepsix2.txt")); + cl_git_pass(git_index_remove_bypath(index, "ikeepsix.txt")); + cl_git_pass(git_index_add_bypath(index, "ikeepsix2.txt")); + + for (i = 0; i < 100; i += 2) { + snprintf(tmp, sizeof(tmp), "renames/newfile%03d", i); + + for (j = 0; j < i * 128; ++j) + git_buf_printf(&b, "more content %d\n", i); + + cl_git_mkfile(tmp, b.ptr); + cl_git_pass(git_index_add_bypath(index, tmp + strlen("renames/"))); + } + git_buf_free(&b); + + cl_git_pass(git_index_write(index)); + + cl_git_pass(git_diff_tree_to_index(&diff, g_repo, tree, index, &diffopts)); + + memset(&exp, 0, sizeof(exp)); + cl_git_pass(git_diff_foreach( + diff, diff_file_cb, NULL, NULL, &exp)); + cl_assert_equal_i(1, exp.file_status[GIT_DELTA_DELETED]); + cl_assert_equal_i(51, exp.file_status[GIT_DELTA_ADDED]); + cl_assert_equal_i(52, exp.files); + + opts.flags = GIT_DIFF_FIND_ALL; + cl_git_pass(git_diff_find_similar(diff, &opts)); + + memset(&exp, 0, sizeof(exp)); + cl_git_pass(git_diff_foreach( + diff, diff_file_cb, NULL, NULL, &exp)); + cl_assert_equal_i(1, exp.file_status[GIT_DELTA_RENAMED]); + cl_assert_equal_i(50, exp.file_status[GIT_DELTA_ADDED]); + cl_assert_equal_i(51, exp.files); + + git_diff_list_free(diff); + git_index_free(index); + git_tree_free(tree); +} From f5c4d0225157a6e15fc08f07aa77b5e8f52cdac5 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Wed, 24 Jul 2013 13:44:35 -0700 Subject: [PATCH 06/16] Fix incorrect comment --- src/diff_tform.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diff_tform.c b/src/diff_tform.c index ab43a3a14..cabcd1f80 100644 --- a/src/diff_tform.c +++ b/src/diff_tform.c @@ -512,7 +512,7 @@ static int similarity_measure( return 0; } - /* check if file sizes too small or nowhere near each other */ + /* check if file sizes are nowhere near each other */ if (a_file->size > 127 && b_file->size > 127 && (a_file->size > (b_file->size << 4) || From a5140f4dda66263a34080b11cfc34a49c9743100 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Wed, 24 Jul 2013 17:11:49 -0700 Subject: [PATCH 07/16] Fix rename detection for tree-to-tree diffs The performance improvements I introduced for rename detection were not able to run successfully for tree-to-tree diffs because the blob size was not known early enough and so the file signature always had to be calculated nonetheless. This change separates loading blobs into memory from calculating the signature. I can't avoid having to load the large blobs into memory, but by moving it forward, I'm able to avoid the signature calculation if the blob won't come into play for renames. --- src/diff_tform.c | 149 ++++++++++++++++++++++++++------------- tests-clar/diff/rename.c | 50 ++++++++++++- 2 files changed, 148 insertions(+), 51 deletions(-) diff --git a/src/diff_tform.c b/src/diff_tform.c index cabcd1f80..08b0d5c38 100644 --- a/src/diff_tform.c +++ b/src/diff_tform.c @@ -408,54 +408,90 @@ GIT_INLINE(git_diff_file *) similarity_get_file(git_diff_list *diff, size_t idx) return (idx & 1) ? &delta->new_file : &delta->old_file; } +typedef struct { + size_t idx; + git_iterator_type_t src; + git_repository *repo; + git_diff_file *file; + git_buf data; + git_blob *blob; + int loaded; +} similarity_info; + +static void similarity_init( + similarity_info *info, git_diff_list *diff, size_t file_idx) +{ + info->idx = file_idx; + info->src = (file_idx & 1) ? diff->new_src : diff->old_src; + info->repo = diff->repo; + info->file = similarity_get_file(diff, file_idx); + info->blob = NULL; + info->loaded = 0; + git_buf_init(&info->data, 0); +} + +static int similarity_load(similarity_info *info) +{ + int error = 0; + git_diff_file *file = info->file; + + if (info->src == GIT_ITERATOR_TYPE_WORKDIR) { + error = git_buf_joinpath( + &info->data, git_repository_workdir(info->repo), file->path); + + /* if path is not a regular file, just skip this item */ + if (!error && !git_path_isfile(info->data.ptr)) + git_buf_free(&info->data); + } else if (git_blob_lookup(&info->blob, info->repo, &file->oid) < 0) { + /* if lookup fails, just skip this item in similarity calc */ + giterr_clear(); + } else { + if (!file->size) + file->size = git_blob_rawsize(info->blob); + assert(file->size == git_blob_rawsize(info->blob)); + + info->data.size = (size_t)(git__is_sizet(file->size) ? file->size : -1); + info->data.ptr = (char *)git_blob_rawcontent(info->blob); + } + + info->loaded = 1; + + return error; +} + +static void similarity_unload(similarity_info *info) +{ + if (info->blob) + git_blob_free(info->blob); + else + git_buf_free(&info->data); + + info->loaded = 0; +} + static int similarity_calc( - git_diff_list *diff, + similarity_info *info, const git_diff_find_options *opts, - size_t file_idx, void **cache) { int error = 0; - git_diff_file *file = similarity_get_file(diff, file_idx); - git_iterator_type_t src = (file_idx & 1) ? diff->new_src : diff->old_src; - if (src == GIT_ITERATOR_TYPE_WORKDIR) { /* compute hashsig from file */ - git_buf path = GIT_BUF_INIT; + if (!info->loaded && (error = similarity_load(info)) < 0) + return error; + if (!info->data.size) + return 0; + + if (info->src == GIT_ITERATOR_TYPE_WORKDIR) { /* TODO: apply wd-to-odb filters to file data if necessary */ - if ((error = git_buf_joinpath( - &path, git_repository_workdir(diff->repo), file->path)) < 0) - return error; - - /* if path is not a regular file, just skip this item */ - if (git_path_isfile(path.ptr)) - error = opts->metric->file_signature( - &cache[file_idx], file, path.ptr, opts->metric->payload); - - git_buf_free(&path); - } else { /* compute hashsig from blob buffer */ - git_blob *blob = NULL; - git_off_t blobsize; - - /* TODO: add max size threshold a la diff? */ - - if (git_blob_lookup(&blob, diff->repo, &file->oid) < 0) { - /* if lookup fails, just skip this item in similarity calc */ - giterr_clear(); - return 0; - } - - blobsize = git_blob_rawsize(blob); - if (!file->size) - file->size = blobsize; - if (!git__is_sizet(blobsize)) /* ? what to do ? */ - blobsize = (size_t)-1; - + error = opts->metric->file_signature( + &cache[info->idx], info->file, + info->data.ptr, opts->metric->payload); + } else { error = opts->metric->buffer_signature( - &cache[file_idx], file, git_blob_rawcontent(blob), - (size_t)blobsize, opts->metric->payload); - - git_blob_free(blob); + &cache[info->idx], info->file, + info->data.ptr, info->data.size, opts->metric->payload); } return error; @@ -478,6 +514,8 @@ static int similarity_measure( git_diff_file *a_file = similarity_get_file(diff, a_idx); git_diff_file *b_file = similarity_get_file(diff, b_idx); bool exact_match = FLAG_SET(opts, GIT_DIFF_FIND_EXACT_MATCH_ONLY); + int error = 0; + similarity_info a_info, b_info; *score = -1; @@ -512,26 +550,39 @@ static int similarity_measure( return 0; } + similarity_init(&a_info, diff, a_idx); + similarity_init(&b_info, diff, b_idx); + + if (!a_file->size && (error = similarity_load(&a_info)) < 0) + goto done; + if (!b_file->size && (error = similarity_load(&b_info)) < 0) + goto done; + /* check if file sizes are nowhere near each other */ if (a_file->size > 127 && b_file->size > 127 && (a_file->size > (b_file->size << 4) || b_file->size > (a_file->size << 4))) - return 0; + goto done; /* update signature cache if needed */ - if (!cache[a_idx] && similarity_calc(diff, opts, a_idx, cache) < 0) - return -1; - if (!cache[b_idx] && similarity_calc(diff, opts, b_idx, cache) < 0) - return -1; + if (!cache[a_idx] && (error = similarity_calc(&a_info, opts, cache)) < 0) + goto done; + if (!cache[b_idx] && (error = similarity_calc(&b_info, opts, cache)) < 0) + goto done; - /* some metrics may not wish to process this file (too big / too small) */ - if (!cache[a_idx] || !cache[b_idx]) - return 0; + /* calculate similarity provided that the metric choose to process + * both the a and b files (some may not if file is too big, etc). + */ + if (cache[a_idx] && cache[b_idx]) + error = opts->metric->similarity( + score, cache[a_idx], cache[b_idx], opts->metric->payload); - /* compare signatures */ - return opts->metric->similarity( - score, cache[a_idx], cache[b_idx], opts->metric->payload); +done: + similarity_unload(&a_info); + similarity_unload(&b_info); + + return error; } static int calc_self_similarity( diff --git a/tests-clar/diff/rename.c b/tests-clar/diff/rename.c index ed58b7aa9..79c89e362 100644 --- a/tests-clar/diff/rename.c +++ b/tests-clar/diff/rename.c @@ -1126,7 +1126,7 @@ void test_diff_rename__unmodified_can_be_renamed(void) void test_diff_rename__many_files(void) { git_index *index; - git_tree *tree; + git_tree *tree, *new_tree; git_diff_list *diff = NULL; diff_expects exp; git_diff_options diffopts = GIT_DIFF_OPTIONS_INIT; @@ -1176,6 +1176,52 @@ void test_diff_rename__many_files(void) cl_assert_equal_i(51, exp.files); git_diff_list_free(diff); - git_index_free(index); + + { + git_object *parent; + git_signature *sig; + git_oid tree_id, commit_id; + git_reference *ref; + + cl_git_pass(git_index_write_tree(&tree_id, index)); + cl_git_pass(git_tree_lookup(&new_tree, g_repo, &tree_id)); + + cl_git_pass(git_revparse_ext(&parent, &ref, g_repo, "HEAD")); + cl_git_pass(git_signature_new( + &sig, "Sm Test", "sm@tester.test", 1372350000, 480)); + + cl_git_pass(git_commit_create_v( + &commit_id, g_repo, git_reference_name(ref), sig, sig, + NULL, "yoyoyo", new_tree, 1, parent)); + + git_object_free(parent); + git_reference_free(ref); + git_signature_free(sig); + } + + cl_git_pass(git_diff_tree_to_tree( + &diff, g_repo, tree, new_tree, &diffopts)); + + memset(&exp, 0, sizeof(exp)); + cl_git_pass(git_diff_foreach( + diff, diff_file_cb, NULL, NULL, &exp)); + cl_assert_equal_i(1, exp.file_status[GIT_DELTA_DELETED]); + cl_assert_equal_i(51, exp.file_status[GIT_DELTA_ADDED]); + cl_assert_equal_i(52, exp.files); + + opts.flags = GIT_DIFF_FIND_ALL; + cl_git_pass(git_diff_find_similar(diff, &opts)); + + memset(&exp, 0, sizeof(exp)); + cl_git_pass(git_diff_foreach( + diff, diff_file_cb, NULL, NULL, &exp)); + cl_assert_equal_i(1, exp.file_status[GIT_DELTA_RENAMED]); + cl_assert_equal_i(50, exp.file_status[GIT_DELTA_ADDED]); + cl_assert_equal_i(51, exp.files); + + git_diff_list_free(diff); + + git_tree_free(new_tree); git_tree_free(tree); + git_index_free(index); } From effdbeb3239b777e2b19fc4944643fc7f2a768c3 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Wed, 24 Jul 2013 17:48:37 -0700 Subject: [PATCH 08/16] Make rename detection file size fix better The previous fix for checking file sizes with rename detection always loads the blob. In this version, if the odb backend can get the object header without loading the whole thing into memory, then we'll just use that, so that we can eliminate possible rename sources & targets without loading them. --- src/diff.h | 29 ++++++++++++ src/diff_file.c | 14 +----- src/diff_tform.c | 118 +++++++++++++++++++++++------------------------ 3 files changed, 90 insertions(+), 71 deletions(-) diff --git a/src/diff.h b/src/diff.h index d09a130bc..d1bec00c6 100644 --- a/src/diff.h +++ b/src/diff.h @@ -16,6 +16,7 @@ #include "iterator.h" #include "repository.h" #include "pool.h" +#include "odb.h" #define DIFF_OLD_PREFIX_DEFAULT "a/" #define DIFF_NEW_PREFIX_DEFAULT "b/" @@ -108,5 +109,33 @@ extern void git_diff_find_similar__hashsig_free(void *sig, void *payload); extern int git_diff_find_similar__calc_similarity( int *score, void *siga, void *sigb, void *payload); +/* + * Sometimes a git_diff_file will have a zero size; this attempts to + * fill in the size without loading the blob if possible. If that is + * not possible, then it will return the git_odb_object that had to be + * loaded and the caller can use it or dispose of it as needed. + */ +GIT_INLINE(int) git_diff_file__resolve_zero_size( + git_diff_file *file, git_odb_object **odb_obj, git_repository *repo) +{ + int error; + git_odb *odb; + size_t len; + git_otype type; + + if ((error = git_repository_odb(&odb, repo)) < 0) + return error; + + error = git_odb__read_header_or_object( + odb_obj, &len, &type, odb, &file->oid); + + git_odb_free(odb); + + if (!error) + file->size = (git_off_t)len; + + return error; +} + #endif diff --git a/src/diff_file.c b/src/diff_file.c index 19bcf2d45..bcfef13cd 100644 --- a/src/diff_file.c +++ b/src/diff_file.c @@ -241,19 +241,9 @@ static int diff_file_content_load_blob(git_diff_file_content *fc) /* if we don't know size, try to peek at object header first */ if (!fc->file->size) { - git_odb *odb; - size_t len; - git_otype type; - - if (!(error = git_repository_odb__weakptr(&odb, fc->repo))) { - error = git_odb__read_header_or_object( - &odb_obj, &len, &type, odb, &fc->file->oid); - git_odb_free(odb); - } - if (error) + if ((error = git_diff_file__resolve_zero_size( + fc->file, &odb_obj, fc->repo)) < 0) return error; - - fc->file->size = len; } if (diff_file_content_binary_by_size(fc)) diff --git a/src/diff_tform.c b/src/diff_tform.c index 08b0d5c38..8fd2a4fe9 100644 --- a/src/diff_tform.c +++ b/src/diff_tform.c @@ -414,59 +414,26 @@ typedef struct { git_repository *repo; git_diff_file *file; git_buf data; + git_odb_object *odb_obj; git_blob *blob; - int loaded; } similarity_info; -static void similarity_init( +static int similarity_init( similarity_info *info, git_diff_list *diff, size_t file_idx) { info->idx = file_idx; info->src = (file_idx & 1) ? diff->new_src : diff->old_src; info->repo = diff->repo; info->file = similarity_get_file(diff, file_idx); + info->odb_obj = NULL; info->blob = NULL; - info->loaded = 0; git_buf_init(&info->data, 0); -} -static int similarity_load(similarity_info *info) -{ - int error = 0; - git_diff_file *file = info->file; + if (info->file->size > 0) + return 0; - if (info->src == GIT_ITERATOR_TYPE_WORKDIR) { - error = git_buf_joinpath( - &info->data, git_repository_workdir(info->repo), file->path); - - /* if path is not a regular file, just skip this item */ - if (!error && !git_path_isfile(info->data.ptr)) - git_buf_free(&info->data); - } else if (git_blob_lookup(&info->blob, info->repo, &file->oid) < 0) { - /* if lookup fails, just skip this item in similarity calc */ - giterr_clear(); - } else { - if (!file->size) - file->size = git_blob_rawsize(info->blob); - assert(file->size == git_blob_rawsize(info->blob)); - - info->data.size = (size_t)(git__is_sizet(file->size) ? file->size : -1); - info->data.ptr = (char *)git_blob_rawcontent(info->blob); - } - - info->loaded = 1; - - return error; -} - -static void similarity_unload(similarity_info *info) -{ - if (info->blob) - git_blob_free(info->blob); - else - git_buf_free(&info->data); - - info->loaded = 0; + return git_diff_file__resolve_zero_size( + info->file, &info->odb_obj, info->repo); } static int similarity_calc( @@ -475,28 +442,59 @@ static int similarity_calc( void **cache) { int error = 0; - - if (!info->loaded && (error = similarity_load(info)) < 0) - return error; - - if (!info->data.size) - return 0; + git_diff_file *file = info->file; if (info->src == GIT_ITERATOR_TYPE_WORKDIR) { + if ((error = git_buf_joinpath( + &info->data, git_repository_workdir(info->repo), file->path)) < 0) + return error; + + /* if path is not a regular file, just skip this item */ + if (!git_path_isfile(info->data.ptr)) + return 0; + /* TODO: apply wd-to-odb filters to file data if necessary */ error = opts->metric->file_signature( &cache[info->idx], info->file, info->data.ptr, opts->metric->payload); } else { - error = opts->metric->buffer_signature( - &cache[info->idx], info->file, - info->data.ptr, info->data.size, opts->metric->payload); + /* if we didn't initially know the size, we might have an odb_obj + * around from earlier, so convert that, otherwise load the blob now + */ + if (info->odb_obj != NULL) + error = git_object__from_odb_object( + (git_object **)&info->blob, info->repo, + info->odb_obj, GIT_OBJ_BLOB); + else + error = git_blob_lookup(&info->blob, info->repo, &file->oid); + + if (error < 0) { + /* if lookup fails, just skip this item in similarity calc */ + giterr_clear(); + } else { + size_t sz = (size_t)(git__is_sizet(file->size) ? file->size : -1); + + error = opts->metric->buffer_signature( + &cache[info->idx], info->file, + git_blob_rawcontent(info->blob), sz, opts->metric->payload); + } } return error; } +static void similarity_unload(similarity_info *info) +{ + if (info->odb_obj) + git_odb_object_free(info->odb_obj); + + if (info->blob) + git_blob_free(info->blob); + else + git_buf_free(&info->data); +} + #define FLAG_SET(opts,flag_name) (((opts)->flags & flag_name) != 0) /* - score < 0 means files cannot be compared @@ -550,26 +548,28 @@ static int similarity_measure( return 0; } - similarity_init(&a_info, diff, a_idx); - similarity_init(&b_info, diff, b_idx); + memset(&a_info, 0, sizeof(a_info)); + memset(&b_info, 0, sizeof(b_info)); - if (!a_file->size && (error = similarity_load(&a_info)) < 0) - goto done; - if (!b_file->size && (error = similarity_load(&b_info)) < 0) - goto done; + /* set up similarity data (will try to update missing file sizes) */ + if (!cache[a_idx] && (error = similarity_init(&a_info, diff, a_idx)) < 0) + return error; + if (!cache[b_idx] && (error = similarity_init(&b_info, diff, b_idx)) < 0) + goto cleanup; /* check if file sizes are nowhere near each other */ if (a_file->size > 127 && b_file->size > 127 && (a_file->size > (b_file->size << 4) || b_file->size > (a_file->size << 4))) - goto done; + goto cleanup; /* update signature cache if needed */ if (!cache[a_idx] && (error = similarity_calc(&a_info, opts, cache)) < 0) - goto done; + goto cleanup; + if (!cache[b_idx] && (error = similarity_calc(&b_info, opts, cache)) < 0) - goto done; + goto cleanup; /* calculate similarity provided that the metric choose to process * both the a and b files (some may not if file is too big, etc). @@ -578,7 +578,7 @@ static int similarity_measure( error = opts->metric->similarity( score, cache[a_idx], cache[b_idx], opts->metric->payload); -done: +cleanup: similarity_unload(&a_info); similarity_unload(&b_info); From a16e41729d5dec4acd30302c4a217622de00d290 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Thu, 25 Jul 2013 12:27:39 -0700 Subject: [PATCH 09/16] Fix rename detection to use actual blob size The size data in the index may not reflect the actual size of the blob data from the ODB when content filtering comes into play. This commit fixes rename detection to use the actual blob size when calculating data signatures instead of the value from the index. Because of a misunderstanding on my part, I first converted the git_index_add_bypath API to use the post-filtered blob data size in creating the index entry. I backed that change out, but I kept the overall refactoring of that routine and the new internal git_blob__create_from_paths API because it eliminates an extra stat() call from the code that adds a file to the index. The existing tests actually cover this code path, at least when running on Windows, so at this point I'm not adding new tests to cover the changes. --- src/blob.c | 89 ++++++++++++++++++++++++++++-------------------- src/blob.h | 9 +++++ src/diff_tform.c | 8 ++++- src/index.c | 34 +++++------------- 4 files changed, 76 insertions(+), 64 deletions(-) diff --git a/src/blob.c b/src/blob.c index 2e4d5f479..0f1c97003 100644 --- a/src/blob.c +++ b/src/blob.c @@ -105,6 +105,7 @@ static int write_file_stream( static int write_file_filtered( git_oid *oid, + git_off_t *size, git_odb *odb, const char *full_path, git_vector *filters) @@ -123,8 +124,11 @@ static int write_file_filtered( git_buf_free(&source); /* Write the file to disk if it was properly filtered */ - if (!error) + if (!error) { + *size = dest.size; + error = git_odb_write(oid, odb, dest.ptr, dest.size, GIT_OBJ_BLOB); + } git_buf_free(&dest); return error; @@ -152,21 +156,46 @@ static int write_symlink( return error; } -static int blob_create_internal(git_oid *oid, git_repository *repo, const char *content_path, const char *hint_path, bool try_load_filters) +int git_blob__create_from_paths( + git_oid *oid, + struct stat *out_st, + git_repository *repo, + const char *content_path, + const char *hint_path, + mode_t hint_mode, + bool try_load_filters) { int error; struct stat st; git_odb *odb = NULL; git_off_t size; + mode_t mode; + git_buf path = GIT_BUF_INIT; assert(hint_path || !try_load_filters); - if ((error = git_path_lstat(content_path, &st)) < 0 || (error = git_repository_odb__weakptr(&odb, repo)) < 0) - return error; + if (!content_path) { + if (git_repository__ensure_not_bare(repo, "create blob from file") < 0) + return GIT_EBAREREPO; + + if (git_buf_joinpath( + &path, git_repository_workdir(repo), hint_path) < 0) + return -1; + + content_path = path.ptr; + } + + if ((error = git_path_lstat(content_path, &st)) < 0 || + (error = git_repository_odb(&odb, repo)) < 0) + goto done; + + if (out_st) + memcpy(out_st, &st, sizeof(st)); size = st.st_size; + mode = hint_mode ? hint_mode : st.st_mode; - if (S_ISLNK(st.st_mode)) { + if (S_ISLNK(hint_mode)) { error = write_symlink(oid, odb, content_path, (size_t)size); } else { git_vector write_filters = GIT_VECTOR_INIT; @@ -187,7 +216,8 @@ static int blob_create_internal(git_oid *oid, git_repository *repo, const char * error = write_file_stream(oid, odb, content_path, size); } else { /* We need to apply one or more filters */ - error = write_file_filtered(oid, odb, content_path, &write_filters); + error = write_file_filtered( + oid, &size, odb, content_path, &write_filters); } git_filters_free(&write_filters); @@ -207,34 +237,21 @@ static int blob_create_internal(git_oid *oid, git_repository *repo, const char * */ } +done: + git_odb_free(odb); + git_buf_free(&path); + return error; } -int git_blob_create_fromworkdir(git_oid *oid, git_repository *repo, const char *path) +int git_blob_create_fromworkdir( + git_oid *oid, git_repository *repo, const char *path) { - git_buf full_path = GIT_BUF_INIT; - const char *workdir; - int error; - - if ((error = git_repository__ensure_not_bare(repo, "create blob from file")) < 0) - return error; - - workdir = git_repository_workdir(repo); - - if (git_buf_joinpath(&full_path, workdir, path) < 0) { - git_buf_free(&full_path); - return -1; - } - - error = blob_create_internal( - oid, repo, git_buf_cstr(&full_path), - git_buf_cstr(&full_path) + strlen(workdir), true); - - git_buf_free(&full_path); - return error; + return git_blob__create_from_paths(oid, NULL, repo, NULL, path, 0, true); } -int git_blob_create_fromdisk(git_oid *oid, git_repository *repo, const char *path) +int git_blob_create_fromdisk( + git_oid *oid, git_repository *repo, const char *path) { int error; git_buf full_path = GIT_BUF_INIT; @@ -251,8 +268,8 @@ int git_blob_create_fromdisk(git_oid *oid, git_repository *repo, const char *pat if (workdir && !git__prefixcmp(hintpath, workdir)) hintpath += strlen(workdir); - error = blob_create_internal( - oid, repo, git_buf_cstr(&full_path), hintpath, true); + error = git_blob__create_from_paths( + oid, NULL, repo, git_buf_cstr(&full_path), hintpath, 0, true); git_buf_free(&full_path); return error; @@ -272,12 +289,9 @@ int git_blob_create_fromchunks( git_filebuf file = GIT_FILEBUF_INIT; git_buf path = GIT_BUF_INIT; - if (git_buf_join_n( - &path, '/', 3, - git_repository_path(repo), - GIT_OBJECTS_DIR, - "streamed") < 0) - goto cleanup; + if (git_buf_joinpath( + &path, git_repository_path(repo), GIT_OBJECTS_DIR "streamed") < 0) + goto cleanup; content = git__malloc(BUFFER_SIZE); GITERR_CHECK_ALLOC(content); @@ -303,7 +317,8 @@ int git_blob_create_fromchunks( if (git_filebuf_flush(&file) < 0) goto cleanup; - error = blob_create_internal(oid, repo, file.path_lock, hintpath, hintpath != NULL); + error = git_blob__create_from_paths( + oid, NULL, repo, file.path_lock, hintpath, 0, hintpath != NULL); cleanup: git_buf_free(&path); diff --git a/src/blob.h b/src/blob.h index 22e37cc3a..4cd9f1e0c 100644 --- a/src/blob.h +++ b/src/blob.h @@ -21,4 +21,13 @@ void git_blob__free(void *blob); int git_blob__parse(void *blob, git_odb_object *obj); int git_blob__getbuf(git_buf *buffer, git_blob *blob); +extern int git_blob__create_from_paths( + git_oid *out_oid, + struct stat *out_st, + git_repository *repo, + const char *full_path, + const char *hint_path, + mode_t hint_mode, + bool apply_filters); + #endif diff --git a/src/diff_tform.c b/src/diff_tform.c index 8fd2a4fe9..d4b8cf30a 100644 --- a/src/diff_tform.c +++ b/src/diff_tform.c @@ -473,7 +473,13 @@ static int similarity_calc( /* if lookup fails, just skip this item in similarity calc */ giterr_clear(); } else { - size_t sz = (size_t)(git__is_sizet(file->size) ? file->size : -1); + size_t sz; + + /* index size may not be actual blob size if filtered */ + if (file->size != git_blob_rawsize(info->blob)) + file->size = git_blob_rawsize(info->blob); + + sz = (size_t)(git__is_sizet(file->size) ? file->size : -1); error = opts->metric->buffer_signature( &cache[info->idx], info->file, diff --git a/src/index.c b/src/index.c index 0610eb5b9..cbdd43bdc 100644 --- a/src/index.c +++ b/src/index.c @@ -16,6 +16,7 @@ #include "iterator.h" #include "pathspec.h" #include "ignore.h" +#include "blob.h" #include "git2/odb.h" #include "git2/oid.h" @@ -604,42 +605,23 @@ int git_index_entry__cmp_icase(const void *a, const void *b) return strcasecmp(entry_a->path, entry_b->path); } -static int index_entry_init(git_index_entry **entry_out, git_index *index, const char *rel_path) +static int index_entry_init( + git_index_entry **entry_out, git_index *index, const char *rel_path) { + int error = 0; git_index_entry *entry = NULL; struct stat st; git_oid oid; - const char *workdir; - git_buf full_path = GIT_BUF_INIT; - int error; if (INDEX_OWNER(index) == NULL) return create_index_error(-1, "Could not initialize index entry. " "Index is not backed up by an existing repository."); - workdir = git_repository_workdir(INDEX_OWNER(index)); - - if (!workdir) - return create_index_error(GIT_EBAREREPO, - "Could not initialize index entry. Repository is bare"); - - if ((error = git_buf_joinpath(&full_path, workdir, rel_path)) < 0) - return error; - - if ((error = git_path_lstat(full_path.ptr, &st)) < 0) { - git_buf_free(&full_path); - return error; - } - - git_buf_free(&full_path); /* done with full path */ - - /* There is no need to validate the rel_path here, since it will be - * immediately validated by the call to git_blob_create_fromfile. - */ - - /* write the blob to disk and get the oid */ - if ((error = git_blob_create_fromworkdir(&oid, INDEX_OWNER(index), rel_path)) < 0) + /* write the blob to disk and get the oid and stat info */ + error = git_blob__create_from_paths( + &oid, &st, INDEX_OWNER(index), NULL, rel_path, 0, true); + if (error < 0) return error; entry = git__calloc(1, sizeof(git_index_entry)); From 8dd8aa480ba46863e9c7df40bb9695e88a0286ee Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Fri, 26 Jul 2013 10:28:57 -0700 Subject: [PATCH 10/16] Fix some warnings --- src/array.h | 11 ++++++----- src/blob.c | 2 +- src/diff_patch.c | 26 +++++++++++++------------- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/src/array.h b/src/array.h index 248010425..c25a1b29e 100644 --- a/src/array.h +++ b/src/array.h @@ -39,25 +39,26 @@ #define GITERR_CHECK_ARRAY(a) GITERR_CHECK_ALLOC((a).ptr) -typedef git_array_t(void) git_array_generic_t; +typedef git_array_t(char) git_array_generic_t; /* use a generic array for growth so this can return the new item */ -GIT_INLINE(void *) git_array_grow(git_array_generic_t *a, size_t item_size) +GIT_INLINE(void *) git_array_grow(void *_a, size_t item_size) { + git_array_generic_t *a = _a; uint32_t new_size = (a->size < 8) ? 8 : a->asize * 3 / 2; - void *new_array = git__realloc(a->ptr, new_size * item_size); + char *new_array = git__realloc(a->ptr, new_size * item_size); if (!new_array) { git_array_clear(*a); return NULL; } else { a->ptr = new_array; a->asize = new_size; a->size++; - return (((char *)a->ptr) + (a->size - 1) * item_size); + return a->ptr + (a->size - 1) * item_size; } } #define git_array_alloc(a) \ ((a).size >= (a).asize) ? \ - git_array_grow((git_array_generic_t *)&(a), sizeof(*(a).ptr)) : \ + git_array_grow(&(a), sizeof(*(a).ptr)) : \ (a).ptr ? &(a).ptr[(a).size++] : NULL #define git_array_last(a) ((a).size ? &(a).ptr[(a).size - 1] : NULL) diff --git a/src/blob.c b/src/blob.c index 0f1c97003..5bb51f7cf 100644 --- a/src/blob.c +++ b/src/blob.c @@ -195,7 +195,7 @@ int git_blob__create_from_paths( size = st.st_size; mode = hint_mode ? hint_mode : st.st_mode; - if (S_ISLNK(hint_mode)) { + if (S_ISLNK(mode)) { error = write_symlink(oid, odb, content_path, (size_t)size); } else { git_vector write_filters = GIT_VECTOR_INIT; diff --git a/src/diff_patch.c b/src/diff_patch.c index 02a45cb1a..69bb08198 100644 --- a/src/diff_patch.c +++ b/src/diff_patch.c @@ -288,8 +288,8 @@ int git_diff_foreach( if (diff_required(diff, "git_diff_foreach") < 0) return -1; - diff_output_init((git_diff_output *)&xo, - &diff->opts, file_cb, hunk_cb, data_cb, payload); + diff_output_init( + &xo.output, &diff->opts, file_cb, hunk_cb, data_cb, payload); git_xdiff_init(&xo, &diff->opts); git_vector_foreach(&diff->deltas, idx, patch.delta) { @@ -300,10 +300,10 @@ int git_diff_foreach( if (!(error = diff_patch_init_from_diff(&patch, diff, idx))) { - error = diff_patch_file_callback(&patch, (git_diff_output *)&xo); + error = diff_patch_file_callback(&patch, &xo.output); if (!error) - error = diff_patch_generate(&patch, (git_diff_output *)&xo); + error = diff_patch_generate(&patch, &xo.output); git_diff_patch_free(&patch); } @@ -441,7 +441,7 @@ int git_diff_blobs( memset(&xo, 0, sizeof(xo)); diff_output_init( - (git_diff_output *)&xo, opts, file_cb, hunk_cb, data_cb, payload); + &xo.output, opts, file_cb, hunk_cb, data_cb, payload); git_xdiff_init(&xo, opts); if (!old_path && new_path) @@ -452,7 +452,7 @@ int git_diff_blobs( error = diff_patch_from_blobs( &pd, &xo, old_blob, old_path, new_blob, new_path, opts); - git_diff_patch_free((git_diff_patch *)&pd); + git_diff_patch_free(&pd.patch); return error; } @@ -477,7 +477,7 @@ int git_diff_patch_from_blobs( memset(&xo, 0, sizeof(xo)); - diff_output_to_patch((git_diff_output *)&xo, &pd->patch); + diff_output_to_patch(&xo.output, &pd->patch); git_xdiff_init(&xo, opts); error = diff_patch_from_blobs( @@ -553,7 +553,7 @@ int git_diff_blob_to_buffer( memset(&xo, 0, sizeof(xo)); diff_output_init( - (git_diff_output *)&xo, opts, file_cb, hunk_cb, data_cb, payload); + &xo.output, opts, file_cb, hunk_cb, data_cb, payload); git_xdiff_init(&xo, opts); if (!old_path && buf_path) @@ -564,7 +564,7 @@ int git_diff_blob_to_buffer( error = diff_patch_from_blob_and_buffer( &pd, &xo, old_blob, old_path, buf, buflen, buf_path, opts); - git_diff_patch_free((git_diff_patch *)&pd); + git_diff_patch_free(&pd.patch); return error; } @@ -590,7 +590,7 @@ int git_diff_patch_from_blob_and_buffer( memset(&xo, 0, sizeof(xo)); - diff_output_to_patch((git_diff_output *)&xo, &pd->patch); + diff_output_to_patch(&xo.output, &pd->patch); git_xdiff_init(&xo, opts); error = diff_patch_from_blob_and_buffer( @@ -642,13 +642,13 @@ int git_diff_get_patch( if ((error = diff_patch_alloc_from_diff(&patch, diff, idx)) < 0) return error; - diff_output_to_patch((git_diff_output *)&xo, patch); + diff_output_to_patch(&xo.output, patch); git_xdiff_init(&xo, &diff->opts); - error = diff_patch_file_callback(patch, (git_diff_output *)&xo); + error = diff_patch_file_callback(patch, &xo.output); if (!error) - error = diff_patch_generate(patch, (git_diff_output *)&xo); + error = diff_patch_generate(patch, &xo.output); if (!error) { /* if cumulative diff size is < 0.5 total size, flatten the patch */ From d730d3f4f0efb269dd760a3100ae86c460b8ba36 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Wed, 31 Jul 2013 16:40:42 -0700 Subject: [PATCH 11/16] Major rename detection changes After doing further profiling, I found that a lot of time was being spent attempting to insert hashes into the file hash signature when using the rolling hash because the rolling hash approach generates a hash per byte of the file instead of one per run/line of data. To optimize this, I decided to convert back to a run-based file signature algorithm which would be more like core Git. After changing this, a number of the existing tests started to fail. In some cases, this appears to have been because the test was coded to be too specific to the particular results of the file similarity metric and in some cases there appear to have been bugs in the core rename detection code where only by the coincidence of the file similarity scoring were the expected results being generated. This renames all the variables in the core rename detection code to be more consistent and hopefully easier to follow which made it a bit easier to reason about the behavior of that code and fix the problems that I was seeing. I think it's in better shape now. There are a couple of tests now that attempt to stress test the rename detection code and they are quite slow. Most of the time is spent setting up the test data on disk and in the index. When we roll out performance improvements for index insertion, it should also speed up these tests I hope. --- src/diff_tform.c | 219 ++++++++++++++++++++++---------------- src/hashsig.c | 208 ++++++++++++++++-------------------- src/util.h | 5 + tests-clar/clar_libgit2.h | 14 +++ tests-clar/core/buffer.c | 42 +++++--- tests-clar/diff/rename.c | 99 ++++++++++++----- 6 files changed, 335 insertions(+), 252 deletions(-) diff --git a/src/diff_tform.c b/src/diff_tform.c index d4b8cf30a..92c4036fb 100644 --- a/src/diff_tform.c +++ b/src/diff_tform.c @@ -436,7 +436,7 @@ static int similarity_init( info->file, &info->odb_obj, info->repo); } -static int similarity_calc( +static int similarity_sig( similarity_info *info, const git_diff_find_options *opts, void **cache) @@ -566,16 +566,19 @@ static int similarity_measure( /* check if file sizes are nowhere near each other */ if (a_file->size > 127 && b_file->size > 127 && - (a_file->size > (b_file->size << 4) || - b_file->size > (a_file->size << 4))) + (a_file->size > (b_file->size << 3) || + b_file->size > (a_file->size << 3))) goto cleanup; /* update signature cache if needed */ - if (!cache[a_idx] && (error = similarity_calc(&a_info, opts, cache)) < 0) - goto cleanup; - - if (!cache[b_idx] && (error = similarity_calc(&b_info, opts, cache)) < 0) - goto cleanup; + if (!cache[a_idx]) { + if ((error = similarity_sig(&a_info, opts, cache)) < 0) + goto cleanup; + } + if (!cache[b_idx]) { + if ((error = similarity_sig(&b_info, opts, cache)) < 0) + goto cleanup; + } /* calculate similarity provided that the metric choose to process * both the a and b files (some may not if file is too big, etc). @@ -759,25 +762,30 @@ int git_diff_find_similar( git_diff_list *diff, git_diff_find_options *given_opts) { - size_t i, j, sigcache_size; + size_t s, t; int error = 0, similarity; - git_diff_delta *from, *to; + git_diff_delta *src, *tgt; git_diff_find_options opts; - size_t num_srcs = 0, num_tgts = 0, tried_srcs = 0, tried_tgts = 0; + size_t num_deltas, num_srcs = 0, num_tgts = 0; + size_t tried_srcs = 0, tried_tgts = 0; size_t num_rewrites = 0, num_updates = 0, num_bumped = 0; void **sigcache; /* cache of similarity metric file signatures */ - diff_find_match *match_srcs = NULL, *match_tgts = NULL, *best_match; + diff_find_match *tgt2src = NULL; + diff_find_match *src2tgt = NULL; + diff_find_match *tgt2src_copy = NULL; + diff_find_match *best_match; git_diff_file swap; if ((error = normalize_find_opts(diff, &opts, given_opts)) < 0) return error; + num_deltas = diff->deltas.length; + /* TODO: maybe abort if deltas.length > rename_limit ??? */ - if (!git__is_uint32(diff->deltas.length)) + if (!git__is_uint32(num_deltas)) return 0; - sigcache_size = diff->deltas.length * 2; /* keep size b/c diff may change */ - sigcache = git__calloc(sigcache_size, sizeof(void *)); + sigcache = git__calloc(num_deltas * 2, sizeof(void *)); GITERR_CHECK_ALLOC(sigcache); /* Label rename sources and targets @@ -785,11 +793,11 @@ int git_diff_find_similar( * This will also set self-similarity scores for MODIFIED files and * mark them for splitting if break-rewrites is enabled */ - git_vector_foreach(&diff->deltas, i, to) { - if (is_rename_source(diff, &opts, i, sigcache)) + git_vector_foreach(&diff->deltas, t, tgt) { + if (is_rename_source(diff, &opts, t, sigcache)) ++num_srcs; - if (is_rename_target(diff, &opts, i, sigcache)) + if (is_rename_target(diff, &opts, t, sigcache)) ++num_tgts; } @@ -797,10 +805,15 @@ int git_diff_find_similar( if (!num_srcs || !num_tgts) goto cleanup; - match_tgts = git__calloc(diff->deltas.length, sizeof(diff_find_match)); - GITERR_CHECK_ALLOC(match_tgts); - match_srcs = git__calloc(diff->deltas.length, sizeof(diff_find_match)); - GITERR_CHECK_ALLOC(match_srcs); + src2tgt = git__calloc(num_deltas, sizeof(diff_find_match)); + GITERR_CHECK_ALLOC(src2tgt); + tgt2src = git__calloc(num_deltas, sizeof(diff_find_match)); + GITERR_CHECK_ALLOC(tgt2src); + + if (FLAG_SET(&opts, GIT_DIFF_FIND_COPIES)) { + tgt2src_copy = git__calloc(num_deltas, sizeof(diff_find_match)); + GITERR_CHECK_ALLOC(tgt2src_copy); + } /* * Find best-fit matches for rename / copy candidates @@ -809,47 +822,61 @@ int git_diff_find_similar( find_best_matches: tried_tgts = num_bumped = 0; - git_vector_foreach(&diff->deltas, i, to) { + git_vector_foreach(&diff->deltas, t, tgt) { /* skip things that are not rename targets */ - if ((to->flags & GIT_DIFF_FLAG__IS_RENAME_TARGET) == 0) + if ((tgt->flags & GIT_DIFF_FLAG__IS_RENAME_TARGET) == 0) continue; tried_srcs = 0; - git_vector_foreach(&diff->deltas, j, from) { + git_vector_foreach(&diff->deltas, s, src) { /* skip things that are not rename sources */ - if ((from->flags & GIT_DIFF_FLAG__IS_RENAME_SOURCE) == 0) + if ((src->flags & GIT_DIFF_FLAG__IS_RENAME_SOURCE) == 0) continue; /* calculate similarity for this pair and find best match */ - if (i == j) + if (s == t) similarity = -1; /* don't measure self-similarity here */ else if ((error = similarity_measure( - &similarity, diff, &opts, sigcache, 2 * j, 2 * i + 1)) < 0) + &similarity, diff, &opts, sigcache, 2 * s, 2 * t + 1)) < 0) goto cleanup; - /* if this pairing is better for the src and the tgt, keep it */ - if (similarity > 0 && - match_tgts[i].similarity < (uint32_t)similarity && - match_srcs[j].similarity < (uint32_t)similarity) + if (similarity < 0) + continue; + + /* is this a better rename? */ + if (tgt2src[t].similarity < (uint32_t)similarity && + src2tgt[s].similarity < (uint32_t)similarity) { - if (match_tgts[i].similarity > 0) { - match_tgts[match_srcs[j].idx].similarity = 0; - match_srcs[match_tgts[i].idx].similarity = 0; - ++num_bumped; + /* eject old mapping */ + if (src2tgt[s].similarity > 0) { + tgt2src[src2tgt[s].idx].similarity = 0; + num_bumped++; + } + if (tgt2src[t].similarity > 0) { + src2tgt[tgt2src[t].idx].similarity = 0; + num_bumped++; } - match_tgts[i].similarity = (uint32_t)similarity; - match_tgts[i].idx = (uint32_t)j; + /* write new mapping */ + tgt2src[t].idx = s; + tgt2src[t].similarity = (uint32_t)similarity; + src2tgt[s].idx = t; + src2tgt[s].similarity = (uint32_t)similarity; + } - match_srcs[j].similarity = (uint32_t)similarity; - match_srcs[j].idx = (uint32_t)i; + /* keep best absolute match for copies */ + if (tgt2src_copy != NULL && + tgt2src_copy[t].similarity < (uint32_t)similarity) + { + tgt2src_copy[t].idx = s; + tgt2src_copy[t].similarity = (uint32_t)similarity; } if (++tried_srcs >= num_srcs) break; - /* cap on maximum targets we'll examine (per "to" file) */ + /* cap on maximum targets we'll examine (per "tgt" file) */ if (tried_srcs > opts.rename_limit) break; } @@ -867,18 +894,21 @@ find_best_matches: tried_tgts = 0; - git_vector_foreach(&diff->deltas, i, to) { + git_vector_foreach(&diff->deltas, t, tgt) { /* skip things that are not rename targets */ - if ((to->flags & GIT_DIFF_FLAG__IS_RENAME_TARGET) == 0) + if ((tgt->flags & GIT_DIFF_FLAG__IS_RENAME_TARGET) == 0) continue; /* check if this delta was the target of a similarity */ - best_match = &match_tgts[i]; - if (!best_match->similarity) + if (tgt2src[t].similarity) + best_match = &tgt2src[t]; + else if (tgt2src_copy && tgt2src_copy[t].similarity) + best_match = &tgt2src_copy[t]; + else continue; - j = best_match->idx; - from = GIT_VECTOR_GET(&diff->deltas, j); + s = best_match->idx; + src = GIT_VECTOR_GET(&diff->deltas, s); /* possible scenarios: * 1. from DELETE to ADD/UNTRACK/IGNORE = RENAME @@ -888,101 +918,107 @@ find_best_matches: * 5. from OTHER to ADD/UNTRACK/IGNORE = OTHER + COPY */ - if (from->status == GIT_DELTA_DELETED) { + if (src->status == GIT_DELTA_DELETED) { - if (delta_is_new_only(to)) { + if (delta_is_new_only(tgt)) { if (best_match->similarity < opts.rename_threshold) continue; - delta_make_rename(to, from, best_match->similarity); + delta_make_rename(tgt, src, best_match->similarity); - from->flags |= GIT_DIFF_FLAG__TO_DELETE; + src->flags |= GIT_DIFF_FLAG__TO_DELETE; num_rewrites++; } else { - assert(delta_is_split(to)); + assert(delta_is_split(tgt)); if (best_match->similarity < opts.rename_from_rewrite_threshold) continue; - memcpy(&swap, &to->old_file, sizeof(swap)); + memcpy(&swap, &tgt->old_file, sizeof(swap)); - delta_make_rename(to, from, best_match->similarity); + delta_make_rename(tgt, src, best_match->similarity); num_rewrites--; - from->status = GIT_DELTA_DELETED; - memcpy(&from->old_file, &swap, sizeof(from->old_file)); - memset(&from->new_file, 0, sizeof(from->new_file)); - from->new_file.path = from->old_file.path; - from->new_file.flags |= GIT_DIFF_FLAG_VALID_OID; + src->status = GIT_DELTA_DELETED; + memcpy(&src->old_file, &swap, sizeof(src->old_file)); + memset(&src->new_file, 0, sizeof(src->new_file)); + src->new_file.path = src->old_file.path; + src->new_file.flags |= GIT_DIFF_FLAG_VALID_OID; num_updates++; } } - else if (delta_is_split(from)) { + else if (delta_is_split(src)) { - if (delta_is_new_only(to)) { + if (delta_is_new_only(tgt)) { if (best_match->similarity < opts.rename_threshold) continue; - delta_make_rename(to, from, best_match->similarity); + delta_make_rename(tgt, src, best_match->similarity); - from->status = (diff->new_src == GIT_ITERATOR_TYPE_WORKDIR) ? + src->status = (diff->new_src == GIT_ITERATOR_TYPE_WORKDIR) ? GIT_DELTA_UNTRACKED : GIT_DELTA_ADDED; - memset(&from->old_file, 0, sizeof(from->old_file)); - from->old_file.path = from->new_file.path; - from->old_file.flags |= GIT_DIFF_FLAG_VALID_OID; + memset(&src->old_file, 0, sizeof(src->old_file)); + src->old_file.path = src->new_file.path; + src->old_file.flags |= GIT_DIFF_FLAG_VALID_OID; - from->flags &= ~GIT_DIFF_FLAG__TO_SPLIT; + src->flags &= ~GIT_DIFF_FLAG__TO_SPLIT; num_rewrites--; num_updates++; } else { - assert(delta_is_split(from)); + assert(delta_is_split(src)); if (best_match->similarity < opts.rename_from_rewrite_threshold) continue; - memcpy(&swap, &to->old_file, sizeof(swap)); + memcpy(&swap, &tgt->old_file, sizeof(swap)); - delta_make_rename(to, from, best_match->similarity); + delta_make_rename(tgt, src, best_match->similarity); num_rewrites--; num_updates++; - memcpy(&from->old_file, &swap, sizeof(from->old_file)); + memcpy(&src->old_file, &swap, sizeof(src->old_file)); /* if we've just swapped the new element into the correct * place, clear the SPLIT flag */ - if (match_tgts[j].idx == i && - match_tgts[j].similarity > + if (tgt2src[s].idx == t && + tgt2src[s].similarity > opts.rename_from_rewrite_threshold) { - - from->status = GIT_DELTA_RENAMED; - from->similarity = match_tgts[j].similarity; - match_tgts[j].similarity = 0; - from->flags &= ~GIT_DIFF_FLAG__TO_SPLIT; + src->status = GIT_DELTA_RENAMED; + src->similarity = tgt2src[s].similarity; + tgt2src[s].similarity = 0; + src->flags &= ~GIT_DIFF_FLAG__TO_SPLIT; num_rewrites--; } /* otherwise, if we just overwrote a source, update mapping */ - else if (j > i && match_srcs[i].similarity > 0) { - match_tgts[match_srcs[i].idx].idx = (uint32_t)j; + else if (s > t && src2tgt[t].similarity > 0) { + /* what used to be at src t is now at src s */ + tgt2src[src2tgt[t].idx].idx = (uint32_t)s; } num_updates++; } } - else if (delta_is_new_only(to)) { - if (!FLAG_SET(&opts, GIT_DIFF_FIND_COPIES) || - best_match->similarity < opts.copy_threshold) + else if (delta_is_new_only(tgt)) { + if (!FLAG_SET(&opts, GIT_DIFF_FIND_COPIES)) continue; - to->status = GIT_DELTA_COPIED; - to->similarity = best_match->similarity; - memcpy(&to->old_file, &from->old_file, sizeof(to->old_file)); + if (tgt2src_copy[t].similarity < opts.copy_threshold) + continue; + + /* always use best possible source for copy */ + best_match = &tgt2src_copy[t]; + src = GIT_VECTOR_GET(&diff->deltas, best_match->idx); + + tgt->status = GIT_DELTA_COPIED; + tgt->similarity = best_match->similarity; + memcpy(&tgt->old_file, &src->old_file, sizeof(tgt->old_file)); num_updates++; } @@ -998,12 +1034,13 @@ find_best_matches: FLAG_SET(&opts, GIT_DIFF_BREAK_REWRITES)); cleanup: - git__free(match_srcs); - git__free(match_tgts); + git__free(tgt2src); + git__free(src2tgt); + git__free(tgt2src_copy); - for (i = 0; i < sigcache_size; ++i) { - if (sigcache[i] != NULL) - opts.metric->free_signature(sigcache[i], opts.metric->payload); + for (t = 0; t < num_deltas * 2; ++t) { + if (sigcache[t] != NULL) + opts.metric->free_signature(sigcache[t], opts.metric->payload); } git__free(sigcache); diff --git a/src/hashsig.c b/src/hashsig.c index a11c4bee7..109f966ba 100644 --- a/src/hashsig.c +++ b/src/hashsig.c @@ -13,12 +13,15 @@ typedef uint64_t hashsig_state; #define HASHSIG_SCALE 100 -#define HASHSIG_HASH_WINDOW 32 -#define HASHSIG_HASH_START 0 +#define HASHSIG_MAX_RUN 80 +#define HASHSIG_HASH_START 0x012345678ABCDEF0LL #define HASHSIG_HASH_SHIFT 5 -#define HASHSIG_HASH_MASK 0x7FFFFFFF + +#define HASHSIG_HASH_MIX(S,CH) \ + (S) = ((S) << HASHSIG_HASH_SHIFT) - (S) + (hashsig_state)(CH) #define HASHSIG_HEAP_SIZE ((1 << 7) - 1) +#define HASHSIG_HEAP_MIN_SIZE 4 typedef int (*hashsig_cmp)(const void *a, const void *b, void *); @@ -28,14 +31,6 @@ typedef struct { hashsig_t values[HASHSIG_HEAP_SIZE]; } hashsig_heap; -typedef struct { - hashsig_state state, shift_n; - char window[HASHSIG_HASH_WINDOW]; - int win_len, win_pos, saw_lf; -} hashsig_in_progress; - -#define HASHSIG_IN_PROGRESS_INIT { HASHSIG_HASH_START, 1, {0}, 0, 0, 1 } - struct git_hashsig { hashsig_heap mins; hashsig_heap maxs; @@ -115,142 +110,109 @@ static void hashsig_heap_sort(hashsig_heap *h) static void hashsig_heap_insert(hashsig_heap *h, hashsig_t val) { - /* if heap is full, pop top if new element should replace it */ - if (h->size == h->asize && h->cmp(&val, &h->values[0], NULL) > 0) { - h->size--; - h->values[0] = h->values[h->size]; - hashsig_heap_down(h, 0); - } - /* if heap is not full, insert new element */ if (h->size < h->asize) { h->values[h->size++] = val; hashsig_heap_up(h, h->size - 1); } + + /* if heap is full, pop top if new element should replace it */ + else if (h->cmp(&val, &h->values[0], NULL) > 0) { + h->size--; + h->values[0] = h->values[h->size]; + hashsig_heap_down(h, 0); + } + } -GIT_INLINE(bool) hashsig_include_char( - char ch, git_hashsig_option_t opt, int *saw_lf) +typedef struct { + int use_ignores; + uint8_t ignore_ch[256]; +} hashsig_in_progress; + +static void hashsig_in_progress_init( + hashsig_in_progress *prog, git_hashsig *sig) { - if ((opt & GIT_HASHSIG_IGNORE_WHITESPACE) && git__isspace(ch)) - return false; + int i; - if (opt & GIT_HASHSIG_SMART_WHITESPACE) { - if (ch == '\r' || (*saw_lf && git__isspace(ch))) - return false; - - *saw_lf = (ch == '\n'); + switch (sig->opt) { + case GIT_HASHSIG_IGNORE_WHITESPACE: + for (i = 0; i < 256; ++i) + prog->ignore_ch[i] = git__isspace_nonlf(i); + prog->use_ignores = 1; + break; + case GIT_HASHSIG_SMART_WHITESPACE: + for (i = 0; i < 256; ++i) + prog->ignore_ch[i] = git__isspace(i); + prog->use_ignores = 1; + break; + default: + memset(prog, 0, sizeof(*prog)); + break; } - - return true; } -static void hashsig_initial_window( - git_hashsig *sig, - const char **data, - size_t size, - hashsig_in_progress *prog) -{ - hashsig_state state, shift_n; - int win_len, saw_lf = prog->saw_lf; - const char *scan, *end; - char *window = &prog->window[0]; - - /* init until we have processed at least HASHSIG_HASH_WINDOW data */ - - if (prog->win_len >= HASHSIG_HASH_WINDOW) - return; - - state = prog->state; - win_len = prog->win_len; - shift_n = prog->shift_n; - - scan = *data; - end = scan + size; - - while (scan < end && win_len < HASHSIG_HASH_WINDOW) { - char ch = *scan++; - - if (!hashsig_include_char(ch, sig->opt, &saw_lf)) - continue; - - state = (state * HASHSIG_HASH_SHIFT + ch) & HASHSIG_HASH_MASK; - - if (!win_len) - shift_n = 1; - else - shift_n = (shift_n * HASHSIG_HASH_SHIFT) & HASHSIG_HASH_MASK; - - window[win_len++] = ch; - } - - /* insert initial hash if we just finished */ - - if (win_len == HASHSIG_HASH_WINDOW) { - hashsig_heap_insert(&sig->mins, (hashsig_t)state); - hashsig_heap_insert(&sig->maxs, (hashsig_t)state); - sig->considered = 1; - } - - prog->state = state; - prog->win_len = win_len; - prog->shift_n = shift_n; - prog->saw_lf = saw_lf; - - *data = scan; -} +#define HASHSIG_IN_PROGRESS_INIT { 1 } static int hashsig_add_hashes( git_hashsig *sig, - const char *data, + const uint8_t *data, size_t size, hashsig_in_progress *prog) { - const char *scan = data, *end = data + size; - hashsig_state state, shift_n, rmv; - int win_pos, saw_lf; - char *window = &prog->window[0]; + const uint8_t *scan = data, *end = data + size; + hashsig_state state = HASHSIG_HASH_START; + int use_ignores = prog->use_ignores, len; + uint8_t ch; - if (prog->win_len < HASHSIG_HASH_WINDOW) - hashsig_initial_window(sig, &scan, size, prog); + while (scan < end) { + state = HASHSIG_HASH_START; - state = prog->state; - shift_n = prog->shift_n; - saw_lf = prog->saw_lf; - win_pos = prog->win_pos; + for (len = 0; scan < end && len < HASHSIG_MAX_RUN; ) { + ch = *scan; - /* advance window, adding new chars and removing old */ + if (use_ignores) + for (; scan < end && git__isspace_nonlf(ch); ch = *scan) + ++scan; + else if (sig->opt != GIT_HASHSIG_NORMAL) + for (; scan < end && ch == '\r'; ch = *scan) + ++scan; - for (; scan < end; ++scan) { - char ch = *scan; + /* peek at next character to decide what to do next */ + if (sig->opt == GIT_HASHSIG_SMART_WHITESPACE) + use_ignores = (ch == '\n'); - if (!hashsig_include_char(ch, sig->opt, &saw_lf)) - continue; + if (scan >= end) + break; + ++scan; - rmv = shift_n * window[win_pos]; + /* check run terminator */ + if (ch == '\n' || ch == '\0') + break; - state = (state - rmv) & HASHSIG_HASH_MASK; - state = (state * HASHSIG_HASH_SHIFT) & HASHSIG_HASH_MASK; - state = (state + ch) & HASHSIG_HASH_MASK; + ++len; + HASHSIG_HASH_MIX(state, ch); + } - hashsig_heap_insert(&sig->mins, (hashsig_t)state); - hashsig_heap_insert(&sig->maxs, (hashsig_t)state); - sig->considered++; + if (len > 0) { + hashsig_heap_insert(&sig->mins, (hashsig_t)state); + hashsig_heap_insert(&sig->maxs, (hashsig_t)state); - window[win_pos] = ch; - win_pos = (win_pos + 1) % HASHSIG_HASH_WINDOW; + sig->considered++; + + while (scan < end && (*scan == '\n' || !*scan)) + ++scan; + } } - prog->state = state; - prog->saw_lf = saw_lf; - prog->win_pos = win_pos; + prog->use_ignores = use_ignores; return 0; } static int hashsig_finalize_hashes(git_hashsig *sig) { - if (sig->mins.size < HASHSIG_HEAP_SIZE) { + if (sig->mins.size < HASHSIG_HEAP_MIN_SIZE) { giterr_set(GITERR_INVALID, "File too small for similarity signature calculation"); return GIT_EBUFS; @@ -282,11 +244,13 @@ int git_hashsig_create( git_hashsig_option_t opts) { int error; - hashsig_in_progress prog = HASHSIG_IN_PROGRESS_INIT; + hashsig_in_progress prog; git_hashsig *sig = hashsig_alloc(opts); GITERR_CHECK_ALLOC(sig); - error = hashsig_add_hashes(sig, buf, buflen, &prog); + hashsig_in_progress_init(&prog, sig); + + error = hashsig_add_hashes(sig, (const uint8_t *)buf, buflen, &prog); if (!error) error = hashsig_finalize_hashes(sig); @@ -304,10 +268,10 @@ int git_hashsig_create_fromfile( const char *path, git_hashsig_option_t opts) { - char buf[0x1000]; + uint8_t buf[0x1000]; ssize_t buflen = 0; int error = 0, fd; - hashsig_in_progress prog = HASHSIG_IN_PROGRESS_INIT; + hashsig_in_progress prog; git_hashsig *sig = hashsig_alloc(opts); GITERR_CHECK_ALLOC(sig); @@ -316,6 +280,8 @@ int git_hashsig_create_fromfile( return fd; } + hashsig_in_progress_init(&prog, sig); + while (!error) { if ((buflen = p_read(fd, buf, sizeof(buf))) <= 0) { if ((error = (int)buflen) < 0) @@ -370,6 +336,12 @@ static int hashsig_heap_compare(const hashsig_heap *a, const hashsig_heap *b) int git_hashsig_compare(const git_hashsig *a, const git_hashsig *b) { - return (hashsig_heap_compare(&a->mins, &b->mins) + - hashsig_heap_compare(&a->maxs, &b->maxs)) / 2; + /* if we have fewer than the maximum number of elements, then just use + * one array since the two arrays will be the same + */ + if (a->mins.size < HASHSIG_HEAP_SIZE) + return hashsig_heap_compare(&a->mins, &b->mins); + else + return (hashsig_heap_compare(&a->mins, &b->mins) + + hashsig_heap_compare(&a->maxs, &b->maxs)) / 2; } diff --git a/src/util.h b/src/util.h index a97c9bf39..ed9624770 100644 --- a/src/util.h +++ b/src/util.h @@ -294,6 +294,11 @@ GIT_INLINE(bool) git__isspace(int c) return (c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\r' || c == '\v' || c == 0x85 /* Unicode CR+LF */); } +GIT_INLINE(bool) git__isspace_nonlf(int c) +{ + return (c == ' ' || c == '\t' || c == '\f' || c == '\r' || c == '\v' || c == 0x85 /* Unicode CR+LF */); +} + GIT_INLINE(bool) git__iswildcard(int c) { return (c == '*' || c == '?' || c == '['); diff --git a/tests-clar/clar_libgit2.h b/tests-clar/clar_libgit2.h index 5371b2864..8c8357e40 100644 --- a/tests-clar/clar_libgit2.h +++ b/tests-clar/clar_libgit2.h @@ -34,6 +34,20 @@ void cl_git_report_failure(int, const char *, int, const char *); #define cl_assert_equal_sz(sz1,sz2) cl_assert_equal_i((int)sz1, (int)(sz2)) +GIT_INLINE(void) clar__assert_in_range( + int lo, int val, int hi, + const char *file, int line, const char *err, int should_abort) +{ + if (lo > val || hi < val) { + char buf[128]; + snprintf(buf, sizeof(buf), "%d not in [%d,%d]", val, lo, hi); + clar__fail(file, line, err, buf, should_abort); + } +} + +#define cl_assert_in_range(L,V,H) \ + clar__assert_in_range((L),(V),(H),__FILE__,__LINE__,"Range check: " #V " in [" #L "," #H "]", 1) + /* * Some utility macros for building long strings */ diff --git a/tests-clar/core/buffer.c b/tests-clar/core/buffer.c index 3d8221e04..9d9628cfd 100644 --- a/tests-clar/core/buffer.c +++ b/tests-clar/core/buffer.c @@ -734,10 +734,11 @@ void test_core_buffer__classify_with_utf8(void) } #define SIMILARITY_TEST_DATA_1 \ - "test data\nright here\ninline\ntada\nneeds more data\nlots of data\n" \ - "is this enough?\nthere has to be enough data to fill the hash array!\n" \ - "Apparently 191 bytes is the minimum amount of data needed.\nHere goes!\n" \ - "Let's make sure we've got plenty to go with here.\n smile \n" + "000\n001\n002\n003\n004\n005\n006\n007\n008\n009\n" \ + "010\n011\n012\n013\n014\n015\n016\n017\n018\n019\n" \ + "020\n021\n022\n023\n024\n025\n026\n027\n028\n029\n" \ + "030\n031\n032\n033\n034\n035\n036\n037\n038\n039\n" \ + "040\n041\n042\n043\n044\n045\n046\n047\n048\n049\n" void test_core_buffer__similarity_metric(void) { @@ -761,15 +762,17 @@ void test_core_buffer__similarity_metric(void) cl_git_pass(git_buf_sets(&buf, SIMILARITY_TEST_DATA_1)); cl_git_pass(git_hashsig_create(&a, buf.ptr, buf.size, GIT_HASHSIG_NORMAL)); cl_git_pass(git_buf_sets(&buf, - "Test data\nright here\ninline\ntada\nneeds more data\nlots of data\n" - "is this enough?\nthere has to be enough data to fill the hash array!\n" - "Apparently 191 bytes is the minimum amount of data needed.\nHere goes!\n" - "Let's make sure we've got plenty to go with here.\n smile \n")); + "000\n001\n002\n003\n004\n005\n006\n007\n008\n009\n" \ + "010\n011\n012\n013\n014\n015\n016\n017\n018\n019\n" \ + "x020x\n021\n022\n023\n024\n025\n026\n027\n028\n029\n" \ + "030\n031\n032\n033\n034\n035\n036\n037\n038\n039\n" \ + "040\n041\n042\n043\n044\n045\n046\n047\n048\n049\n" + )); cl_git_pass(git_hashsig_create(&b, buf.ptr, buf.size, GIT_HASHSIG_NORMAL)); sim = git_hashsig_compare(a, b); - cl_assert(95 < sim && sim < 100); /* expect >95% similarity */ + cl_assert_in_range(95, sim, 100); /* expect >95% similarity */ git_hashsig_free(a); git_hashsig_free(b); @@ -779,12 +782,13 @@ void test_core_buffer__similarity_metric(void) cl_git_pass(git_buf_sets(&buf, SIMILARITY_TEST_DATA_1)); cl_git_pass(git_hashsig_create(&a, buf.ptr, buf.size, GIT_HASHSIG_NORMAL)); cl_git_pass(git_buf_sets(&buf, SIMILARITY_TEST_DATA_1 - "and if I add some more, it should still be pretty similar, yes?\n")); + "050\n051\n052\n053\n054\n055\n056\n057\n058\n059\n")); cl_git_pass(git_hashsig_create(&b, buf.ptr, buf.size, GIT_HASHSIG_NORMAL)); sim = git_hashsig_compare(a, b); + /* 20% lines added ~= 10% lines changed */ - cl_assert(70 < sim && sim < 80); /* expect in the 70-80% similarity range */ + cl_assert_in_range(85, sim, 95); /* expect similarity around 90% */ git_hashsig_free(a); git_hashsig_free(b); @@ -794,15 +798,19 @@ void test_core_buffer__similarity_metric(void) cl_git_pass(git_buf_sets(&buf, SIMILARITY_TEST_DATA_1)); cl_git_pass(git_hashsig_create(&a, buf.ptr, buf.size, GIT_HASHSIG_NORMAL)); cl_git_pass(git_buf_sets(&buf, - "test data\nright here\ninline\ntada\nneeds more data\nlots of data\n" - "is this enough?\nthere has to be enough data to fill the hash array!\n" - "okay, that's half the original\nwhat else can we add?\nmore data\n" - "one more line will complete this\nshort\nlines\ndon't\nmatter\n")); + "000\n001\n002\n003\n004\n005\n006\n007\n008\n009\n" \ + "010\n011\n012\n013\n014\n015\n016\n017\n018\n019\n" \ + "020x\n021\n022\n023\n024\n" \ + "x25\nx26\nx27\nx28\nx29\n" \ + "x30\nx31\nx32\nx33\nx34\nx35\nx36\nx37\nx38\nx39\n" \ + "x40\nx41\nx42\nx43\nx44\nx45\nx46\nx47\nx48\nx49\n" + )); cl_git_pass(git_hashsig_create(&b, buf.ptr, buf.size, GIT_HASHSIG_NORMAL)); sim = git_hashsig_compare(a, b); + /* 50% lines changed */ - cl_assert(40 < sim && sim < 60); /* expect in the 40-60% similarity range */ + cl_assert_in_range(40, sim, 60); /* expect in the 40-60% similarity range */ git_hashsig_free(a); git_hashsig_free(b); @@ -891,7 +899,7 @@ void test_core_buffer__similarity_metric_whitespace(void) if (i == j) cl_assert_equal_i(100, sim); else - cl_assert(sim < 30); /* expect pretty different */ + cl_assert_in_range(0, sim, 30); /* pretty different */ } else { cl_assert_equal_i(100, sim); } diff --git a/tests-clar/diff/rename.c b/tests-clar/diff/rename.c index 79c89e362..20ee66288 100644 --- a/tests-clar/diff/rename.c +++ b/tests-clar/diff/rename.c @@ -236,6 +236,8 @@ void test_diff_rename__not_exact_match(void) &diff, g_repo, old_tree, new_tree, &diffopts)); opts.flags = GIT_DIFF_FIND_ALL; + opts.break_rewrite_threshold = 70; + cl_git_pass(git_diff_find_similar(diff, &opts)); memset(&exp, 0, sizeof(exp)); @@ -312,8 +314,8 @@ void test_diff_rename__not_exact_match(void) /* the default match algorithm is going to find the internal * whitespace differences in the lines of sixserving.txt to be - * significant enough that this will decide to split it into - * an ADD and a DELETE + * significant enough that this will decide to split it into an ADD + * and a DELETE */ memset(&exp, 0, sizeof(exp)); @@ -480,6 +482,7 @@ void test_diff_rename__working_directory_changes(void) cl_git_pass(git_diff_tree_to_workdir(&diff, g_repo, tree, &diffopts)); opts.flags = GIT_DIFF_FIND_ALL | GIT_DIFF_FIND_DONT_IGNORE_WHITESPACE; + opts.rename_threshold = 70; cl_git_pass(git_diff_find_similar(diff, &opts)); memset(&exp, 0, sizeof(exp)); @@ -1123,7 +1126,10 @@ void test_diff_rename__unmodified_can_be_renamed(void) git_tree_free(tree); } -void test_diff_rename__many_files(void) +#define ANOTHER_POEM \ +"OH, glorious are the guarded heights\nWhere guardian souls abide—\nSelf-exiled from our gross delights—\nAbove, beyond, outside:\nAn ampler arc their spirit swings—\nCommands a juster view—\nWe have their word for all these things,\nNo doubt their words are true.\n\nYet we, the bond slaves of our day,\nWhom dirt and danger press—\nCo-heirs of insolence, delay,\nAnd leagued unfaithfulness—\nSuch is our need must seek indeed\nAnd, having found, engage\nThe men who merely do the work\nFor which they draw the wage.\n\nFrom forge and farm and mine and bench,\nDeck, altar, outpost lone—\nMill, school, battalion, counter, trench,\nRail, senate, sheepfold, throne—\nCreation's cry goes up on high\nFrom age to cheated age:\n\"Send us the men who do the work\n\"For which they draw the wage!\"\n" + +static void test_with_many(size_t expected_new) { git_index *index; git_tree *tree, *new_tree; @@ -1131,9 +1137,6 @@ void test_diff_rename__many_files(void) diff_expects exp; git_diff_options diffopts = GIT_DIFF_OPTIONS_INIT; git_diff_find_options opts = GIT_DIFF_FIND_OPTIONS_INIT; - git_buf b = GIT_BUF_INIT; - int i, j; - char tmp[64]; cl_git_pass(git_repository_index(&index, g_repo)); cl_git_pass( @@ -1142,18 +1145,6 @@ void test_diff_rename__many_files(void) cl_git_pass(p_rename("renames/ikeepsix.txt", "renames/ikeepsix2.txt")); cl_git_pass(git_index_remove_bypath(index, "ikeepsix.txt")); cl_git_pass(git_index_add_bypath(index, "ikeepsix2.txt")); - - for (i = 0; i < 100; i += 2) { - snprintf(tmp, sizeof(tmp), "renames/newfile%03d", i); - - for (j = 0; j < i * 128; ++j) - git_buf_printf(&b, "more content %d\n", i); - - cl_git_mkfile(tmp, b.ptr); - cl_git_pass(git_index_add_bypath(index, tmp + strlen("renames/"))); - } - git_buf_free(&b); - cl_git_pass(git_index_write(index)); cl_git_pass(git_diff_tree_to_index(&diff, g_repo, tree, index, &diffopts)); @@ -1162,8 +1153,8 @@ void test_diff_rename__many_files(void) cl_git_pass(git_diff_foreach( diff, diff_file_cb, NULL, NULL, &exp)); cl_assert_equal_i(1, exp.file_status[GIT_DELTA_DELETED]); - cl_assert_equal_i(51, exp.file_status[GIT_DELTA_ADDED]); - cl_assert_equal_i(52, exp.files); + cl_assert_equal_i(expected_new + 1, exp.file_status[GIT_DELTA_ADDED]); + cl_assert_equal_i(expected_new + 2, exp.files); opts.flags = GIT_DIFF_FIND_ALL; cl_git_pass(git_diff_find_similar(diff, &opts)); @@ -1172,8 +1163,8 @@ void test_diff_rename__many_files(void) cl_git_pass(git_diff_foreach( diff, diff_file_cb, NULL, NULL, &exp)); cl_assert_equal_i(1, exp.file_status[GIT_DELTA_RENAMED]); - cl_assert_equal_i(50, exp.file_status[GIT_DELTA_ADDED]); - cl_assert_equal_i(51, exp.files); + cl_assert_equal_i(expected_new, exp.file_status[GIT_DELTA_ADDED]); + cl_assert_equal_i(expected_new + 1, exp.files); git_diff_list_free(diff); @@ -1206,8 +1197,8 @@ void test_diff_rename__many_files(void) cl_git_pass(git_diff_foreach( diff, diff_file_cb, NULL, NULL, &exp)); cl_assert_equal_i(1, exp.file_status[GIT_DELTA_DELETED]); - cl_assert_equal_i(51, exp.file_status[GIT_DELTA_ADDED]); - cl_assert_equal_i(52, exp.files); + cl_assert_equal_i(expected_new + 1, exp.file_status[GIT_DELTA_ADDED]); + cl_assert_equal_i(expected_new + 2, exp.files); opts.flags = GIT_DIFF_FIND_ALL; cl_git_pass(git_diff_find_similar(diff, &opts)); @@ -1216,8 +1207,8 @@ void test_diff_rename__many_files(void) cl_git_pass(git_diff_foreach( diff, diff_file_cb, NULL, NULL, &exp)); cl_assert_equal_i(1, exp.file_status[GIT_DELTA_RENAMED]); - cl_assert_equal_i(50, exp.file_status[GIT_DELTA_ADDED]); - cl_assert_equal_i(51, exp.files); + cl_assert_equal_i(expected_new, exp.file_status[GIT_DELTA_ADDED]); + cl_assert_equal_i(expected_new + 1, exp.files); git_diff_list_free(diff); @@ -1225,3 +1216,59 @@ void test_diff_rename__many_files(void) git_tree_free(tree); git_index_free(index); } + +void test_diff_rename__many_files(void) +{ + git_index *index; + char tmp[64]; + int i, j; + git_buf b = GIT_BUF_INIT; + + cl_git_pass(git_repository_index(&index, g_repo)); + + for (i = 0; i < 100; i += 1) { + snprintf(tmp, sizeof(tmp), "renames/newfile%03d", i); + for (j = i * 256; j > 0; --j) + git_buf_printf(&b, "more content %d\n", i); + cl_git_mkfile(tmp, b.ptr); + } + + for (i = 0; i < 100; i += 1) { + snprintf(tmp, sizeof(tmp), "renames/newfile%03d", i); + cl_git_pass(git_index_add_bypath(index, tmp + strlen("renames/"))); + } + + git_buf_free(&b); + git_index_free(index); + + test_with_many(100); +} + +void test_diff_rename__again_many_files(void) +{ + git_index *index; + char tmp[64]; + int i; + git_buf b = GIT_BUF_INIT; + + cl_git_pass(git_repository_index(&index, g_repo)); + + git_buf_printf(&b, "%08d\n" ANOTHER_POEM "%08d\n" ANOTHER_POEM ANOTHER_POEM, 0, 0); + + for (i = 0; i < 2500; i += 1) { + snprintf(tmp, sizeof(tmp), "renames/newfile%03d", i); + snprintf(b.ptr, 9, "%08d", i); + b.ptr[8] = '\n'; + cl_git_mkfile(tmp, b.ptr); + } + git_buf_free(&b); + + for (i = 0; i < 2500; i += 1) { + snprintf(tmp, sizeof(tmp), "renames/newfile%03d", i); + cl_git_pass(git_index_add_bypath(index, tmp + strlen("renames/"))); + } + + git_index_free(index); + + test_with_many(2500); +} From a42c2a8c89189afbcfccbc17f798971c18c15de6 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Wed, 31 Jul 2013 21:51:50 -0500 Subject: [PATCH 12/16] Rename test for multiple similar matches A rename test that illustrates a source matching multiple targets. --- tests-clar/diff/rename.c | 66 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 3 deletions(-) diff --git a/tests-clar/diff/rename.c b/tests-clar/diff/rename.c index 20ee66288..108b34621 100644 --- a/tests-clar/diff/rename.c +++ b/tests-clar/diff/rename.c @@ -871,6 +871,8 @@ void test_diff_rename__from_deleted_to_split(void) struct rename_expected { size_t len; + + unsigned int *status; const char **sources; const char **targets; @@ -885,7 +887,7 @@ int test_names_expected(const git_diff_delta *delta, float progress, void *p) cl_assert(expected->idx < expected->len); - cl_assert_equal_i(delta->status, GIT_DELTA_RENAMED); + cl_assert_equal_i(delta->status, expected->status[expected->idx]); cl_assert(git__strcmp(expected->sources[expected->idx], delta->old_file.path) == 0); @@ -907,9 +909,10 @@ void test_diff_rename__rejected_match_can_match_others(void) git_diff_options diffopts = GIT_DIFF_OPTIONS_INIT; git_diff_find_options findopts = GIT_DIFF_FIND_OPTIONS_INIT; git_buf one = GIT_BUF_INIT, two = GIT_BUF_INIT; + unsigned int status[] = { GIT_DELTA_RENAMED, GIT_DELTA_RENAMED }; const char *sources[] = { "Class1.cs", "Class2.cs" }; const char *targets[] = { "ClassA.cs", "ClassB.cs" }; - struct rename_expected expect = { 2, sources, targets }; + struct rename_expected expect = { 2, status, sources, targets }; char *ptr; opts.checkout_strategy = GIT_CHECKOUT_FORCE; @@ -991,9 +994,10 @@ void test_diff_rename__rejected_match_can_match_others_two(void) git_diff_list *diff; git_diff_options diffopts = GIT_DIFF_OPTIONS_INIT; git_diff_find_options findopts = GIT_DIFF_FIND_OPTIONS_INIT; + unsigned int status[] = { GIT_DELTA_RENAMED, GIT_DELTA_RENAMED }; const char *sources[] = { "a.txt", "b.txt" }; const char *targets[] = { "c.txt", "d.txt" }; - struct rename_expected expect = { 2, sources, targets }; + struct rename_expected expect = { 2, status, sources, targets }; opts.checkout_strategy = GIT_CHECKOUT_FORCE; @@ -1036,6 +1040,62 @@ void test_diff_rename__rejected_match_can_match_others_two(void) git_reference_free(selfsimilar); } +void test_diff_rename__rejected_match_can_match_others_three(void) +{ + git_reference *head, *selfsimilar; + git_index *index; + git_tree *tree; + git_checkout_opts opts = GIT_CHECKOUT_OPTS_INIT; + git_diff_list *diff; + git_diff_options diffopts = GIT_DIFF_OPTIONS_INIT; + git_diff_find_options findopts = GIT_DIFF_FIND_OPTIONS_INIT; + + /* Both cannot be renames from a.txt */ + unsigned int status[] = { GIT_DELTA_ADDED, GIT_DELTA_RENAMED }; + const char *sources[] = { "0001.txt", "a.txt" }; + const char *targets[] = { "0001.txt", "0002.txt" }; + struct rename_expected expect = { 2, status, sources, targets }; + + opts.checkout_strategy = GIT_CHECKOUT_FORCE; + + cl_git_pass(git_reference_lookup(&head, g_repo, "HEAD")); + cl_git_pass(git_reference_symbolic_set_target( + &selfsimilar, head, "refs/heads/renames_similar_two")); + cl_git_pass(git_checkout_head(g_repo, &opts)); + cl_git_pass(git_repository_index(&index, g_repo)); + + cl_git_pass(p_unlink("renames/a.txt")); + + cl_git_pass(git_index_remove_bypath(index, "a.txt")); + + write_similarity_file_two("renames/0001.txt", 7); + write_similarity_file_two("renames/0002.txt", 0); + + cl_git_pass(git_index_add_bypath(index, "0001.txt")); + cl_git_pass(git_index_add_bypath(index, "0002.txt")); + + cl_git_pass(git_index_write(index)); + + cl_git_pass( + git_revparse_single((git_object **)&tree, g_repo, "HEAD^{tree}")); + + cl_git_pass( + git_diff_tree_to_index(&diff, g_repo, tree, index, &diffopts)); + + cl_git_pass(git_diff_find_similar(diff, &findopts)); + + cl_git_pass( + git_diff_foreach(diff, test_names_expected, NULL, NULL, &expect)); + + cl_assert(expect.idx == expect.len); + + git_diff_list_free(diff); + git_tree_free(tree); + git_index_free(index); + git_reference_free(head); + git_reference_free(selfsimilar); +} + void test_diff_rename__case_changes_are_split(void) { git_index *index; From 7edb74d374f401cea50be143b63c7d2d141d18be Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Sun, 4 Aug 2013 14:06:13 -0700 Subject: [PATCH 13/16] Update rename src map for any split src When using a rename source that is actually a to-be-split record, we have to update the best-fit mapping data in both the case where the target is also a split record and the case where the target is a simple added record. Before this commit, we were only doing the update when the target was itself a split record (and even in that case, the test was slightly wrong). --- src/diff_tform.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/diff_tform.c b/src/diff_tform.c index 92c4036fb..ba35d3c14 100644 --- a/src/diff_tform.c +++ b/src/diff_tform.c @@ -947,6 +947,11 @@ find_best_matches: src->new_file.flags |= GIT_DIFF_FLAG_VALID_OID; num_updates++; + + if (src2tgt[t].similarity > 0 && src2tgt[t].idx > t) { + /* what used to be at src t is now at src s */ + tgt2src[src2tgt[t].idx].idx = (uint32_t)s; + } } } @@ -996,7 +1001,7 @@ find_best_matches: num_rewrites--; } /* otherwise, if we just overwrote a source, update mapping */ - else if (s > t && src2tgt[t].similarity > 0) { + else if (src2tgt[t].similarity > 0 && src2tgt[t].idx > t) { /* what used to be at src t is now at src s */ tgt2src[src2tgt[t].idx].idx = (uint32_t)s; } From 0a38eb42ca207f7aeb9e48a172fdbbc587201eb1 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Wed, 31 Jul 2013 22:36:50 -0500 Subject: [PATCH 14/16] Rename test for rename from rewrite A rename test that illustrates a rename from a rewrite. --- tests-clar/diff/rename.c | 56 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests-clar/diff/rename.c b/tests-clar/diff/rename.c index 108b34621..b78122c7e 100644 --- a/tests-clar/diff/rename.c +++ b/tests-clar/diff/rename.c @@ -1096,6 +1096,56 @@ void test_diff_rename__rejected_match_can_match_others_three(void) git_reference_free(selfsimilar); } +void test_diff_rename__can_rename_from_rewrite(void) +{ + git_index *index; + git_tree *tree; + git_checkout_opts opts = GIT_CHECKOUT_OPTS_INIT; + git_diff_list *diff; + git_diff_options diffopts = GIT_DIFF_OPTIONS_INIT; + git_diff_find_options findopts = GIT_DIFF_FIND_OPTIONS_INIT; + + unsigned int status[] = { GIT_DELTA_RENAMED, GIT_DELTA_RENAMED }; + const char *sources[] = { "ikeepsix.txt", "songof7cities.txt" }; + const char *targets[] = { "songof7cities.txt", "this-is-a-rename.txt" }; + struct rename_expected expect = { 2, status, sources, targets }; + + opts.checkout_strategy = GIT_CHECKOUT_FORCE; + + cl_git_pass(git_repository_index(&index, g_repo)); + + cl_git_pass(p_rename("renames/songof7cities.txt", "renames/this-is-a-rename.txt")); + cl_git_pass(p_rename("renames/ikeepsix.txt", "renames/songof7cities.txt")); + + cl_git_pass(git_index_remove_bypath(index, "ikeepsix.txt")); + + cl_git_pass(git_index_add_bypath(index, "songof7cities.txt")); + cl_git_pass(git_index_add_bypath(index, "this-is-a-rename.txt")); + + cl_git_pass(git_index_write(index)); + + cl_git_pass( + git_revparse_single((git_object **)&tree, g_repo, "HEAD^{tree}")); + + cl_git_pass( + git_diff_tree_to_index(&diff, g_repo, tree, index, &diffopts)); + + findopts.flags |= GIT_DIFF_FIND_AND_BREAK_REWRITES | + GIT_DIFF_FIND_REWRITES | + GIT_DIFF_FIND_RENAMES_FROM_REWRITES; + + cl_git_pass(git_diff_find_similar(diff, &findopts)); + + cl_git_pass( + git_diff_foreach(diff, test_names_expected, NULL, NULL, &expect)); + + cl_assert(expect.idx == expect.len); + + git_diff_list_free(diff); + git_tree_free(tree); + git_index_free(index); +} + void test_diff_rename__case_changes_are_split(void) { git_index *index; @@ -1284,6 +1334,8 @@ void test_diff_rename__many_files(void) int i, j; git_buf b = GIT_BUF_INIT; +/* + cl_git_pass(git_repository_index(&index, g_repo)); for (i = 0; i < 100; i += 1) { @@ -1302,6 +1354,7 @@ void test_diff_rename__many_files(void) git_index_free(index); test_with_many(100); +*/ } void test_diff_rename__again_many_files(void) @@ -1311,6 +1364,8 @@ void test_diff_rename__again_many_files(void) int i; git_buf b = GIT_BUF_INIT; +/* + cl_git_pass(git_repository_index(&index, g_repo)); git_buf_printf(&b, "%08d\n" ANOTHER_POEM "%08d\n" ANOTHER_POEM ANOTHER_POEM, 0, 0); @@ -1331,4 +1386,5 @@ void test_diff_rename__again_many_files(void) git_index_free(index); test_with_many(2500); +*/ } From 31b42eacce29defe07bdc1fa50a21aa11b23ea83 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Sun, 4 Aug 2013 14:09:44 -0700 Subject: [PATCH 15/16] Restore commented out tests This restores the commented out tests (even though they're slow) and fixes some trailing whitespace. --- tests-clar/diff/rename.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests-clar/diff/rename.c b/tests-clar/diff/rename.c index b78122c7e..55555012a 100644 --- a/tests-clar/diff/rename.c +++ b/tests-clar/diff/rename.c @@ -871,7 +871,7 @@ void test_diff_rename__from_deleted_to_split(void) struct rename_expected { size_t len; - + unsigned int *status; const char **sources; const char **targets; @@ -1129,7 +1129,7 @@ void test_diff_rename__can_rename_from_rewrite(void) cl_git_pass( git_diff_tree_to_index(&diff, g_repo, tree, index, &diffopts)); - + findopts.flags |= GIT_DIFF_FIND_AND_BREAK_REWRITES | GIT_DIFF_FIND_REWRITES | GIT_DIFF_FIND_RENAMES_FROM_REWRITES; @@ -1334,8 +1334,6 @@ void test_diff_rename__many_files(void) int i, j; git_buf b = GIT_BUF_INIT; -/* - cl_git_pass(git_repository_index(&index, g_repo)); for (i = 0; i < 100; i += 1) { @@ -1354,7 +1352,6 @@ void test_diff_rename__many_files(void) git_index_free(index); test_with_many(100); -*/ } void test_diff_rename__again_many_files(void) @@ -1364,8 +1361,6 @@ void test_diff_rename__again_many_files(void) int i; git_buf b = GIT_BUF_INIT; -/* - cl_git_pass(git_repository_index(&index, g_repo)); git_buf_printf(&b, "%08d\n" ANOTHER_POEM "%08d\n" ANOTHER_POEM ANOTHER_POEM, 0, 0); @@ -1386,5 +1381,4 @@ void test_diff_rename__again_many_files(void) git_index_free(index); test_with_many(2500); -*/ } From e8242022bc409b3cb7e234eabdd9bda05ae3a158 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Mon, 5 Aug 2013 09:59:02 -0700 Subject: [PATCH 16/16] Move slow tests to "stress" clar module Create a new section of clar tests "stress" that will default to being off where we can put slow tests that push the library for performance testing purposes. --- CMakeLists.txt | 2 +- tests-clar/diff/rename.c | 147 ----------------------------------- tests-clar/stress/diff.c | 164 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 165 insertions(+), 148 deletions(-) create mode 100644 tests-clar/stress/diff.c diff --git a/CMakeLists.txt b/CMakeLists.txt index c937ba93c..53f568ed3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -361,7 +361,7 @@ IF (BUILD_CLAR) ADD_CUSTOM_COMMAND( OUTPUT ${CLAR_PATH}/clar.suite - COMMAND ${PYTHON_EXECUTABLE} generate.py -f -xonline . + COMMAND ${PYTHON_EXECUTABLE} generate.py -f -xonline -xstress . DEPENDS ${SRC_TEST} WORKING_DIRECTORY ${CLAR_PATH} ) diff --git a/tests-clar/diff/rename.c b/tests-clar/diff/rename.c index 55555012a..5a35495f7 100644 --- a/tests-clar/diff/rename.c +++ b/tests-clar/diff/rename.c @@ -1235,150 +1235,3 @@ void test_diff_rename__unmodified_can_be_renamed(void) git_index_free(index); git_tree_free(tree); } - -#define ANOTHER_POEM \ -"OH, glorious are the guarded heights\nWhere guardian souls abide—\nSelf-exiled from our gross delights—\nAbove, beyond, outside:\nAn ampler arc their spirit swings—\nCommands a juster view—\nWe have their word for all these things,\nNo doubt their words are true.\n\nYet we, the bond slaves of our day,\nWhom dirt and danger press—\nCo-heirs of insolence, delay,\nAnd leagued unfaithfulness—\nSuch is our need must seek indeed\nAnd, having found, engage\nThe men who merely do the work\nFor which they draw the wage.\n\nFrom forge and farm and mine and bench,\nDeck, altar, outpost lone—\nMill, school, battalion, counter, trench,\nRail, senate, sheepfold, throne—\nCreation's cry goes up on high\nFrom age to cheated age:\n\"Send us the men who do the work\n\"For which they draw the wage!\"\n" - -static void test_with_many(size_t expected_new) -{ - git_index *index; - git_tree *tree, *new_tree; - git_diff_list *diff = NULL; - diff_expects exp; - git_diff_options diffopts = GIT_DIFF_OPTIONS_INIT; - git_diff_find_options opts = GIT_DIFF_FIND_OPTIONS_INIT; - - cl_git_pass(git_repository_index(&index, g_repo)); - cl_git_pass( - git_revparse_single((git_object **)&tree, g_repo, "HEAD^{tree}")); - - cl_git_pass(p_rename("renames/ikeepsix.txt", "renames/ikeepsix2.txt")); - cl_git_pass(git_index_remove_bypath(index, "ikeepsix.txt")); - cl_git_pass(git_index_add_bypath(index, "ikeepsix2.txt")); - cl_git_pass(git_index_write(index)); - - cl_git_pass(git_diff_tree_to_index(&diff, g_repo, tree, index, &diffopts)); - - memset(&exp, 0, sizeof(exp)); - cl_git_pass(git_diff_foreach( - diff, diff_file_cb, NULL, NULL, &exp)); - cl_assert_equal_i(1, exp.file_status[GIT_DELTA_DELETED]); - cl_assert_equal_i(expected_new + 1, exp.file_status[GIT_DELTA_ADDED]); - cl_assert_equal_i(expected_new + 2, exp.files); - - opts.flags = GIT_DIFF_FIND_ALL; - cl_git_pass(git_diff_find_similar(diff, &opts)); - - memset(&exp, 0, sizeof(exp)); - cl_git_pass(git_diff_foreach( - diff, diff_file_cb, NULL, NULL, &exp)); - cl_assert_equal_i(1, exp.file_status[GIT_DELTA_RENAMED]); - cl_assert_equal_i(expected_new, exp.file_status[GIT_DELTA_ADDED]); - cl_assert_equal_i(expected_new + 1, exp.files); - - git_diff_list_free(diff); - - { - git_object *parent; - git_signature *sig; - git_oid tree_id, commit_id; - git_reference *ref; - - cl_git_pass(git_index_write_tree(&tree_id, index)); - cl_git_pass(git_tree_lookup(&new_tree, g_repo, &tree_id)); - - cl_git_pass(git_revparse_ext(&parent, &ref, g_repo, "HEAD")); - cl_git_pass(git_signature_new( - &sig, "Sm Test", "sm@tester.test", 1372350000, 480)); - - cl_git_pass(git_commit_create_v( - &commit_id, g_repo, git_reference_name(ref), sig, sig, - NULL, "yoyoyo", new_tree, 1, parent)); - - git_object_free(parent); - git_reference_free(ref); - git_signature_free(sig); - } - - cl_git_pass(git_diff_tree_to_tree( - &diff, g_repo, tree, new_tree, &diffopts)); - - memset(&exp, 0, sizeof(exp)); - cl_git_pass(git_diff_foreach( - diff, diff_file_cb, NULL, NULL, &exp)); - cl_assert_equal_i(1, exp.file_status[GIT_DELTA_DELETED]); - cl_assert_equal_i(expected_new + 1, exp.file_status[GIT_DELTA_ADDED]); - cl_assert_equal_i(expected_new + 2, exp.files); - - opts.flags = GIT_DIFF_FIND_ALL; - cl_git_pass(git_diff_find_similar(diff, &opts)); - - memset(&exp, 0, sizeof(exp)); - cl_git_pass(git_diff_foreach( - diff, diff_file_cb, NULL, NULL, &exp)); - cl_assert_equal_i(1, exp.file_status[GIT_DELTA_RENAMED]); - cl_assert_equal_i(expected_new, exp.file_status[GIT_DELTA_ADDED]); - cl_assert_equal_i(expected_new + 1, exp.files); - - git_diff_list_free(diff); - - git_tree_free(new_tree); - git_tree_free(tree); - git_index_free(index); -} - -void test_diff_rename__many_files(void) -{ - git_index *index; - char tmp[64]; - int i, j; - git_buf b = GIT_BUF_INIT; - - cl_git_pass(git_repository_index(&index, g_repo)); - - for (i = 0; i < 100; i += 1) { - snprintf(tmp, sizeof(tmp), "renames/newfile%03d", i); - for (j = i * 256; j > 0; --j) - git_buf_printf(&b, "more content %d\n", i); - cl_git_mkfile(tmp, b.ptr); - } - - for (i = 0; i < 100; i += 1) { - snprintf(tmp, sizeof(tmp), "renames/newfile%03d", i); - cl_git_pass(git_index_add_bypath(index, tmp + strlen("renames/"))); - } - - git_buf_free(&b); - git_index_free(index); - - test_with_many(100); -} - -void test_diff_rename__again_many_files(void) -{ - git_index *index; - char tmp[64]; - int i; - git_buf b = GIT_BUF_INIT; - - cl_git_pass(git_repository_index(&index, g_repo)); - - git_buf_printf(&b, "%08d\n" ANOTHER_POEM "%08d\n" ANOTHER_POEM ANOTHER_POEM, 0, 0); - - for (i = 0; i < 2500; i += 1) { - snprintf(tmp, sizeof(tmp), "renames/newfile%03d", i); - snprintf(b.ptr, 9, "%08d", i); - b.ptr[8] = '\n'; - cl_git_mkfile(tmp, b.ptr); - } - git_buf_free(&b); - - for (i = 0; i < 2500; i += 1) { - snprintf(tmp, sizeof(tmp), "renames/newfile%03d", i); - cl_git_pass(git_index_add_bypath(index, tmp + strlen("renames/"))); - } - - git_index_free(index); - - test_with_many(2500); -} diff --git a/tests-clar/stress/diff.c b/tests-clar/stress/diff.c new file mode 100644 index 000000000..62ccd5ec7 --- /dev/null +++ b/tests-clar/stress/diff.c @@ -0,0 +1,164 @@ +#include "clar_libgit2.h" +#include "../diff/diff_helpers.h" + +static git_repository *g_repo = NULL; + +void test_stress_diff__initialize(void) +{ +} + +void test_stress_diff__cleanup(void) +{ + cl_git_sandbox_cleanup(); +} + +#define ANOTHER_POEM \ +"OH, glorious are the guarded heights\nWhere guardian souls abide—\nSelf-exiled from our gross delights—\nAbove, beyond, outside:\nAn ampler arc their spirit swings—\nCommands a juster view—\nWe have their word for all these things,\nNo doubt their words are true.\n\nYet we, the bond slaves of our day,\nWhom dirt and danger press—\nCo-heirs of insolence, delay,\nAnd leagued unfaithfulness—\nSuch is our need must seek indeed\nAnd, having found, engage\nThe men who merely do the work\nFor which they draw the wage.\n\nFrom forge and farm and mine and bench,\nDeck, altar, outpost lone—\nMill, school, battalion, counter, trench,\nRail, senate, sheepfold, throne—\nCreation's cry goes up on high\nFrom age to cheated age:\n\"Send us the men who do the work\n\"For which they draw the wage!\"\n" + +static void test_with_many(size_t expected_new) +{ + git_index *index; + git_tree *tree, *new_tree; + git_diff_list *diff = NULL; + diff_expects exp; + git_diff_options diffopts = GIT_DIFF_OPTIONS_INIT; + git_diff_find_options opts = GIT_DIFF_FIND_OPTIONS_INIT; + + cl_git_pass(git_repository_index(&index, g_repo)); + cl_git_pass( + git_revparse_single((git_object **)&tree, g_repo, "HEAD^{tree}")); + + cl_git_pass(p_rename("renames/ikeepsix.txt", "renames/ikeepsix2.txt")); + cl_git_pass(git_index_remove_bypath(index, "ikeepsix.txt")); + cl_git_pass(git_index_add_bypath(index, "ikeepsix2.txt")); + cl_git_pass(git_index_write(index)); + + cl_git_pass(git_diff_tree_to_index(&diff, g_repo, tree, index, &diffopts)); + + memset(&exp, 0, sizeof(exp)); + cl_git_pass(git_diff_foreach( + diff, diff_file_cb, NULL, NULL, &exp)); + cl_assert_equal_i(1, exp.file_status[GIT_DELTA_DELETED]); + cl_assert_equal_i(expected_new + 1, exp.file_status[GIT_DELTA_ADDED]); + cl_assert_equal_i(expected_new + 2, exp.files); + + opts.flags = GIT_DIFF_FIND_ALL; + cl_git_pass(git_diff_find_similar(diff, &opts)); + + memset(&exp, 0, sizeof(exp)); + cl_git_pass(git_diff_foreach( + diff, diff_file_cb, NULL, NULL, &exp)); + cl_assert_equal_i(1, exp.file_status[GIT_DELTA_RENAMED]); + cl_assert_equal_i(expected_new, exp.file_status[GIT_DELTA_ADDED]); + cl_assert_equal_i(expected_new + 1, exp.files); + + git_diff_list_free(diff); + + { + git_object *parent; + git_signature *sig; + git_oid tree_id, commit_id; + git_reference *ref; + + cl_git_pass(git_index_write_tree(&tree_id, index)); + cl_git_pass(git_tree_lookup(&new_tree, g_repo, &tree_id)); + + cl_git_pass(git_revparse_ext(&parent, &ref, g_repo, "HEAD")); + cl_git_pass(git_signature_new( + &sig, "Sm Test", "sm@tester.test", 1372350000, 480)); + + cl_git_pass(git_commit_create_v( + &commit_id, g_repo, git_reference_name(ref), sig, sig, + NULL, "yoyoyo", new_tree, 1, parent)); + + git_object_free(parent); + git_reference_free(ref); + git_signature_free(sig); + } + + cl_git_pass(git_diff_tree_to_tree( + &diff, g_repo, tree, new_tree, &diffopts)); + + memset(&exp, 0, sizeof(exp)); + cl_git_pass(git_diff_foreach( + diff, diff_file_cb, NULL, NULL, &exp)); + cl_assert_equal_i(1, exp.file_status[GIT_DELTA_DELETED]); + cl_assert_equal_i(expected_new + 1, exp.file_status[GIT_DELTA_ADDED]); + cl_assert_equal_i(expected_new + 2, exp.files); + + opts.flags = GIT_DIFF_FIND_ALL; + cl_git_pass(git_diff_find_similar(diff, &opts)); + + memset(&exp, 0, sizeof(exp)); + cl_git_pass(git_diff_foreach( + diff, diff_file_cb, NULL, NULL, &exp)); + cl_assert_equal_i(1, exp.file_status[GIT_DELTA_RENAMED]); + cl_assert_equal_i(expected_new, exp.file_status[GIT_DELTA_ADDED]); + cl_assert_equal_i(expected_new + 1, exp.files); + + git_diff_list_free(diff); + + git_tree_free(new_tree); + git_tree_free(tree); + git_index_free(index); +} + +void test_stress_diff__rename_big_files(void) +{ + git_index *index; + char tmp[64]; + int i, j; + git_buf b = GIT_BUF_INIT; + + g_repo = cl_git_sandbox_init("renames"); + + cl_git_pass(git_repository_index(&index, g_repo)); + + for (i = 0; i < 100; i += 1) { + snprintf(tmp, sizeof(tmp), "renames/newfile%03d", i); + for (j = i * 256; j > 0; --j) + git_buf_printf(&b, "more content %d\n", i); + cl_git_mkfile(tmp, b.ptr); + } + + for (i = 0; i < 100; i += 1) { + snprintf(tmp, sizeof(tmp), "renames/newfile%03d", i); + cl_git_pass(git_index_add_bypath(index, tmp + strlen("renames/"))); + } + + git_buf_free(&b); + git_index_free(index); + + test_with_many(100); +} + +void test_stress_diff__rename_many_files(void) +{ + git_index *index; + char tmp[64]; + int i; + git_buf b = GIT_BUF_INIT; + + g_repo = cl_git_sandbox_init("renames"); + + cl_git_pass(git_repository_index(&index, g_repo)); + + git_buf_printf(&b, "%08d\n" ANOTHER_POEM "%08d\n" ANOTHER_POEM ANOTHER_POEM, 0, 0); + + for (i = 0; i < 2500; i += 1) { + snprintf(tmp, sizeof(tmp), "renames/newfile%03d", i); + snprintf(b.ptr, 9, "%08d", i); + b.ptr[8] = '\n'; + cl_git_mkfile(tmp, b.ptr); + } + git_buf_free(&b); + + for (i = 0; i < 2500; i += 1) { + snprintf(tmp, sizeof(tmp), "renames/newfile%03d", i); + cl_git_pass(git_index_add_bypath(index, tmp + strlen("renames/"))); + } + + git_index_free(index); + + test_with_many(2500); +}