Merge pull request #4119 from libgit2/ethomson/submodule_renames

rename detection: don't try to detect submodule renames

src/diff_tform.c

@@ -553,8 +553,8 @@ static int similarity_measure(
 
 	*score = -1;
 
-	/* don't try to compare files of different types */
-	if (GIT_MODE_TYPE(a_file->mode) != GIT_MODE_TYPE(b_file->mode))
+	/* don't try to compare things that aren't files */
+	if (!GIT_MODE_ISBLOB(a_file->mode) || !GIT_MODE_ISBLOB(b_file->mode))
 		return 0;
 
 	/* if exact match is requested, force calculation of missing OIDs now */

src/merge.c

@@ -1075,7 +1075,7 @@ static int index_entry_similarity_inexact(
 	int score = 0;
 	int error = 0;
 
-	if (GIT_MODE_TYPE(a->mode) != GIT_MODE_TYPE(b->mode))
+	if (!GIT_MODE_ISBLOB(a->mode) || !GIT_MODE_ISBLOB(b->mode))
 		return 0;
 
 	/* update signature cache if needed */

tests/merge/trees/renames.c

@@ -252,3 +252,25 @@ void test_merge_trees_renames__no_rename_index(void)
 
 	git_index_free(index);
 }
+
+void test_merge_trees_renames__submodules(void)
+{
+	git_index *index;
+	git_merge_options *opts = NULL;
+
+	struct merge_index_entry merge_index_entries[] = {
+		{ 0100644, "cd3e8d4aa06bdc781f264171030bc28f2b370fee", 0, ".gitmodules" },
+		{ 0100644, "4dd1ef7569b18d92d93c0a35bb6b93049137b355", 1, "file.txt" },
+		{ 0100644, "a2d8d1824c68541cca94ffb90f79291eba495921", 2, "file.txt" },
+		{ 0100644, "63ec604d491161ddafdae4179843c26d54bd999a", 3, "file.txt" },
+		{ 0160000, "0000000000000000000000000000000000000001", 1, "submodule1" },
+		{ 0160000, "0000000000000000000000000000000000000002", 3, "submodule1" },
+		{ 0160000, "0000000000000000000000000000000000000003", 0, "submodule2" },
+	};
+
+	cl_git_pass(merge_trees_from_branches(&index, repo,
+		"submodule_rename1", "submodule_rename2",
+		opts));
+	cl_assert(merge_test_index(index, merge_index_entries, 7));
+	git_index_free(index);
+}