diff: don't feed large files to xdiff

src/checkout.c

@@ -18,6 +18,7 @@
 #include "git2/submodule.h"
 #include "git2/sys/index.h"
 #include "git2/sys/filter.h"
+#include "git2/merge.h"
 
 #include "refs.h"
 #include "repository.h"
@@ -27,7 +28,7 @@
 #include "diff.h"
 #include "pathspec.h"
 #include "buf_text.h"
-#include "merge_file.h"
+#include "diff_xdiff.h"
 #include "path.h"
 #include "attr.h"
 #include "pool.h"

src/diff_patch.c

@@ -30,6 +30,10 @@ static void diff_patch_update_binary(git_patch *patch)
 		(patch->nfile.file->flags & GIT_DIFF_FLAG_BINARY) != 0)
 		patch->delta->flags |= GIT_DIFF_FLAG_BINARY;
 
+	else if (patch->ofile.file->size > GIT_XDIFF_MAX_SIZE ||
+			 patch->nfile.file->size > GIT_XDIFF_MAX_SIZE)
+		patch->delta->flags |= GIT_DIFF_FLAG_BINARY;
+
 	else if ((patch->ofile.file->flags & DIFF_FLAGS_NOT_BINARY) != 0 &&
 			 (patch->nfile.file->flags & DIFF_FLAGS_NOT_BINARY) != 0)
 		patch->delta->flags |= GIT_DIFF_FLAG_NOT_BINARY;

src/diff_xdiff.c

@@ -4,6 +4,7 @@
  * This file is part of libgit2, distributed under the GNU GPL v2 with
  * a Linking Exception. For full terms see the included COPYING file.
  */
+#include "git2/errors.h"
 #include "common.h"
 #include "diff.h"
 #include "diff_driver.h"
@@ -208,6 +209,12 @@ static int git_xdiff(git_diff_output *output, git_patch *patch)
 	git_patch__old_data(&info.xd_old_data.ptr, &info.xd_old_data.size, patch);
 	git_patch__new_data(&info.xd_new_data.ptr, &info.xd_new_data.size, patch);
 
+	if (info.xd_old_data.size > GIT_XDIFF_MAX_SIZE ||
+		info.xd_new_data.size > GIT_XDIFF_MAX_SIZE) {
+		giterr_set(GITERR_INVALID, "files too large for diff");
+		return -1;
+	}
+
 	xdl_diff(&info.xd_old_data, &info.xd_new_data,
 		&xo->params, &xo->config, &xo->callback);
 

src/diff_xdiff.h

@@ -11,6 +11,11 @@
 #include "diff_patch.h"
 #include "xdiff/xdiff.h"
 
+/* xdiff cannot cope with large files.  these files should not be passed to
+ * xdiff.  callers should treat these large files as binary.
+ */
+#define GIT_XDIFF_MAX_SIZE (1024LL * 1024 * 1023)
+
 /* A git_xdiff_output is a git_diff_output with extra fields necessary
  * to use libxdiff.  Calling git_xdiff_init() will set the diff_cb field
  * of the output to use xdiff to generate the diffs.

src/merge.c

@@ -20,7 +20,6 @@
 #include "diff.h"
 #include "checkout.h"
 #include "tree.h"
-#include "merge_file.h"
 #include "blob.h"
 #include "oid.h"
 #include "index.h"

src/merge_file.c

@@ -7,10 +7,10 @@
 
 #include "common.h"
 #include "repository.h"
-#include "merge_file.h"
 #include "posix.h"
 #include "fileops.h"
 #include "index.h"
+#include "diff_xdiff.h"
 
 #include "git2/repository.h"
 #include "git2/object.h"
@@ -199,7 +199,7 @@ static bool merge_file__is_binary(const git_merge_file_input *file)
 {
 	size_t len = file ? file->size : 0;
 
-	if (len > GIT_MERGE_FILE_XDIFF_MAX)
+	if (len > GIT_XDIFF_MAX_SIZE)
 		return true;
 	if (len > GIT_MERGE_FILE_BINARY_SIZE)
 		len = GIT_MERGE_FILE_BINARY_SIZE;

src/merge_file.h

@@ -1,13 +0,0 @@
-/*
- * Copyright (C) the libgit2 contributors. All rights reserved.
- *
- * This file is part of libgit2, distributed under the GNU GPL v2 with
- * a Linking Exception. For full terms see the included COPYING file.
- */
-#ifndef INCLUDE_filediff_h__
-#define INCLUDE_filediff_h__
-
-/* xdiff cannot cope with large files, just treat them as binary */
-#define GIT_MERGE_FILE_XDIFF_MAX (1024UL * 1024 * 1023)
-
-#endif

tests/merge/files.c

@@ -3,10 +3,10 @@
 #include "git2/merge.h"
 #include "buffer.h"
 #include "merge.h"
-#include "merge_file.h"
 #include "merge_helpers.h"
 #include "refs.h"
 #include "fileops.h"
+#include "diff_xdiff.h"
 
 #define TEST_REPO_PATH "merge-resolve"
 #define TEST_INDEX_PATH TEST_REPO_PATH "/.git/index"
@@ -296,11 +296,11 @@ void test_merge_files__skips_large_files(void)
 	git_merge_file_options opts = GIT_MERGE_FILE_OPTIONS_INIT;
 	git_merge_file_result result = {0};
 
-	ours.size = GIT_MERGE_FILE_XDIFF_MAX + 1;
+	ours.size = GIT_XDIFF_MAX_SIZE + 1;
 	ours.path = "testfile.txt";
 	ours.mode = 0100755;
 
-	theirs.size = GIT_MERGE_FILE_XDIFF_MAX + 1;
+	theirs.size = GIT_XDIFF_MAX_SIZE + 1;
 	theirs.path = "testfile.txt";
 	theirs.mode = 0100755;