diff --git a/CHANGELOG.md b/CHANGELOG.md index 96bd9a16e..e1c02f965 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ v0.22 + 1 ### Changes or improvements +* Updated binary identification in CRLF filtering to avoid false positives in + UTF-8 files. + * Rename and copy detection is enabled for small files. ### API additions diff --git a/src/buf_text.c b/src/buf_text.c index cead599f4..cb3661edb 100644 --- a/src/buf_text.c +++ b/src/buf_text.c @@ -191,7 +191,10 @@ bool git_buf_text_is_binary(const git_buf *buf) while (scan < end) { unsigned char c = *scan++; - if (c > 0x1F && c < 0x7F) + /* Printable characters are those above SPACE (0x1F) excluding DEL, + * and including BS, ESC and FF. + */ + if ((c > 0x1F && c != 127) || c == '\b' || c == '\033' || c == '\014') printable++; else if (c == '\0') return true; diff --git a/tests/checkout/crlf.c b/tests/checkout/crlf.c index 496f83d5d..b6d4e949a 100644 --- a/tests/checkout/crlf.c +++ b/tests/checkout/crlf.c @@ -106,6 +106,31 @@ void test_checkout_crlf__all_crlf_autocrlf_true(void) check_file_contents("./crlf/all-crlf", ALL_CRLF_TEXT_RAW); } +void test_checkout_crlf__detect_crlf_autocrlf_true_utf8(void) +{ + git_checkout_options opts = GIT_CHECKOUT_OPTIONS_INIT; + opts.checkout_strategy = GIT_CHECKOUT_SAFE_CREATE; + + cl_repo_set_bool(g_repo, "core.autocrlf", true); + + git_repository_set_head(g_repo, "refs/heads/utf8", NULL, NULL); + git_checkout_head(g_repo, &opts); + + if (GIT_EOL_NATIVE == GIT_EOL_LF) + { + check_file_contents("./crlf/few-utf8-chars-lf.txt", FEW_UTF8_LF_RAW); + check_file_contents("./crlf/many-utf8-chars-lf.txt", MANY_UTF8_LF_RAW); + } + else + { + check_file_contents("./crlf/few-utf8-chars-lf.txt", FEW_UTF8_CRLF_RAW); + check_file_contents("./crlf/many-utf8-chars-lf.txt", MANY_UTF8_CRLF_RAW); + } + + check_file_contents("./crlf/few-utf8-chars-crlf.txt", FEW_UTF8_CRLF_RAW); + check_file_contents("./crlf/many-utf8-chars-crlf.txt", MANY_UTF8_CRLF_RAW); +} + void test_checkout_crlf__autocrlf_true_index_size_is_filtered_size(void) { git_index *index; diff --git a/tests/core/buffer.c b/tests/core/buffer.c index 87dec4607..d28aa218f 100644 --- a/tests/core/buffer.c +++ b/tests/core/buffer.c @@ -830,7 +830,7 @@ void test_core_buffer__classify_with_utf8(void) cl_assert(!git_buf_text_contains_nul(&b)); b.ptr = data1; b.size = b.asize = data1len; - cl_assert(git_buf_text_is_binary(&b)); + cl_assert(!git_buf_text_is_binary(&b)); cl_assert(!git_buf_text_contains_nul(&b)); b.ptr = data2; b.size = b.asize = data2len; diff --git a/tests/filter/crlf.h b/tests/filter/crlf.h index 9cb98ad4c..786edfc96 100644 --- a/tests/filter/crlf.h +++ b/tests/filter/crlf.h @@ -22,4 +22,9 @@ #define MORE_CRLF_TEXT_AS_LF "crlf\ncrlf\nlf\ncrlf\ncrlf\n" #define MORE_LF_TEXT_AS_LF "lf\nlf\ncrlf\nlf\nlf\n" +#define FEW_UTF8_CRLF_RAW "\xe2\x9a\xbdThe rest is ASCII01.\r\nThe rest is ASCII02.\r\nThe rest is ASCII03.\r\nThe rest is ASCII04.\r\nThe rest is ASCII05.\r\nThe rest is ASCII06.\r\nThe rest is ASCII07.\r\nThe rest is ASCII08.\r\nThe rest is ASCII09.\r\nThe rest is ASCII10.\r\nThe rest is ASCII11.\r\nThe rest is ASCII12.\r\nThe rest is ASCII13.\r\nThe rest is ASCII14.\r\nThe rest is ASCII15.\r\nThe rest is ASCII16.\r\nThe rest is ASCII17.\r\nThe rest is ASCII18.\r\nThe rest is ASCII19.\r\nThe rest is ASCII20.\r\nThe rest is ASCII21.\r\nThe rest is ASCII22.\r\n" +#define FEW_UTF8_LF_RAW "\xe2\x9a\xbdThe rest is ASCII01.\nThe rest is ASCII02.\nThe rest is ASCII03.\nThe rest is ASCII04.\nThe rest is ASCII05.\nThe rest is ASCII06.\nThe rest is ASCII07.\nThe rest is ASCII08.\nThe rest is ASCII09.\nThe rest is ASCII10.\nThe rest is ASCII11.\nThe rest is ASCII12.\nThe rest is ASCII13.\nThe rest is ASCII14.\nThe rest is ASCII15.\nThe rest is ASCII16.\nThe rest is ASCII17.\nThe rest is ASCII18.\nThe rest is ASCII19.\nThe rest is ASCII20.\nThe rest is ASCII21.\nThe rest is ASCII22.\n" +#define MANY_UTF8_CRLF_RAW "Lets sing!\r\n\xe2\x99\xab\xe2\x99\xaa\xe2\x99\xac\xe2\x99\xa9\r\nEat food\r\n\xf0\x9f\x8d\x85\xf0\x9f\x8d\x95\r\n" +#define MANY_UTF8_LF_RAW "Lets sing!\n\xe2\x99\xab\xe2\x99\xaa\xe2\x99\xac\xe2\x99\xa9\nEat food\n\xf0\x9f\x8d\x85\xf0\x9f\x8d\x95\n" + #endif diff --git a/tests/resources/crlf/.gitted/objects/0e/052888828a954ca17e5882638e3c6a083e75c0 b/tests/resources/crlf/.gitted/objects/0e/052888828a954ca17e5882638e3c6a083e75c0 new file mode 100644 index 000000000..746143f85 Binary files /dev/null and b/tests/resources/crlf/.gitted/objects/0e/052888828a954ca17e5882638e3c6a083e75c0 differ diff --git a/tests/resources/crlf/.gitted/objects/9a/6c3533fef19abd6eec8e61206b5c51982b80d9 b/tests/resources/crlf/.gitted/objects/9a/6c3533fef19abd6eec8e61206b5c51982b80d9 new file mode 100644 index 000000000..78fc8aeb7 Binary files /dev/null and b/tests/resources/crlf/.gitted/objects/9a/6c3533fef19abd6eec8e61206b5c51982b80d9 differ diff --git a/tests/resources/crlf/.gitted/objects/a2/34455d62297f1856c4603686150c59fcb0aafe b/tests/resources/crlf/.gitted/objects/a2/34455d62297f1856c4603686150c59fcb0aafe new file mode 100644 index 000000000..7d204f4c8 Binary files /dev/null and b/tests/resources/crlf/.gitted/objects/a2/34455d62297f1856c4603686150c59fcb0aafe differ diff --git a/tests/resources/crlf/.gitted/objects/c3/e11722855ff260bd27418988ac1467c4e9e73a b/tests/resources/crlf/.gitted/objects/c3/e11722855ff260bd27418988ac1467c4e9e73a new file mode 100644 index 000000000..5f96dc76c Binary files /dev/null and b/tests/resources/crlf/.gitted/objects/c3/e11722855ff260bd27418988ac1467c4e9e73a differ diff --git a/tests/resources/crlf/.gitted/objects/cd/574f5a2baa4c79504f8837b730fa0b11defe99 b/tests/resources/crlf/.gitted/objects/cd/574f5a2baa4c79504f8837b730fa0b11defe99 new file mode 100644 index 000000000..e8d020246 Binary files /dev/null and b/tests/resources/crlf/.gitted/objects/cd/574f5a2baa4c79504f8837b730fa0b11defe99 differ diff --git a/tests/resources/crlf/.gitted/objects/f4/d25b796d86387205a5498175d66e91d1e5006a b/tests/resources/crlf/.gitted/objects/f4/d25b796d86387205a5498175d66e91d1e5006a new file mode 100644 index 000000000..792b1659d Binary files /dev/null and b/tests/resources/crlf/.gitted/objects/f4/d25b796d86387205a5498175d66e91d1e5006a differ diff --git a/tests/resources/crlf/.gitted/refs/heads/utf8 b/tests/resources/crlf/.gitted/refs/heads/utf8 index f8e6cf51c..30ab61cd3 100644 Binary files a/tests/resources/crlf/.gitted/refs/heads/utf8 and b/tests/resources/crlf/.gitted/refs/heads/utf8 differ