From c67fd4c9d5e1ff715df28b884d7f7f9f20fad1ec Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Fri, 7 Feb 2014 11:20:36 -0800 Subject: [PATCH 01/20] Some vector utility tweaks This is just laying some groundwork for internal index changes that I'm working on. --- src/checkout.c | 8 ++++++-- src/index.c | 7 +++++-- src/merge.c | 6 ++++-- src/vector.c | 21 +++++++++++++++++++-- src/vector.h | 8 +++++++- tests/core/vector.c | 15 ++++++++------- tests/index/tests.c | 20 +++----------------- 7 files changed, 52 insertions(+), 33 deletions(-) diff --git a/src/checkout.c b/src/checkout.c index 141fc1331..f9bc5e9d8 100644 --- a/src/checkout.c +++ b/src/checkout.c @@ -653,10 +653,13 @@ static int checkout_conflictdata_cmp(const void *a, const void *b) return diff; } -int checkout_conflictdata_empty(const git_vector *conflicts, size_t idx) +int checkout_conflictdata_empty( + const git_vector *conflicts, size_t idx, void *payload) { checkout_conflictdata *conflict; + GIT_UNUSED(payload); + if ((conflict = git_vector_get(conflicts, idx)) == NULL) return -1; @@ -954,7 +957,8 @@ static int checkout_conflicts_coalesce_renames( ancestor_conflict->one_to_two = 1; } - git_vector_remove_matching(&data->conflicts, checkout_conflictdata_empty); + git_vector_remove_matching( + &data->conflicts, checkout_conflictdata_empty, NULL); done: return error; diff --git a/src/index.c b/src/index.c index b0b5eae9d..08c4b79b6 100644 --- a/src/index.c +++ b/src/index.c @@ -1228,10 +1228,13 @@ int git_index_conflict_remove(git_index *index, const char *path) return 0; } -static int index_conflicts_match(const git_vector *v, size_t idx) +static int index_conflicts_match(const git_vector *v, size_t idx, void *p) { + git_index *index = p; git_index_entry *entry = git_vector_get(v, idx); + GIT_UNUSED(index); + if (GIT_IDXENTRY_STAGE(entry) > 0) { index_entry_free(entry); return 1; @@ -1243,7 +1246,7 @@ static int index_conflicts_match(const git_vector *v, size_t idx) void git_index_conflict_cleanup(git_index *index) { assert(index); - git_vector_remove_matching(&index->entries, index_conflicts_match); + git_vector_remove_matching(&index->entries, index_conflicts_match, index); } int git_index_has_conflicts(const git_index *index) diff --git a/src/merge.c b/src/merge.c index 10c19b5c5..9c4a07b58 100644 --- a/src/merge.c +++ b/src/merge.c @@ -995,10 +995,12 @@ static void merge_diff_list_coalesce_renames( } } -static int merge_diff_empty(const git_vector *conflicts, size_t idx) +static int merge_diff_empty(const git_vector *conflicts, size_t idx, void *p) { git_merge_diff *conflict = conflicts->contents[idx]; + GIT_UNUSED(p); + return (!GIT_MERGE_INDEX_ENTRY_EXISTS(conflict->ancestor_entry) && !GIT_MERGE_INDEX_ENTRY_EXISTS(conflict->our_entry) && !GIT_MERGE_INDEX_ENTRY_EXISTS(conflict->their_entry)); @@ -1079,7 +1081,7 @@ int git_merge_diff_list__find_renames( merge_diff_list_coalesce_renames(diff_list, similarity_ours, similarity_theirs, opts); /* And remove any entries that were merged and are now empty. */ - git_vector_remove_matching(&diff_list->conflicts, merge_diff_empty); + git_vector_remove_matching(&diff_list->conflicts, merge_diff_empty, NULL); done: if (cache != NULL) { diff --git a/src/vector.c b/src/vector.c index 37ea07f0b..9f7eed5a3 100644 --- a/src/vector.c +++ b/src/vector.c @@ -276,14 +276,16 @@ void git_vector_uniq(git_vector *v, void (*git_free_cb)(void *)) } void git_vector_remove_matching( - git_vector *v, int (*match)(const git_vector *v, size_t idx)) + git_vector *v, + int (*match)(const git_vector *v, size_t idx, void *payload), + void *payload) { size_t i, j; for (i = 0, j = 0; j < v->length; ++j) { v->contents[i] = v->contents[j]; - if (!match(v, i)) + if (!match(v, i, payload)) i++; } @@ -339,3 +341,18 @@ int git_vector_set(void **old, git_vector *v, size_t position, void *value) return 0; } + +int git_vector_verify_sorted(const git_vector *v) +{ + size_t i; + + if (!git_vector_is_sorted(v)) + return -1; + + for (i = 1; i < v->length; ++i) { + if (v->_cmp(v->contents[i - 1], v->contents[i]) > 0) + return -1; + } + + return 0; +} diff --git a/src/vector.h b/src/vector.h index 682b6ad27..aac46c4b3 100644 --- a/src/vector.h +++ b/src/vector.h @@ -85,8 +85,11 @@ int git_vector_insert_sorted(git_vector *v, void *element, int git_vector_remove(git_vector *v, size_t idx); void git_vector_pop(git_vector *v); void git_vector_uniq(git_vector *v, void (*git_free_cb)(void *)); + void git_vector_remove_matching( - git_vector *v, int (*match)(const git_vector *v, size_t idx)); + git_vector *v, + int (*match)(const git_vector *v, size_t idx, void *payload), + void *payload); int git_vector_resize_to(git_vector *v, size_t new_length); int git_vector_set(void **old, git_vector *v, size_t position, void *value); @@ -108,4 +111,7 @@ GIT_INLINE(void) git_vector_set_cmp(git_vector *v, git_vector_cmp cmp) } } +/* Just use this in tests, not for realz. returns -1 if not sorted */ +int git_vector_verify_sorted(const git_vector *v); + #endif diff --git a/tests/core/vector.c b/tests/core/vector.c index db52c004f..66f90b82b 100644 --- a/tests/core/vector.c +++ b/tests/core/vector.c @@ -190,8 +190,9 @@ void test_core_vector__5(void) git_vector_free(&x); } -static int remove_ones(const git_vector *v, size_t idx) +static int remove_ones(const git_vector *v, size_t idx, void *p) { + GIT_UNUSED(p); return (git_vector_get(v, idx) == (void *)0x001); } @@ -206,7 +207,7 @@ void test_core_vector__remove_matching(void) git_vector_insert(&x, (void*) 0x001); cl_assert(x.length == 1); - git_vector_remove_matching(&x, remove_ones); + git_vector_remove_matching(&x, remove_ones, NULL); cl_assert(x.length == 0); git_vector_insert(&x, (void*) 0x001); @@ -214,7 +215,7 @@ void test_core_vector__remove_matching(void) git_vector_insert(&x, (void*) 0x001); cl_assert(x.length == 3); - git_vector_remove_matching(&x, remove_ones); + git_vector_remove_matching(&x, remove_ones, NULL); cl_assert(x.length == 0); git_vector_insert(&x, (void*) 0x002); @@ -223,7 +224,7 @@ void test_core_vector__remove_matching(void) git_vector_insert(&x, (void*) 0x001); cl_assert(x.length == 4); - git_vector_remove_matching(&x, remove_ones); + git_vector_remove_matching(&x, remove_ones, NULL); cl_assert(x.length == 2); git_vector_foreach(&x, i, compare) { @@ -238,7 +239,7 @@ void test_core_vector__remove_matching(void) git_vector_insert(&x, (void*) 0x001); cl_assert(x.length == 4); - git_vector_remove_matching(&x, remove_ones); + git_vector_remove_matching(&x, remove_ones, NULL); cl_assert(x.length == 2); git_vector_foreach(&x, i, compare) { @@ -253,7 +254,7 @@ void test_core_vector__remove_matching(void) git_vector_insert(&x, (void*) 0x001); cl_assert(x.length == 4); - git_vector_remove_matching(&x, remove_ones); + git_vector_remove_matching(&x, remove_ones, NULL); cl_assert(x.length == 2); git_vector_foreach(&x, i, compare) { @@ -268,7 +269,7 @@ void test_core_vector__remove_matching(void) git_vector_insert(&x, (void*) 0x003); cl_assert(x.length == 4); - git_vector_remove_matching(&x, remove_ones); + git_vector_remove_matching(&x, remove_ones, NULL); cl_assert(x.length == 4); git_vector_free(&x); diff --git a/tests/index/tests.c b/tests/index/tests.c index 6e28af1f7..fa5c0bb1a 100644 --- a/tests/index/tests.c +++ b/tests/index/tests.c @@ -544,36 +544,22 @@ void test_index_tests__corrupted_extension(void) cl_git_fail_with(git_index_open(&index, TEST_INDEXBAD_PATH), GIT_ERROR); } -static void assert_index_is_sorted(git_index *index) -{ - git_vector *entries = &index->entries; - size_t i; - - cl_assert(git_vector_is_sorted(entries)); - - for (i = 1; i < git_vector_length(entries); ++i) { - git_index_entry *prev = git_vector_get(entries, i - 1); - git_index_entry *curr = git_vector_get(entries, i); - cl_assert(index->entries._cmp(prev, curr) <= 0); - } -} - void test_index_tests__reload_while_ignoring_case(void) { git_index *index; unsigned int caps; cl_git_pass(git_index_open(&index, TEST_INDEX_PATH)); - assert_index_is_sorted(index); + cl_git_pass(git_vector_verify_sorted(&index->entries)); caps = git_index_caps(index); cl_git_pass(git_index_set_caps(index, caps &= ~GIT_INDEXCAP_IGNORE_CASE)); cl_git_pass(git_index_read(index, true)); - assert_index_is_sorted(index); + cl_git_pass(git_vector_verify_sorted(&index->entries)); cl_git_pass(git_index_set_caps(index, caps | GIT_INDEXCAP_IGNORE_CASE)); cl_git_pass(git_index_read(index, true)); - assert_index_is_sorted(index); + cl_git_pass(git_vector_verify_sorted(&index->entries)); git_index_free(index); } From 3dbee456564a9baf24631bfe219f81434d8fdfa6 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Fri, 7 Feb 2014 14:10:35 -0800 Subject: [PATCH 02/20] Some index internals refactoring Again, laying groundwork for some index iterator changes, this contains a bunch of code refactorings for index internals that should make it easier down the line to add locking around index modifications. Also this removes the redundant prefix_position function and fixes some potential memory leaks. --- include/git2/index.h | 15 +- src/checkout.c | 10 +- src/index.c | 360 +++++++++++++++++++++++-------------------- src/index.h | 6 +- src/iterator.c | 6 +- 5 files changed, 220 insertions(+), 177 deletions(-) diff --git a/include/git2/index.h b/include/git2/index.h index dd6a28e40..4d33f13d2 100644 --- a/include/git2/index.h +++ b/include/git2/index.h @@ -77,7 +77,12 @@ typedef struct git_index_entry { #define GIT_IDXENTRY_VALID (0x8000) #define GIT_IDXENTRY_STAGESHIFT 12 -#define GIT_IDXENTRY_STAGE(E) (((E)->flags & GIT_IDXENTRY_STAGEMASK) >> GIT_IDXENTRY_STAGESHIFT) +#define GIT_IDXENTRY_STAGE(E) \ + (((E)->flags & GIT_IDXENTRY_STAGEMASK) >> GIT_IDXENTRY_STAGESHIFT) + +#define GIT_IDXENTRY_STAGE_SET(E,S) do { \ + (E)->flags = ((E)->flags & ~GIT_IDXENTRY_STAGEMASK) | \ + (((S) & 0x03) << GIT_IDXENTRY_STAGESHIFT); } while (0) /** * Bitmasks for on-disk fields of `git_index_entry`'s `flags_extended` @@ -327,12 +332,14 @@ GIT_EXTERN(size_t) git_index_entrycount(const git_index *index); /** * Clear the contents (all the entries) of an index object. - * This clears the index object in memory; changes must be manually - * written to disk for them to take effect. + * + * This clears the index object in memory; changes must be explicitly + * written to disk for them to take effect persistently. * * @param index an existing index object + * @return 0 on success, error code < 0 on failure */ -GIT_EXTERN(void) git_index_clear(git_index *index); +GIT_EXTERN(int) git_index_clear(git_index *index); /** * Get a pointer to one of the entries in the index diff --git a/src/checkout.c b/src/checkout.c index f9bc5e9d8..a251c08c0 100644 --- a/src/checkout.c +++ b/src/checkout.c @@ -277,19 +277,23 @@ static int checkout_action_wd_only( /* check if item is tracked in the index but not in the checkout diff */ if (data->index != NULL) { + size_t pos; + + error = git_index__find( + &pos, data->index, wd->path, 0, GIT_INDEX_STAGE_ANY); + if (wd->mode != GIT_FILEMODE_TREE) { - if (!(error = git_index_find(NULL, data->index, wd->path))) { + if (!error) { /* found by git_index__find call */ notify = GIT_CHECKOUT_NOTIFY_DIRTY; remove = ((data->strategy & GIT_CHECKOUT_FORCE) != 0); } else if (error != GIT_ENOTFOUND) return error; else - giterr_clear(); + error = 0; /* git_index__find does not set error msg */ } else { /* for tree entries, we have to see if there are any index * entries that are contained inside that tree */ - size_t pos = git_index__prefix_position(data->index, wd->path); const git_index_entry *e = git_index_get_byindex(data->index, pos); if (e != NULL && data->diff->pfxcomp(e->path, wd->path) == 0) { diff --git a/src/index.c b/src/index.c index 08c4b79b6..8bc5cb13c 100644 --- a/src/index.c +++ b/src/index.c @@ -345,7 +345,7 @@ void git_index__set_ignore_case(git_index *index, bool ignore_case) int git_index_open(git_index **index_out, const char *index_path) { git_index *index; - int error; + int error = -1; assert(index_out); @@ -354,7 +354,8 @@ int git_index_open(git_index **index_out, const char *index_path) if (index_path != NULL) { index->index_file_path = git__strdup(index_path); - GITERR_CHECK_ALLOC(index->index_file_path); + if (!index->index_file_path) + goto fail; /* Check if index file is stored on disk already */ if (git_path_exists(index->index_file_path) == true) @@ -364,22 +365,24 @@ int git_index_open(git_index **index_out, const char *index_path) if (git_vector_init(&index->entries, 32, index_cmp) < 0 || git_vector_init(&index->names, 32, conflict_name_cmp) < 0 || git_vector_init(&index->reuc, 32, reuc_cmp) < 0) - return -1; + goto fail; index->entries_cmp_path = index_cmp_path; index->entries_search = index_srch; index->entries_search_path = index_srch_path; index->reuc_search = reuc_srch; - if ((index_path != NULL) && ((error = git_index_read(index, true)) < 0)) { - git_index_free(index); - return error; - } + if (index_path != NULL && (error = git_index_read(index, true)) < 0) + goto fail; *index_out = index; GIT_REFCOUNT_INC(index); return 0; + +fail: + git_index_free(index); + return error; } int git_index_new(git_index **out) @@ -418,18 +421,20 @@ static void index_entries_free(git_vector *entries) git_vector_clear(entries); } -void git_index_clear(git_index *index) +int git_index_clear(git_index *index) { assert(index); + git_tree_cache_free(index->tree); + index->tree = NULL; + index_entries_free(&index->entries); git_index_reuc_clear(index); git_index_name_clear(index); git_futils_filestamp_set(&index->stamp, NULL); - git_tree_cache_free(index->tree); - index->tree = NULL; + return 0; } static int create_index_error(int error, const char *msg) @@ -495,7 +500,7 @@ int git_index_read(git_index *index, int force) if (!index->on_disk) { if (force) - git_index_clear(index); + return git_index_clear(index); return 0; } @@ -507,8 +512,10 @@ int git_index_read(git_index *index, int force) if (error < 0) return error; - git_index_clear(index); - error = parse_index(index, buffer.ptr, buffer.size); + error = git_index_clear(index); + + if (!error) + error = parse_index(index, buffer.ptr, buffer.size); if (!error) git_futils_filestamp_set(&index->stamp, &stamp); @@ -679,12 +686,31 @@ static int index_entry_init( entry->id = oid; entry->path = git__strdup(rel_path); - GITERR_CHECK_ALLOC(entry->path); + if (!entry->path) { + git__free(entry); + return -1; + } *entry_out = entry; return 0; } +static int index_remove_entry(git_index *index, size_t pos) +{ + int error = 0; + git_index_entry *entry = git_vector_get(&index->entries, pos); + + if (entry != NULL) + git_tree_cache_invalidate_path(index->tree, entry->path); + + error = git_vector_remove(&index->entries, pos); + + if (!error) + index_entry_free(entry); + + return error; +} + static int index_entry_reuc_init(git_index_reuc_entry **reuc_out, const char *path, int ancestor_mode, const git_oid *ancestor_oid, @@ -701,8 +727,10 @@ static int index_entry_reuc_init(git_index_reuc_entry **reuc_out, GITERR_CHECK_ALLOC(reuc); reuc->path = git__strdup(path); - if (reuc->path == NULL) + if (reuc->path == NULL) { + git__free(reuc); return -1; + } if ((reuc->mode[0] = ancestor_mode) > 0) git_oid_cpy(&reuc->oid[0], ancestor_oid); @@ -717,22 +745,29 @@ static int index_entry_reuc_init(git_index_reuc_entry **reuc_out, return 0; } -static git_index_entry *index_entry_dup(const git_index_entry *source_entry) +static int index_entry_dup(git_index_entry **out, const git_index_entry *src) { git_index_entry *entry; - entry = git__malloc(sizeof(git_index_entry)); - if (!entry) - return NULL; + if (!src) { + *out = NULL; + return 0; + } - memcpy(entry, source_entry, sizeof(git_index_entry)); + entry = git__malloc(sizeof(git_index_entry)); + GITERR_CHECK_ALLOC(entry); + + memcpy(entry, src, sizeof(git_index_entry)); /* duplicate the path string so we own it */ entry->path = git__strdup(entry->path); - if (!entry->path) - return NULL; + if (!entry->path) { + git__free(entry); + return -1; + } - return entry; + *out = entry; + return 0; } static int has_file_name(git_index *index, @@ -757,7 +792,9 @@ static int has_file_name(git_index *index, retval = -1; if (!ok_to_replace) break; - git_vector_remove(&index->entries, --pos); + + if (index_remove_entry(index, --pos) < 0) + break; } return retval; } @@ -790,7 +827,8 @@ static int has_dir_name(git_index *index, if (!ok_to_replace) break; - git_vector_remove(&index->entries, position); + if (index_remove_entry(index, position) < 0) + break; continue; } @@ -830,12 +868,22 @@ static int check_file_directory_collision(git_index *index, return 0; } -static int index_insert(git_index *index, git_index_entry *entry, int replace) +/* index_insert takes ownership of the new entry - if it can't insert + * it, then it will return an error **and also free the entry**. When + * it replaces an existing entry, it will update the entry_ptr with the + * actual entry in the index (and free the passed in one). + */ +static int index_insert( + git_index *index, git_index_entry **entry_ptr, int replace) { + int error = 0; size_t path_length, position; - git_index_entry **existing = NULL; + git_index_entry *existing = NULL, *entry; - assert(index && entry && entry->path != NULL); + assert(index && entry_ptr); + + entry = *entry_ptr; + assert(entry && entry->path); /* make sure that the path length flag is correct */ path_length = strlen(entry->path); @@ -850,27 +898,41 @@ static int index_insert(git_index *index, git_index_entry *entry, int replace) /* look if an entry with this path already exists */ if (!git_index__find( &position, index, entry->path, 0, GIT_IDXENTRY_STAGE(entry))) { - existing = (git_index_entry **)&index->entries.contents[position]; + existing = index->entries.contents[position]; /* update filemode to existing values if stat is not trusted */ - entry->mode = index_merge_mode(index, *existing, entry->mode); + entry->mode = index_merge_mode(index, existing, entry->mode); } - if (check_file_directory_collision(index, entry, position, replace) < 0) - return -1; + error = check_file_directory_collision(index, entry, position, replace); + if (error < 0) + goto done; + + /* if we are replacing an existing item, overwrite the existing entry + * and return it in place of the passed in one. + */ + if (existing && replace) { + git__free(entry->path); + entry->path = existing->path; + + memcpy(existing, entry, sizeof(*entry)); + *entry_ptr = existing; + + git__free(entry); + return 0; + } /* if replacing is not requested or no existing entry exists, just * insert entry at the end; the index is no longer sorted */ - if (!replace || !existing) - return git_vector_insert(&index->entries, entry); + error = git_vector_insert(&index->entries, entry); - /* exists, replace it (preserving name from existing entry) */ - git__free(entry->path); - entry->path = (*existing)->path; - git__free(*existing); - *existing = entry; +done: + if (error < 0) { + index_entry_free(*entry_ptr); + *entry_ptr = NULL; + } - return 0; + return error; } static int index_conflict_to_reuc(git_index *index, const char *path) @@ -907,19 +969,15 @@ int git_index_add_bypath(git_index *index, const char *path) assert(index && path); if ((ret = index_entry_init(&entry, index, path)) < 0 || - (ret = index_insert(index, entry, 1)) < 0) - goto on_error; + (ret = index_insert(index, &entry, 1)) < 0) + return ret; /* Adding implies conflict was resolved, move conflict entries to REUC */ if ((ret = index_conflict_to_reuc(index, path)) < 0 && ret != GIT_ENOTFOUND) - goto on_error; + return ret; git_tree_cache_invalidate_path(index->tree, entry->path); return 0; - -on_error: - index_entry_free(entry); - return ret; } int git_index_remove_bypath(git_index *index, const char *path) @@ -942,14 +1000,11 @@ int git_index_add(git_index *index, const git_index_entry *source_entry) git_index_entry *entry = NULL; int ret; - entry = index_entry_dup(source_entry); - if (entry == NULL) - return -1; + assert(index && source_entry); - if ((ret = index_insert(index, entry, 1)) < 0) { - index_entry_free(entry); + if ((ret = index_entry_dup(&entry, source_entry)) < 0 || + (ret = index_insert(index, &entry, 1)) < 0) return ret; - } git_tree_cache_invalidate_path(index->tree, entry->path); return 0; @@ -958,8 +1013,6 @@ int git_index_add(git_index *index, const git_index_entry *source_entry) int git_index_remove(git_index *index, const char *path, int stage) { size_t position; - int error; - git_index_entry *entry; if (git_index__find(&position, index, path, 0, stage) < 0) { giterr_set(GITERR_INDEX, "Index does not contain %s at stage %d", @@ -967,16 +1020,7 @@ int git_index_remove(git_index *index, const char *path, int stage) return GIT_ENOTFOUND; } - entry = git_vector_get(&index->entries, position); - if (entry != NULL) - git_tree_cache_invalidate_path(index->tree, entry->path); - - error = git_vector_remove(&index->entries, position); - - if (!error) - index_entry_free(entry); - - return error; + return index_remove_entry(index, position); } int git_index_remove_directory(git_index *index, const char *dir, int stage) @@ -989,9 +1033,7 @@ int git_index_remove_directory(git_index *index, const char *dir, int stage) if (git_buf_sets(&pfx, dir) < 0 || git_path_to_dir(&pfx) < 0) return -1; - git_vector_sort(&index->entries); - - pos = git_index__prefix_position(index, pfx.ptr); + git_index__find(&pos, index, pfx.ptr, pfx.size, GIT_INDEX_STAGE_ANY); while (1) { entry = git_vector_get(&index->entries, pos); @@ -1003,11 +1045,8 @@ int git_index_remove_directory(git_index *index, const char *dir, int stage) continue; } - git_tree_cache_invalidate_path(index->tree, entry->path); - - if ((error = git_vector_remove(&index->entries, pos)) < 0) + if ((error = index_remove_entry(index, pos)) < 0) break; - index_entry_free(entry); /* removed entry at 'pos' so we don't need to increment it */ } @@ -1063,22 +1102,6 @@ int git_index_find(size_t *at_pos, git_index *index, const char *path) return 0; } -size_t git_index__prefix_position(git_index *index, const char *path) -{ - struct entry_srch_key srch_key; - size_t pos; - - srch_key.path = path; - srch_key.path_len = strlen(path); - srch_key.stage = 0; - - git_vector_sort(&index->entries); - git_vector_bsearch2( - &pos, &index->entries, index->entries_search, &srch_key); - - return pos; -} - int git_index_conflict_add(git_index *index, const git_index_entry *ancestor_entry, const git_index_entry *our_entry, @@ -1090,21 +1113,22 @@ int git_index_conflict_add(git_index *index, assert (index); - if ((ancestor_entry != NULL && (entries[0] = index_entry_dup(ancestor_entry)) == NULL) || - (our_entry != NULL && (entries[1] = index_entry_dup(our_entry)) == NULL) || - (their_entry != NULL && (entries[2] = index_entry_dup(their_entry)) == NULL)) - return -1; + if ((ret = index_entry_dup(&entries[0], ancestor_entry)) < 0 || + (ret = index_entry_dup(&entries[1], our_entry)) < 0 || + (ret = index_entry_dup(&entries[2], their_entry)) < 0) + goto on_error; for (i = 0; i < 3; i++) { if (entries[i] == NULL) continue; /* Make sure stage is correct */ - entries[i]->flags = (entries[i]->flags & ~GIT_IDXENTRY_STAGEMASK) | - ((i+1) << GIT_IDXENTRY_STAGESHIFT); + GIT_IDXENTRY_STAGE_SET(entries[i], i + 1); - if ((ret = index_insert(index, entries[i], 1)) < 0) + if ((ret = index_insert(index, &entries[i], 1)) < 0) goto on_error; + + entries[i] = NULL; /* don't free if later entry fails */ } return 0; @@ -1196,7 +1220,7 @@ int git_index_conflict_get( int git_index_conflict_remove(git_index *index, const char *path) { - size_t pos, posmax; + size_t pos; git_index_entry *conflict_entry; int error = 0; @@ -1205,10 +1229,7 @@ int git_index_conflict_remove(git_index *index, const char *path) if (git_index_find(&pos, index, path) < 0) return GIT_ENOTFOUND; - posmax = git_index_entrycount(index); - - while (pos < posmax) { - conflict_entry = git_vector_get(&index->entries, pos); + while ((conflict_entry = git_vector_get(&index->entries, pos)) != NULL) { if (index->entries_cmp_path(conflict_entry->path, path) != 0) break; @@ -1218,14 +1239,11 @@ int git_index_conflict_remove(git_index *index, const char *path) continue; } - if ((error = git_vector_remove(&index->entries, pos)) < 0) - return error; - - index_entry_free(conflict_entry); - posmax--; + if ((error = index_remove_entry(index, pos)) < 0) + break; } - return 0; + return error; } static int index_conflicts_match(const git_vector *v, size_t idx, void *p) @@ -1341,32 +1359,36 @@ const git_index_name_entry *git_index_name_get_byindex( return git_vector_get(&index->names, n); } +static void index_name_entry_free(git_index_name_entry *ne) +{ + if (!ne) + return; + git__free(ne->ancestor); + git__free(ne->ours); + git__free(ne->theirs); + git__free(ne); +} + int git_index_name_add(git_index *index, const char *ancestor, const char *ours, const char *theirs) { git_index_name_entry *conflict_name; - assert ((ancestor && ours) || (ancestor && theirs) || (ours && theirs)); + assert((ancestor && ours) || (ancestor && theirs) || (ours && theirs)); conflict_name = git__calloc(1, sizeof(git_index_name_entry)); GITERR_CHECK_ALLOC(conflict_name); - if (ancestor) { - conflict_name->ancestor = git__strdup(ancestor); - GITERR_CHECK_ALLOC(conflict_name->ancestor); + if ((ancestor && !(conflict_name->ancestor = git__strdup(ancestor))) || + (ours && !(conflict_name->ours = git__strdup(ours))) || + (theirs && !(conflict_name->theirs = git__strdup(theirs))) || + git_vector_insert(&index->names, conflict_name) < 0) + { + index_name_entry_free(conflict_name); + return -1; } - if (ours) { - conflict_name->ours = git__strdup(ours); - GITERR_CHECK_ALLOC(conflict_name->ours); - } - - if (theirs) { - conflict_name->theirs = git__strdup(theirs); - GITERR_CHECK_ALLOC(conflict_name->theirs); - } - - return git_vector_insert(&index->names, conflict_name); + return 0; } void git_index_name_clear(git_index *index) @@ -1376,18 +1398,8 @@ void git_index_name_clear(git_index *index) assert(index); - git_vector_foreach(&index->names, i, conflict_name) { - if (conflict_name->ancestor) - git__free(conflict_name->ancestor); - - if (conflict_name->ours) - git__free(conflict_name->ours); - - if (conflict_name->theirs) - git__free(conflict_name->theirs); - - git__free(conflict_name); - } + git_vector_foreach(&index->names, i, conflict_name) + index_name_entry_free(conflict_name); git_vector_clear(&index->names); } @@ -1432,15 +1444,13 @@ int git_index_reuc_add(git_index *index, const char *path, assert(index && path); - if ((error = index_entry_reuc_init(&reuc, path, ancestor_mode, ancestor_oid, our_mode, our_oid, their_mode, their_oid)) < 0 || + if ((error = index_entry_reuc_init(&reuc, path, ancestor_mode, + ancestor_oid, our_mode, our_oid, their_mode, their_oid)) < 0 || (error = index_reuc_insert(index, reuc, 1)) < 0) - { index_entry_reuc_free(reuc); - return error; - } return error; -} +} int git_index_reuc_find(size_t *at_pos, git_index *index, const char *path) { @@ -1752,6 +1762,7 @@ static size_t read_extension(git_index *index, const char *buffer, size_t buffer static int parse_index(git_index *index, const char *buffer, size_t buffer_size) { + int error = 0; unsigned int i; struct index_header header = { 0 }; git_oid checksum_calculated, checksum_expected; @@ -1771,8 +1782,8 @@ static int parse_index(git_index *index, const char *buffer, size_t buffer_size) git_hash_buf(&checksum_calculated, buffer, buffer_size - INDEX_FOOTER_SIZE); /* Parse header */ - if (read_header(&header, buffer) < 0) - return -1; + if ((error = read_header(&header, buffer)) < 0) + return error; seek_forward(INDEX_HEADER_SIZE); @@ -1784,22 +1795,29 @@ static int parse_index(git_index *index, const char *buffer, size_t buffer_size) git_index_entry *entry; entry = git__malloc(sizeof(git_index_entry)); - GITERR_CHECK_ALLOC(entry); + if (!entry) { + error = -1; + goto done; + } entry_size = read_entry(entry, buffer, buffer_size); /* 0 bytes read means an object corruption */ - if (entry_size == 0) - return index_error_invalid("invalid entry"); + if (entry_size == 0) { + error = index_error_invalid("invalid entry"); + goto done; + } - if (git_vector_insert(&index->entries, entry) < 0) - return -1; + if ((error = git_vector_insert(&index->entries, entry)) < 0) + goto done; seek_forward(entry_size); } - if (i != header.entry_count) - return index_error_invalid("header entries changed while parsing"); + if (i != header.entry_count) { + error = index_error_invalid("header entries changed while parsing"); + goto done; + } /* There's still space for some extensions! */ while (buffer_size > INDEX_FOOTER_SIZE) { @@ -1808,20 +1826,28 @@ static int parse_index(git_index *index, const char *buffer, size_t buffer_size) extension_size = read_extension(index, buffer, buffer_size); /* see if we have read any bytes from the extension */ - if (extension_size == 0) - return index_error_invalid("extension is truncated"); + if (extension_size == 0) { + error = index_error_invalid("extension is truncated"); + goto done; + } seek_forward(extension_size); } - if (buffer_size != INDEX_FOOTER_SIZE) - return index_error_invalid("buffer size does not match index footer size"); + if (buffer_size != INDEX_FOOTER_SIZE) { + error = index_error_invalid( + "buffer size does not match index footer size"); + goto done; + } /* 160-bit SHA-1 over the content of the index file before this checksum. */ git_oid_fromraw(&checksum_expected, (const unsigned char *)buffer); - if (git_oid__cmp(&checksum_calculated, &checksum_expected) != 0) - return index_error_invalid("calculated checksum does not match expected"); + if (git_oid__cmp(&checksum_calculated, &checksum_expected) != 0) { + error = index_error_invalid( + "calculated checksum does not match expected"); + goto done; + } #undef seek_forward @@ -1831,7 +1857,8 @@ static int parse_index(git_index *index, const char *buffer, size_t buffer_size) git_vector_set_sorted(&index->entries, !index->ignore_case); git_vector_sort(&index->entries); - return 0; +done: + return error; } static bool is_index_extended(git_index *index) @@ -1916,19 +1943,20 @@ static int write_entries(git_index *index, git_filebuf *file) { int error = 0; size_t i; - git_vector case_sorted; + git_vector case_sorted, *entries; git_index_entry *entry; - git_vector *out = &index->entries; /* If index->entries is sorted case-insensitively, then we need * to re-sort it case-sensitively before writing */ if (index->ignore_case) { git_vector_dup(&case_sorted, &index->entries, index_cmp); git_vector_sort(&case_sorted); - out = &case_sorted; + entries = &case_sorted; + } else { + entries = &index->entries; } - git_vector_foreach(out, i, entry) + git_vector_foreach(entries, i, entry) if ((error = write_disk_entry(file, entry)) < 0) break; @@ -2179,8 +2207,11 @@ int git_index_read_tree(git_index *index, const git_tree *tree) if (!error) { git_vector_sort(&entries); - git_index_clear(index); - git_vector_swap(&entries, &index->entries); + + if ((error = git_index_clear(index)) < 0) + /* well, this isn't good */; + else + git_vector_swap(&entries, &index->entries); } git_vector_free(&entries); @@ -2272,17 +2303,14 @@ int git_index_add_all( break; /* make the new entry to insert */ - if ((entry = index_entry_dup(wd)) == NULL) { - error = -1; + if ((error = index_entry_dup(&entry, wd)) < 0) break; - } + entry->id = blobid; /* add working directory item to index */ - if ((error = index_insert(index, entry, 1)) < 0) { - index_entry_free(entry); + if ((error = index_insert(index, &entry, 1)) < 0) break; - } git_tree_cache_invalidate_path(index->tree, wd->path); diff --git a/src/index.h b/src/index.h index 17f04f0ad..cabdbca30 100644 --- a/src/index.h +++ b/src/index.h @@ -50,11 +50,13 @@ struct git_index_conflict_iterator { extern void git_index_entry__init_from_stat( git_index_entry *entry, struct stat *st, bool trust_mode); -extern size_t git_index__prefix_position(git_index *index, const char *path); - extern int git_index_entry__cmp(const void *a, const void *b); extern int git_index_entry__cmp_icase(const void *a, const void *b); +/* Search index for `path`, returning GIT_ENOTFOUND if it does not exist. + * `at_pos` is set to the position where it is or would be inserted. + * Pass `path_len` as strlen of path or 0 to call strlen internally. + */ extern int git_index__find( size_t *at_pos, git_index *index, const char *path, size_t path_len, int stage); diff --git a/src/iterator.c b/src/iterator.c index a7a44914c..45655254c 100644 --- a/src/iterator.c +++ b/src/iterator.c @@ -815,8 +815,10 @@ static int index_iterator__reset( if (iterator__reset_range(self, start, end) < 0) return -1; - ii->current = ii->base.start ? - git_index__prefix_position(ii->index, ii->base.start) : 0; + ii->current = 0; + + if (ii->base.start) + git_index__find(&ii->current, ii->index, ii->base.start, 0, 0); if ((ie = index_iterator__skip_conflicts(ii)) == NULL) return 0; From 27e54bcf82fe27bdfadbaa9c5383ee8a948ea33c Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Fri, 7 Feb 2014 14:17:19 -0800 Subject: [PATCH 03/20] Add public diff print helpers The usefulness of these helpers came up for me while debugging some of the iterator changes that I was making, so since they have also been requested (albeit indirectly) I thought I'd include them. --- include/git2/sys/diff.h | 63 +++++++++++++++++++++++++++++++++++++++ src/diff_print.c | 28 +++++++++++++++-- tests/diff/diff_helpers.c | 29 +++++------------- 3 files changed, 96 insertions(+), 24 deletions(-) create mode 100644 include/git2/sys/diff.h diff --git a/include/git2/sys/diff.h b/include/git2/sys/diff.h new file mode 100644 index 000000000..bc6cdf393 --- /dev/null +++ b/include/git2/sys/diff.h @@ -0,0 +1,63 @@ +/* + * Copyright (C) the libgit2 contributors. All rights reserved. + * + * This file is part of libgit2, distributed under the GNU GPL v2 with + * a Linking Exception. For full terms see the included COPYING file. + */ +#ifndef INCLUDE_sys_git_diff_h__ +#define INCLUDE_sys_git_diff_h__ + +#include "git2/common.h" +#include "git2/types.h" +#include "git2/oid.h" + +/** + * @file git2/sys/diff.h + * @brief Low-level Git diff utilities + * @ingroup Git + * @{ + */ +GIT_BEGIN_DECL + +/** + * Diff print callback that writes to a git_buf. + * + * This function is provided not for you to call it directly, but instead + * so you can use it as a function pointer to the `git_diff_print` or + * `git_patch_print` APIs. When using those APIs, you specify a callback + * to actually handle the diff and/or patch data. + * + * Use this callback to easily write that data to a `git_buf` buffer. You + * must pass a `git_buf *` value as the payload to the `git_diff_print` + * and/or `git_patch_print` function. The data will be appended to the + * buffer (after any existing content). + */ +GIT_EXTERN(int) git_diff_print_callback__to_buf( + const git_diff_delta *delta, + const git_diff_hunk *hunk, + const git_diff_line *line, + void *payload); /*< payload must be a `git_buf *` */ + +/** + * Diff print callback that writes to stdio FILE handle. + * + * This function is provided not for you to call it directly, but instead + * so you can use it as a function pointer to the `git_diff_print` or + * `git_patch_print` APIs. When using those APIs, you specify a callback + * to actually handle the diff and/or patch data. + * + * Use this callback to easily write that data to a stdio FILE handle. You + * must pass a `FILE *` value (such as `stdout` or `stderr` or the return + * value from `fopen()`) as the payload to the `git_diff_print` + * and/or `git_patch_print` function. If you pass NULL, this will write + * data to `stdout`. + */ +GIT_EXTERN(int) git_diff_print_callback__to_file_handle( + const git_diff_delta *delta, + const git_diff_hunk *hunk, + const git_diff_line *line, + void *payload); /*< payload must be a `FILE *` */ + +/** @} */ +GIT_END_DECL +#endif diff --git a/src/diff_print.c b/src/diff_print.c index 1a09bed54..a7f7b6fe8 100644 --- a/src/diff_print.c +++ b/src/diff_print.c @@ -8,6 +8,7 @@ #include "diff.h" #include "diff_patch.h" #include "fileops.h" +#include "git2/sys/diff.h" typedef struct { git_diff *diff; @@ -435,7 +436,7 @@ int git_patch_print( return error; } -static int diff_print_to_buffer_cb( +int git_diff_print_callback__to_buf( const git_diff_delta *delta, const git_diff_hunk *hunk, const git_diff_line *line, @@ -444,6 +445,11 @@ static int diff_print_to_buffer_cb( git_buf *output = payload; GIT_UNUSED(delta); GIT_UNUSED(hunk); + if (!output) { + giterr_set(GITERR_INVALID, "Buffer pointer must be provided"); + return -1; + } + if (line->origin == GIT_DIFF_LINE_ADDITION || line->origin == GIT_DIFF_LINE_DELETION || line->origin == GIT_DIFF_LINE_CONTEXT) @@ -452,10 +458,28 @@ static int diff_print_to_buffer_cb( return git_buf_put(output, line->content, line->content_len); } +int git_diff_print_callback__to_file_handle( + const git_diff_delta *delta, + const git_diff_hunk *hunk, + const git_diff_line *line, + void *payload) +{ + FILE *fp = payload ? payload : stdout; + + GIT_UNUSED(delta); GIT_UNUSED(hunk); + + if (line->origin == GIT_DIFF_LINE_CONTEXT || + line->origin == GIT_DIFF_LINE_ADDITION || + line->origin == GIT_DIFF_LINE_DELETION) + fputc(line->origin, fp); + fwrite(line->content, 1, line->content_len, fp); + return 0; +} + /* print a git_patch to a git_buf */ int git_patch_to_buf( git_buf *out, git_patch *patch) { - return git_patch_print(patch, diff_print_to_buffer_cb, out); + return git_patch_print(patch, git_diff_print_callback__to_buf, out); } diff --git a/tests/diff/diff_helpers.c b/tests/diff/diff_helpers.c index 33bb561f6..279cb20c5 100644 --- a/tests/diff/diff_helpers.c +++ b/tests/diff/diff_helpers.c @@ -1,5 +1,6 @@ #include "clar_libgit2.h" #include "diff_helpers.h" +#include "git2/sys/diff.h" git_tree *resolve_commit_oid_to_tree( git_repository *repo, @@ -215,32 +216,16 @@ abort: return GIT_EUSER; } -static int diff_print_cb( - const git_diff_delta *delta, - const git_diff_hunk *hunk, - const git_diff_line *line, - void *payload) -{ - FILE *fp = payload; - - GIT_UNUSED(delta); GIT_UNUSED(hunk); - - if (line->origin == GIT_DIFF_LINE_CONTEXT || - line->origin == GIT_DIFF_LINE_ADDITION || - line->origin == GIT_DIFF_LINE_DELETION) - fputc(line->origin, fp); - fwrite(line->content, 1, line->content_len, fp); - return 0; -} - void diff_print(FILE *fp, git_diff *diff) { - cl_git_pass(git_diff_print( - diff, GIT_DIFF_FORMAT_PATCH, diff_print_cb, fp ? fp : stderr)); + cl_git_pass( + git_diff_print(diff, GIT_DIFF_FORMAT_PATCH, + git_diff_print_callback__to_file_handle, fp ? fp : stderr)); } void diff_print_raw(FILE *fp, git_diff *diff) { - cl_git_pass(git_diff_print( - diff, GIT_DIFF_FORMAT_RAW, diff_print_cb, fp ? fp : stderr)); + cl_git_pass( + git_diff_print(diff, GIT_DIFF_FORMAT_RAW, + git_diff_print_callback__to_file_handle, fp ? fp : stderr)); } From 54edbb9871ad543baefc62310512719fc70539be Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Fri, 7 Feb 2014 16:48:27 -0800 Subject: [PATCH 04/20] Add index snapshot and use it for iterator --- src/index.c | 42 +++++++++++++++++++++++++++++++++--------- src/index.h | 12 ++++++++++++ src/iterator.c | 35 ++++++++++++++++++++++------------- 3 files changed, 67 insertions(+), 22 deletions(-) diff --git a/src/index.c b/src/index.c index 8bc5cb13c..58b359bfe 100644 --- a/src/index.c +++ b/src/index.c @@ -1056,21 +1056,26 @@ int git_index_remove_directory(git_index *index, const char *dir, int stage) return error; } -int git_index__find( - size_t *out, git_index *index, const char *path, size_t path_len, int stage) +int git_index__find_in_entries( + size_t *out, git_vector *entries, git_vector_cmp entry_cmp, + const char *path, size_t path_len, int stage) { struct entry_srch_key srch_key; - - assert(path); - - git_vector_sort(&index->entries); - srch_key.path = path; srch_key.path_len = !path_len ? strlen(path) : path_len; srch_key.stage = stage; + return git_vector_bsearch2(out, entries, entry_cmp, &srch_key); +} - return git_vector_bsearch2( - out, &index->entries, index->entries_search, &srch_key); +int git_index__find( + size_t *out, git_index *index, const char *path, size_t path_len, int stage) +{ + assert(index && path); + + git_vector_sort(&index->entries); + + return git_index__find_in_entries( + out, &index->entries, index->entries_search, path, path_len, stage); } int git_index_find(size_t *at_pos, git_index *index, const char *path) @@ -2442,3 +2447,22 @@ int git_index_update_all( return error; } + +int git_index__snapshot(git_vector *entries, git_index *index) +{ + int error; + + GIT_REFCOUNT_INC(index); + git_vector_sort(&index->entries); + error = git_vector_dup(entries, &index->entries, index->entries._cmp); + + if (error < 0) + git_index_free(index); + + return error; +} + +void git_index__release_snapshot(git_index *index) +{ + git_index_free(index); +} diff --git a/src/index.h b/src/index.h index cabdbca30..2b6f5c98b 100644 --- a/src/index.h +++ b/src/index.h @@ -71,4 +71,16 @@ GIT_INLINE(const git_futils_filestamp *) git_index__filestamp(git_index *index) extern int git_index__changed_relative_to(git_index *index, const git_futils_filestamp *fs); +/* Copy the current entries vector *and* increment the index refcount. + * Call `git_index__release_snapshot` when done. + */ +extern int git_index__snapshot(git_vector *entries, git_index *index); +extern void git_index__release_snapshot(git_index *index); + +/* Allow searching in a snapshot; entries must already be sorted! */ +extern int git_index__find_in_entries( + size_t *at_pos, + git_vector *entries, git_vector_cmp entry_cmp, + const char *path, size_t path_len, int stage); + #endif diff --git a/src/iterator.c b/src/iterator.c index 45655254c..a84f4d3db 100644 --- a/src/iterator.c +++ b/src/iterator.c @@ -644,6 +644,8 @@ typedef struct { git_iterator base; git_iterator_callbacks cb; git_index *index; + git_vector entries; + git_vector_cmp entry_srch; size_t current; /* when not in autoexpand mode, use these to represent "tree" state */ git_buf partial; @@ -654,10 +656,10 @@ typedef struct { static const git_index_entry *index_iterator__index_entry(index_iterator *ii) { - const git_index_entry *ie = git_index_get_byindex(ii->index, ii->current); + const git_index_entry *ie = git_vector_get(&ii->entries, ii->current); if (ie != NULL && iterator__past_end(ii, ie->path)) { - ii->current = git_index_entrycount(ii->index); + ii->current = git_vector_length(&ii->entries); ie = NULL; } @@ -726,7 +728,7 @@ static int index_iterator__current( const git_index_entry **entry, git_iterator *self) { index_iterator *ii = (index_iterator *)self; - const git_index_entry *ie = git_index_get_byindex(ii->index, ii->current); + const git_index_entry *ie = git_vector_get(&ii->entries, ii->current); if (ie != NULL && index_iterator__at_tree(ii)) { ii->tree_entry.path = ii->partial.ptr; @@ -744,14 +746,14 @@ static int index_iterator__current( static int index_iterator__at_end(git_iterator *self) { index_iterator *ii = (index_iterator *)self; - return (ii->current >= git_index_entrycount(ii->index)); + return (ii->current >= git_vector_length(&ii->entries)); } static int index_iterator__advance( const git_index_entry **entry, git_iterator *self) { index_iterator *ii = (index_iterator *)self; - size_t entrycount = git_index_entrycount(ii->index); + size_t entrycount = git_vector_length(&ii->entries); const git_index_entry *ie; if (!iterator__has_been_accessed(ii)) @@ -766,7 +768,7 @@ static int index_iterator__advance( while (ii->current < entrycount) { ii->current++; - if (!(ie = git_index_get_byindex(ii->index, ii->current)) || + if (!(ie = git_vector_get(&ii->entries, ii->current)) || ii->base.prefixcomp(ie->path, ii->partial.ptr) != 0) break; } @@ -789,7 +791,7 @@ static int index_iterator__advance_into( const git_index_entry **entry, git_iterator *self) { index_iterator *ii = (index_iterator *)self; - const git_index_entry *ie = git_index_get_byindex(ii->index, ii->current); + const git_index_entry *ie = git_vector_get(&ii->entries, ii->current); if (ie != NULL && index_iterator__at_tree(ii)) { if (ii->restore_terminator) @@ -818,7 +820,8 @@ static int index_iterator__reset( ii->current = 0; if (ii->base.start) - git_index__find(&ii->current, ii->index, ii->base.start, 0, 0); + git_index__find_in_entries( + &ii->current, &ii->entries, ii->entry_srch, ii->base.start, 0, 0); if ((ie = index_iterator__skip_conflicts(ii)) == NULL) return 0; @@ -843,9 +846,9 @@ static int index_iterator__reset( static void index_iterator__free(git_iterator *self) { index_iterator *ii = (index_iterator *)self; - git_index_free(ii->index); + git_index__release_snapshot(ii->index); ii->index = NULL; - + git_vector_free(&ii->entries); git_buf_free(&ii->partial); } @@ -856,9 +859,17 @@ int git_iterator_for_index( const char *start, const char *end) { + int error = 0; index_iterator *ii = git__calloc(1, sizeof(index_iterator)); GITERR_CHECK_ALLOC(ii); + if ((error = git_index__snapshot(&ii->entries, index)) < 0) { + git__free(ii); + return error; + } + ii->index = index; + ii->entry_srch = index->entries_search; + ITERATOR_BASE_INIT(ii, index, INDEX, git_index_owner(index)); if (index->ignore_case) { @@ -866,8 +877,7 @@ int git_iterator_for_index( ii->base.prefixcomp = git__prefixcmp_icase; } - ii->index = index; - GIT_REFCOUNT_INC(index); + /* TODO: resort entries to match desired ignore_case behavior */ git_buf_init(&ii->partial, 0); ii->tree_entry.mode = GIT_FILEMODE_TREE; @@ -875,7 +885,6 @@ int git_iterator_for_index( index_iterator__reset((git_iterator *)ii, NULL, NULL); *iter = (git_iterator *)ii; - return 0; } From dac160489bbf8de90d2f1ae152df68ded2603598 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Sat, 8 Feb 2014 16:42:26 -0800 Subject: [PATCH 05/20] Add mutex around index entries changes This surrounds any function that mutates the entries vector with a mutex so it can be safely snapshotted. --- src/index.c | 225 ++++++++++++++++++++++++++++++++++++++-------------- src/index.h | 7 +- 2 files changed, 169 insertions(+), 63 deletions(-) diff --git a/src/index.c b/src/index.c index 58b359bfe..f28cbef27 100644 --- a/src/index.c +++ b/src/index.c @@ -325,6 +325,28 @@ static unsigned int index_merge_mode( return git_index__create_mode(mode); } +static int index_sort_if_needed(git_index *index, bool need_lock) +{ + /* not truly threadsafe because between when this checks and/or + * sorts the array another thread could come in and unsort it + */ + + if (git_vector_is_sorted(&index->entries)) + return 0; + + if (need_lock && git_mutex_lock(&index->lock) < 0) { + giterr_set(GITERR_OS, "Unable to lock index"); + return -1; + } + + git_vector_sort(&index->entries); + + if (need_lock) + git_mutex_unlock(&index->lock); + + return 0; +} + void git_index__set_ignore_case(git_index *index, bool ignore_case) { index->ignore_case = ignore_case; @@ -334,7 +356,7 @@ void git_index__set_ignore_case(git_index *index, bool ignore_case) index->entries_search_path = ignore_case ? index_isrch_path : index_srch_path; git_vector_set_cmp(&index->entries, ignore_case ? index_icmp : index_cmp); - git_vector_sort(&index->entries); + index_sort_if_needed(index, true); index->reuc_search = ignore_case ? reuc_isrch : reuc_srch; @@ -352,6 +374,12 @@ int git_index_open(git_index **index_out, const char *index_path) index = git__calloc(1, sizeof(git_index)); GITERR_CHECK_ALLOC(index); + if (git_mutex_init(&index->lock)) { + giterr_set(GITERR_OS, "Failed to initialize lock"); + git__free(index); + return -1; + } + if (index_path != NULL) { index->index_file_path = git__strdup(index_path); if (!index->index_file_path) @@ -364,7 +392,8 @@ int git_index_open(git_index **index_out, const char *index_path) if (git_vector_init(&index->entries, 32, index_cmp) < 0 || git_vector_init(&index->names, 32, conflict_name_cmp) < 0 || - git_vector_init(&index->reuc, 32, reuc_cmp) < 0) + git_vector_init(&index->reuc, 32, reuc_cmp) < 0 || + git_vector_init(&index->deleted, 2, index_cmp) < 0) goto fail; index->entries_cmp_path = index_cmp_path; @@ -392,12 +421,19 @@ int git_index_new(git_index **out) static void index_free(git_index *index) { + /* index iterators increment the refcount of the index, so if we + * get here then there should be no outstanding iterators. + */ + assert(!git_atomic_get(&index->readers)); + git_index_clear(index); git_vector_free(&index->entries); git_vector_free(&index->names); git_vector_free(&index->reuc); + git_vector_free(&index->deleted); git__free(index->index_file_path); + git_mutex_free(&index->lock); git__memzero(index, sizeof(*index)); git__free(index); @@ -411,30 +447,74 @@ void git_index_free(git_index *index) GIT_REFCOUNT_DEC(index, index_free); } -static void index_entries_free(git_vector *entries) +/* call with locked index */ +static void index_free_deleted(git_index *index) { size_t i; - for (i = 0; i < entries->length; ++i) - index_entry_free(git__swap(entries->contents[i], NULL)); + if (git_atomic_get(&index->readers) > 0) + return; - git_vector_clear(entries); + for (i = 0; i < index->deleted.length; ++i) + index_entry_free(git__swap(index->deleted.contents[i], NULL)); + + git_vector_clear(&index->deleted); +} + +static int index_remove_entry(git_index *index, size_t pos, bool need_lock) +{ + int error = 0; + git_index_entry *entry = git_vector_get(&index->entries, pos); + + if (entry != NULL) + git_tree_cache_invalidate_path(index->tree, entry->path); + + if (need_lock && git_mutex_lock(&index->lock) < 0) { + giterr_set(GITERR_OS, "Unable to lock index"); + return -1; + } + + error = git_vector_remove(&index->entries, pos); + + if (!error) { + if (!git_atomic_get(&index->readers)) + index_entry_free(entry); + else + error = git_vector_insert(&index->deleted, entry); + } + + if (need_lock) + git_mutex_unlock(&index->lock); + + return error; } int git_index_clear(git_index *index) { + int error = 0; + assert(index); git_tree_cache_free(index->tree); index->tree = NULL; - index_entries_free(&index->entries); + if (git_mutex_lock(&index->lock) < 0) { + giterr_set(GITERR_OS, "Failed to lock index"); + return -1; + } + + while (!error && index->entries.length > 0) + error = index_remove_entry(index, index->entries.length - 1, false); + index_free_deleted(index); + + git_mutex_unlock(&index->lock); + git_index_reuc_clear(index); git_index_name_clear(index); git_futils_filestamp_set(&index->stamp, NULL); - return 0; + return error; } static int create_index_error(int error, const char *msg) @@ -545,7 +625,8 @@ int git_index_write(git_index *index) return create_index_error(-1, "Failed to read index: The index is in-memory only"); - git_vector_sort(&index->entries); + if (index_sort_if_needed(index, true) < 0) + return -1; git_vector_sort(&index->reuc); if ((error = git_filebuf_open( @@ -593,7 +674,8 @@ int git_index_write_tree(git_oid *oid, git_index *index) return git_tree__write_index(oid, index, repo); } -int git_index_write_tree_to(git_oid *oid, git_index *index, git_repository *repo) +int git_index_write_tree_to( + git_oid *oid, git_index *index, git_repository *repo) { assert(oid && index && repo); return git_tree__write_index(oid, index, repo); @@ -609,7 +691,8 @@ const git_index_entry *git_index_get_byindex( git_index *index, size_t n) { assert(index); - git_vector_sort(&index->entries); + if (index_sort_if_needed(index, true) < 0) + return NULL; return git_vector_get(&index->entries, n); } @@ -695,22 +778,6 @@ static int index_entry_init( return 0; } -static int index_remove_entry(git_index *index, size_t pos) -{ - int error = 0; - git_index_entry *entry = git_vector_get(&index->entries, pos); - - if (entry != NULL) - git_tree_cache_invalidate_path(index->tree, entry->path); - - error = git_vector_remove(&index->entries, pos); - - if (!error) - index_entry_free(entry); - - return error; -} - static int index_entry_reuc_init(git_index_reuc_entry **reuc_out, const char *path, int ancestor_mode, const git_oid *ancestor_oid, @@ -793,7 +860,7 @@ static int has_file_name(git_index *index, if (!ok_to_replace) break; - if (index_remove_entry(index, --pos) < 0) + if (index_remove_entry(index, --pos, true) < 0) break; } return retval; @@ -827,7 +894,7 @@ static int has_dir_name(git_index *index, if (!ok_to_replace) break; - if (index_remove_entry(index, position) < 0) + if (index_remove_entry(index, position, true) < 0) break; continue; } @@ -924,7 +991,12 @@ static int index_insert( /* if replacing is not requested or no existing entry exists, just * insert entry at the end; the index is no longer sorted */ - error = git_vector_insert(&index->entries, entry); + if ((error = git_mutex_lock(&index->lock)) < 0) { + giterr_set(GITERR_OS, "Unable to acquire index lock"); + } else { + error = git_vector_insert(&index->entries, entry); + git_mutex_unlock(&index->lock); + } done: if (error < 0) { @@ -1020,7 +1092,7 @@ int git_index_remove(git_index *index, const char *path, int stage) return GIT_ENOTFOUND; } - return index_remove_entry(index, position); + return index_remove_entry(index, position, true); } int git_index_remove_directory(git_index *index, const char *dir, int stage) @@ -1045,7 +1117,7 @@ int git_index_remove_directory(git_index *index, const char *dir, int stage) continue; } - if ((error = index_remove_entry(index, pos)) < 0) + if ((error = index_remove_entry(index, pos, true)) < 0) break; /* removed entry at 'pos' so we don't need to increment it */ @@ -1072,7 +1144,8 @@ int git_index__find( { assert(index && path); - git_vector_sort(&index->entries); + if (index_sort_if_needed(index, true) < 0) + return -1; return git_index__find_in_entries( out, &index->entries, index->entries_search, path, path_len, stage); @@ -1244,7 +1317,7 @@ int git_index_conflict_remove(git_index *index, const char *path) continue; } - if ((error = index_remove_entry(index, pos)) < 0) + if ((error = index_remove_entry(index, pos, true)) < 0) break; } @@ -1256,12 +1329,9 @@ static int index_conflicts_match(const git_vector *v, size_t idx, void *p) git_index *index = p; git_index_entry *entry = git_vector_get(v, idx); - GIT_UNUSED(index); - - if (GIT_IDXENTRY_STAGE(entry) > 0) { - index_entry_free(entry); + if (GIT_IDXENTRY_STAGE(entry) > 0 && + !index_remove_entry(index, idx, false)) return 1; - } return 0; } @@ -1269,7 +1339,12 @@ static int index_conflicts_match(const git_vector *v, size_t idx, void *p) void git_index_conflict_cleanup(git_index *index) { assert(index); + + if (git_mutex_lock(&index->lock) < 0) + return; git_vector_remove_matching(&index->entries, index_conflicts_match, index); + index_free_deleted(index); + git_mutex_unlock(&index->lock); } int git_index_has_conflicts(const git_index *index) @@ -1792,6 +1867,11 @@ static int parse_index(git_index *index, const char *buffer, size_t buffer_size) seek_forward(INDEX_HEADER_SIZE); + if (git_mutex_lock(&index->lock) < 0) { + giterr_set(GITERR_OS, "Unable to acquire index lock"); + return -1; + } + git_vector_clear(&index->entries); /* Parse all the entries */ @@ -1860,9 +1940,10 @@ static int parse_index(git_index *index, const char *buffer, size_t buffer_size) * in-memory index is supposed to be case-insensitive */ git_vector_set_sorted(&index->entries, !index->ignore_case); - git_vector_sort(&index->entries); + error = index_sort_if_needed(index, false); done: + git_mutex_unlock(&index->lock); return error; } @@ -1951,6 +2032,11 @@ static int write_entries(git_index *index, git_filebuf *file) git_vector case_sorted, *entries; git_index_entry *entry; + if (git_mutex_lock(&index->lock) < 0) { + giterr_set(GITERR_OS, "Failed to lock index"); + return -1; + } + /* If index->entries is sorted case-insensitively, then we need * to re-sort it case-sensitively before writing */ if (index->ignore_case) { @@ -1965,6 +2051,8 @@ static int write_entries(git_index *index, git_filebuf *file) if ((error = write_disk_entry(file, entry)) < 0) break; + git_mutex_unlock(&index->lock); + if (index->ignore_case) git_vector_free(&case_sorted); @@ -2136,7 +2224,7 @@ int git_index_entry_stage(const git_index_entry *entry) typedef struct read_tree_data { git_vector *old_entries; git_vector *new_entries; - git_vector_cmp entries_search; + git_vector_cmp entry_cmp; } read_tree_data; static int read_tree_cb( @@ -2145,6 +2233,7 @@ static int read_tree_cb( read_tree_data *data = payload; git_index_entry *entry = NULL, *old_entry; git_buf path = GIT_BUF_INIT; + size_t pos; if (git_tree_entry__is_tree(tentry)) return 0; @@ -2159,23 +2248,15 @@ static int read_tree_cb( entry->id = tentry->oid; /* look for corresponding old entry and copy data to new entry */ - if (data->old_entries) { - size_t pos; - struct entry_srch_key skey; - - skey.path = path.ptr; - skey.path_len = strlen(path.ptr); - skey.stage = 0; - - if (!git_vector_bsearch2( - &pos, data->old_entries, data->entries_search, &skey) && - (old_entry = git_vector_get(data->old_entries, pos)) != NULL && - entry->mode == old_entry->mode && - git_oid_equal(&entry->id, &old_entry->id)) - { - memcpy(entry, old_entry, sizeof(*entry)); - entry->flags_extended = 0; - } + if (data->old_entries != NULL && + !git_index__find_in_entries( + &pos, data->old_entries, data->entry_cmp, path.ptr, 0, 0) && + (old_entry = git_vector_get(data->old_entries, pos)) != NULL && + entry->mode == old_entry->mode && + git_oid_equal(&entry->id, &old_entry->id)) + { + memcpy(entry, old_entry, sizeof(*entry)); + entry->flags_extended = 0; } if (path.size < GIT_IDXENTRY_NAMEMASK) @@ -2204,9 +2285,10 @@ int git_index_read_tree(git_index *index, const git_tree *tree) data.old_entries = &index->entries; data.new_entries = &entries; - data.entries_search = index->entries_search; + data.entry_cmp = index->entries_search; - git_vector_sort(&index->entries); + if (index_sort_if_needed(index, true) < 0) + return -1; error = git_tree_walk(tree, GIT_TREEWALK_POST, read_tree_cb, &data); @@ -2215,8 +2297,13 @@ int git_index_read_tree(git_index *index, const git_tree *tree) if ((error = git_index_clear(index)) < 0) /* well, this isn't good */; - else + else if (git_mutex_lock(&index->lock) < 0) { + giterr_set(GITERR_OS, "Unable to acquire index lock"); + error = -1; + } else { git_vector_swap(&entries, &index->entries); + git_mutex_unlock(&index->lock); + } } git_vector_free(&entries); @@ -2453,9 +2540,19 @@ int git_index__snapshot(git_vector *entries, git_index *index) int error; GIT_REFCOUNT_INC(index); + + if (git_mutex_lock(&index->lock) < 0) { + giterr_set(GITERR_OS, "Failed to lock index"); + return -1; + } + + git_atomic_inc(&index->readers); git_vector_sort(&index->entries); + error = git_vector_dup(entries, &index->entries, index->entries._cmp); + git_mutex_unlock(&index->lock); + if (error < 0) git_index_free(index); @@ -2464,5 +2561,11 @@ int git_index__snapshot(git_vector *entries, git_index *index) void git_index__release_snapshot(git_index *index) { + git_atomic_dec(&index->readers); git_index_free(index); + + if (!git_mutex_lock(&index->lock)) { + index_free_deleted(index); /* try to free pending deleted items */ + git_mutex_unlock(&index->lock); + } } diff --git a/src/index.h b/src/index.h index 2b6f5c98b..c2a932218 100644 --- a/src/index.h +++ b/src/index.h @@ -21,12 +21,15 @@ struct git_index { git_refcount rc; char *index_file_path; - git_futils_filestamp stamp; + git_vector entries; - unsigned int on_disk:1; + git_mutex lock; /* lock held while entries is being changed */ + git_vector deleted; /* deleted entries if readers > 0 */ + git_atomic readers; /* number of active iterators */ + unsigned int on_disk:1; unsigned int ignore_case:1; unsigned int distrust_filemode:1; unsigned int no_symlinks:1; From 3b4c401a38ce912d5be8c9bf4ab1c4912a4f08bd Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Mon, 10 Feb 2014 13:20:08 -0800 Subject: [PATCH 06/20] Decouple index iterator sort from index This makes the index iterator honor the GIT_ITERATOR_IGNORE_CASE and GIT_ITERATOR_DONT_IGNORE_CASE flags without modifying the index data itself. To take advantage of this, I had to export a number of the internal index entry comparison functions. I also wrote some new tests to exercise the capability. --- src/diff.c | 79 +++++++++++++++++++++---------------------- src/index.c | 73 +++++++++++++++------------------------ src/index.h | 11 ++++-- src/iterator.c | 14 +++++--- src/merge.c | 2 +- src/util.c | 8 +++-- src/util.h | 1 + src/vector.c | 3 +- tests/diff/iterator.c | 58 +++++++++++++++++++++++++++---- 9 files changed, 142 insertions(+), 107 deletions(-) diff --git a/src/diff.c b/src/diff.c index cb05a5faf..0d1aed4ad 100644 --- a/src/diff.c +++ b/src/diff.c @@ -318,6 +318,31 @@ static const char *diff_mnemonic_prefix( return pfx; } +static void diff_set_ignore_case(git_diff *diff, bool ignore_case) +{ + if (!ignore_case) { + diff->opts.flags &= ~GIT_DIFF_IGNORE_CASE; + + diff->strcomp = git__strcmp; + diff->strncomp = git__strncmp; + diff->pfxcomp = git__prefixcmp; + diff->entrycomp = git_index_entry_cmp; + + git_vector_set_cmp(&diff->deltas, git_diff_delta__cmp); + } else { + diff->opts.flags |= GIT_DIFF_IGNORE_CASE; + + diff->strcomp = git__strcasecmp; + diff->strncomp = git__strncasecmp; + diff->pfxcomp = git__prefixcmp_icase; + diff->entrycomp = git_index_entry_icmp; + + git_vector_set_cmp(&diff->deltas, git_diff_delta__casecmp); + } + + git_vector_sort(&diff->deltas); +} + static git_diff *diff_list_alloc( git_repository *repo, git_iterator *old_iter, @@ -344,24 +369,10 @@ static git_diff *diff_list_alloc( /* Use case-insensitive compare if either iterator has * the ignore_case bit set */ - if (!git_iterator_ignore_case(old_iter) && - !git_iterator_ignore_case(new_iter)) { - diff->opts.flags &= ~GIT_DIFF_IGNORE_CASE; - - diff->strcomp = git__strcmp; - diff->strncomp = git__strncmp; - diff->pfxcomp = git__prefixcmp; - diff->entrycomp = git_index_entry__cmp; - } else { - diff->opts.flags |= GIT_DIFF_IGNORE_CASE; - - diff->strcomp = git__strcasecmp; - diff->strncomp = git__strncasecmp; - diff->pfxcomp = git__prefixcmp_icase; - diff->entrycomp = git_index_entry__cmp_icase; - - git_vector_set_cmp(&diff->deltas, git_diff_delta__casecmp); - } + diff_set_ignore_case( + diff, + git_iterator_ignore_case(old_iter) || + git_iterator_ignore_case(new_iter)); return diff; } @@ -1183,39 +1194,25 @@ int git_diff_tree_to_index( const git_diff_options *opts) { int error = 0; - bool reset_index_ignore_case = false; + bool index_ignore_case = false; assert(diff && repo); if (!index && (error = diff_load_index(&index, repo)) < 0) return error; - if (index->ignore_case) { - git_index__set_ignore_case(index, false); - reset_index_ignore_case = true; - } + index_ignore_case = index->ignore_case; DIFF_FROM_ITERATORS( - git_iterator_for_tree(&a, old_tree, 0, pfx, pfx), - git_iterator_for_index(&b, index, 0, pfx, pfx) + git_iterator_for_tree( + &a, old_tree, GIT_ITERATOR_DONT_IGNORE_CASE, pfx, pfx), + git_iterator_for_index( + &b, index, GIT_ITERATOR_DONT_IGNORE_CASE, pfx, pfx) ); - if (reset_index_ignore_case) { - git_index__set_ignore_case(index, true); - - if (!error) { - git_diff *d = *diff; - - d->opts.flags |= GIT_DIFF_IGNORE_CASE; - d->strcomp = git__strcasecmp; - d->strncomp = git__strncasecmp; - d->pfxcomp = git__prefixcmp_icase; - d->entrycomp = git_index_entry__cmp_icase; - - git_vector_set_cmp(&d->deltas, git_diff_delta__casecmp); - git_vector_sort(&d->deltas); - } - } + /* if index is in case-insensitive order, re-sort deltas to match */ + if (!error && index_ignore_case) + diff_set_ignore_case(*diff, true); return error; } diff --git a/src/index.c b/src/index.c index f28cbef27..d733e4e48 100644 --- a/src/index.c +++ b/src/index.c @@ -106,7 +106,7 @@ static int write_index(git_index *index, git_filebuf *file); static void index_entry_free(git_index_entry *entry); static void index_entry_reuc_free(git_index_reuc_entry *reuc); -static int index_srch(const void *key, const void *array_member) +int git_index_entry_srch(const void *key, const void *array_member) { const struct entry_srch_key *srch_key = key; const git_index_entry *entry = array_member; @@ -131,7 +131,7 @@ static int index_srch(const void *key, const void *array_member) return 0; } -static int index_isrch(const void *key, const void *array_member) +int git_index_entry_isrch(const void *key, const void *array_member) { const struct entry_srch_key *srch_key = key; const git_index_entry *entry = array_member; @@ -157,31 +157,21 @@ static int index_isrch(const void *key, const void *array_member) return 0; } -static int index_cmp_path(const void *a, const void *b) -{ - return strcmp((const char *)a, (const char *)b); -} - -static int index_icmp_path(const void *a, const void *b) -{ - return strcasecmp((const char *)a, (const char *)b); -} - -static int index_srch_path(const void *path, const void *array_member) +static int index_entry_srch_path(const void *path, const void *array_member) { const git_index_entry *entry = array_member; return strcmp((const char *)path, entry->path); } -static int index_isrch_path(const void *path, const void *array_member) +static int index_entry_isrch_path(const void *path, const void *array_member) { const git_index_entry *entry = array_member; return strcasecmp((const char *)path, entry->path); } -static int index_cmp(const void *a, const void *b) +int git_index_entry_cmp(const void *a, const void *b) { int diff; const git_index_entry *entry_a = a; @@ -195,7 +185,7 @@ static int index_cmp(const void *a, const void *b) return diff; } -static int index_icmp(const void *a, const void *b) +int git_index_entry_icmp(const void *a, const void *b) { int diff; const git_index_entry *entry_a = a; @@ -351,15 +341,22 @@ void git_index__set_ignore_case(git_index *index, bool ignore_case) { index->ignore_case = ignore_case; - index->entries_cmp_path = ignore_case ? index_icmp_path : index_cmp_path; - index->entries_search = ignore_case ? index_isrch : index_srch; - index->entries_search_path = ignore_case ? index_isrch_path : index_srch_path; + if (ignore_case) { + index->entries_cmp_path = git__strcasecmp_cb; + index->entries_search = git_index_entry_isrch; + index->entries_search_path = index_entry_isrch_path; + index->reuc_search = reuc_isrch; + } else { + index->entries_cmp_path = git__strcmp_cb; + index->entries_search = git_index_entry_srch; + index->entries_search_path = index_entry_srch_path; + index->reuc_search = reuc_srch; + } - git_vector_set_cmp(&index->entries, ignore_case ? index_icmp : index_cmp); + git_vector_set_cmp(&index->entries, + ignore_case ? git_index_entry_icmp : git_index_entry_cmp); index_sort_if_needed(index, true); - index->reuc_search = ignore_case ? reuc_isrch : reuc_srch; - git_vector_set_cmp(&index->reuc, ignore_case ? reuc_icmp : reuc_cmp); git_vector_sort(&index->reuc); } @@ -390,15 +387,15 @@ int git_index_open(git_index **index_out, const char *index_path) index->on_disk = 1; } - if (git_vector_init(&index->entries, 32, index_cmp) < 0 || + if (git_vector_init(&index->entries, 32, git_index_entry_cmp) < 0 || git_vector_init(&index->names, 32, conflict_name_cmp) < 0 || git_vector_init(&index->reuc, 32, reuc_cmp) < 0 || - git_vector_init(&index->deleted, 2, index_cmp) < 0) + git_vector_init(&index->deleted, 2, git_index_entry_cmp) < 0) goto fail; - index->entries_cmp_path = index_cmp_path; - index->entries_search = index_srch; - index->entries_search_path = index_srch_path; + index->entries_cmp_path = git__strcmp_cb; + index->entries_search = git_index_entry_srch; + index->entries_search_path = index_entry_srch_path; index->reuc_search = reuc_srch; if (index_path != NULL && (error = git_index_read(index, true)) < 0) @@ -727,22 +724,6 @@ void git_index_entry__init_from_stat( entry->file_size = st->st_size; } -int git_index_entry__cmp(const void *a, const void *b) -{ - const git_index_entry *entry_a = a; - const git_index_entry *entry_b = b; - - return strcmp(entry_a->path, entry_b->path); -} - -int git_index_entry__cmp_icase(const void *a, const void *b) -{ - const git_index_entry *entry_a = a; - const git_index_entry *entry_b = b; - - return strcasecmp(entry_a->path, entry_b->path); -} - static int index_entry_init( git_index_entry **entry_out, git_index *index, const char *rel_path) { @@ -1129,14 +1110,14 @@ int git_index_remove_directory(git_index *index, const char *dir, int stage) } int git_index__find_in_entries( - size_t *out, git_vector *entries, git_vector_cmp entry_cmp, + size_t *out, git_vector *entries, git_vector_cmp entry_srch, const char *path, size_t path_len, int stage) { struct entry_srch_key srch_key; srch_key.path = path; srch_key.path_len = !path_len ? strlen(path) : path_len; srch_key.stage = stage; - return git_vector_bsearch2(out, entries, entry_cmp, &srch_key); + return git_vector_bsearch2(out, entries, entry_srch, &srch_key); } int git_index__find( @@ -2040,7 +2021,7 @@ static int write_entries(git_index *index, git_filebuf *file) /* If index->entries is sorted case-insensitively, then we need * to re-sort it case-sensitively before writing */ if (index->ignore_case) { - git_vector_dup(&case_sorted, &index->entries, index_cmp); + git_vector_dup(&case_sorted, &index->entries, git_index_entry_cmp); git_vector_sort(&case_sorted); entries = &case_sorted; } else { diff --git a/src/index.h b/src/index.h index c2a932218..cb4425885 100644 --- a/src/index.h +++ b/src/index.h @@ -53,8 +53,13 @@ struct git_index_conflict_iterator { extern void git_index_entry__init_from_stat( git_index_entry *entry, struct stat *st, bool trust_mode); -extern int git_index_entry__cmp(const void *a, const void *b); -extern int git_index_entry__cmp_icase(const void *a, const void *b); +/* Index entry comparison functions for array sorting */ +extern int git_index_entry_cmp(const void *a, const void *b); +extern int git_index_entry_icmp(const void *a, const void *b); + +/* Index entry search functions for search using a search spec */ +extern int git_index_entry_srch(const void *a, const void *b); +extern int git_index_entry_isrch(const void *a, const void *b); /* Search index for `path`, returning GIT_ENOTFOUND if it does not exist. * `at_pos` is set to the position where it is or would be inserted. @@ -83,7 +88,7 @@ extern void git_index__release_snapshot(git_index *index); /* Allow searching in a snapshot; entries must already be sorted! */ extern int git_index__find_in_entries( size_t *at_pos, - git_vector *entries, git_vector_cmp entry_cmp, + git_vector *entries, git_vector_cmp entry_srch, const char *path, size_t path_len, int stage); #endif diff --git a/src/iterator.c b/src/iterator.c index a84f4d3db..53ef278d1 100644 --- a/src/iterator.c +++ b/src/iterator.c @@ -868,16 +868,20 @@ int git_iterator_for_index( return error; } ii->index = index; - ii->entry_srch = index->entries_search; ITERATOR_BASE_INIT(ii, index, INDEX, git_index_owner(index)); - if (index->ignore_case) { - ii->base.flags |= GIT_ITERATOR_IGNORE_CASE; - ii->base.prefixcomp = git__prefixcmp_icase; + if ((error = iterator__update_ignore_case((git_iterator *)ii, flags)) < 0) { + git_iterator_free((git_iterator *)ii); + return error; } - /* TODO: resort entries to match desired ignore_case behavior */ + ii->entry_srch = iterator__ignore_case(ii) ? + git_index_entry_isrch : git_index_entry_srch; + + git_vector_set_cmp(&ii->entries, iterator__ignore_case(ii) ? + git_index_entry_icmp : git_index_entry_cmp); + git_vector_sort(&ii->entries); git_buf_init(&ii->partial, 0); ii->tree_entry.mode = GIT_FILEMODE_TREE; diff --git a/src/merge.c b/src/merge.c index 9c4a07b58..68105d483 100644 --- a/src/merge.c +++ b/src/merge.c @@ -1332,7 +1332,7 @@ int git_merge_diff_list__find_differences( { git_iterator *iterators[3] = {0}; const git_index_entry *items[3] = {0}, *best_cur_item, *cur_items[3]; - git_vector_cmp entry_compare = git_index_entry__cmp; + git_vector_cmp entry_compare = git_index_entry_cmp; struct merge_diff_df_data df_data = {0}; int cur_item_modified; size_t i, j; diff --git a/src/util.c b/src/util.c index 3767b890c..39858254f 100644 --- a/src/util.c +++ b/src/util.c @@ -542,10 +542,12 @@ int git__bsearch_r( */ int git__strcmp_cb(const void *a, const void *b) { - const char *stra = (const char *)a; - const char *strb = (const char *)b; + return strcmp((const char *)a, (const char *)b); +} - return strcmp(stra, strb); +int git__strcasecmp_cb(const void *a, const void *b) +{ + return strcasecmp((const char *)a, (const char *)b); } int git__parse_bool(int *out, const char *value) diff --git a/src/util.h b/src/util.h index 5c2c563d6..d94463c65 100644 --- a/src/util.h +++ b/src/util.h @@ -196,6 +196,7 @@ extern int git__bsearch_r( size_t *position); extern int git__strcmp_cb(const void *a, const void *b); +extern int git__strcasecmp_cb(const void *a, const void *b); extern int git__strcmp(const char *a, const char *b); extern int git__strcasecmp(const char *a, const char *b); diff --git a/src/vector.c b/src/vector.c index 9f7eed5a3..c769b696a 100644 --- a/src/vector.c +++ b/src/vector.c @@ -177,7 +177,8 @@ void git_vector_sort(git_vector *v) if (git_vector_is_sorted(v) || !v->_cmp) return; - git__tsort(v->contents, v->length, v->_cmp); + if (v->length > 1) + git__tsort(v->contents, v->length, v->_cmp); git_vector_set_sorted(v, 1); } diff --git a/tests/diff/iterator.c b/tests/diff/iterator.c index 891d8a6e5..19a9a0077 100644 --- a/tests/diff/iterator.c +++ b/tests/diff/iterator.c @@ -355,6 +355,7 @@ static void index_iterator_test( const char *sandbox, const char *start, const char *end, + git_iterator_flag_t flags, int expected_count, const char **expected_names, const char **expected_oids) @@ -364,9 +365,12 @@ static void index_iterator_test( const git_index_entry *entry; int error, count = 0; git_repository *repo = cl_git_sandbox_init(sandbox); + unsigned int caps; cl_git_pass(git_repository_index(&index, repo)); - cl_git_pass(git_iterator_for_index(&i, index, 0, start, end)); + caps = git_index_caps(index); + + cl_git_pass(git_iterator_for_index(&i, index, flags, start, end)); while (!(error = git_iterator_advance(&entry, i))) { cl_assert(entry); @@ -388,6 +392,8 @@ static void index_iterator_test( cl_assert_equal_i(expected_count, count); git_iterator_free(i); + + cl_assert(caps == git_index_caps(index)); git_index_free(index); } @@ -446,7 +452,8 @@ static const char *expected_index_oids_0[] = { void test_diff_iterator__index_0(void) { index_iterator_test( - "attr", NULL, NULL, 23, expected_index_0, expected_index_oids_0); + "attr", NULL, NULL, 0, ARRAY_SIZE(expected_index_0), + expected_index_0, expected_index_oids_0); } static const char *expected_index_range[] = { @@ -466,25 +473,26 @@ static const char *expected_index_oids_range[] = { void test_diff_iterator__index_range(void) { index_iterator_test( - "attr", "root", "root", 4, expected_index_range, expected_index_oids_range); + "attr", "root", "root", 0, ARRAY_SIZE(expected_index_range), + expected_index_range, expected_index_oids_range); } void test_diff_iterator__index_range_empty_0(void) { index_iterator_test( - "attr", "empty", "empty", 0, NULL, NULL); + "attr", "empty", "empty", 0, 0, NULL, NULL); } void test_diff_iterator__index_range_empty_1(void) { index_iterator_test( - "attr", "z_empty_after", NULL, 0, NULL, NULL); + "attr", "z_empty_after", NULL, 0, 0, NULL, NULL); } void test_diff_iterator__index_range_empty_2(void) { index_iterator_test( - "attr", NULL, ".aaa_empty_before", 0, NULL, NULL); + "attr", NULL, ".aaa_empty_before", 0, 0, NULL, NULL); } static const char *expected_index_1[] = { @@ -522,9 +530,45 @@ static const char* expected_index_oids_1[] = { void test_diff_iterator__index_1(void) { index_iterator_test( - "status", NULL, NULL, 13, expected_index_1, expected_index_oids_1); + "status", NULL, NULL, 0, ARRAY_SIZE(expected_index_1), + expected_index_1, expected_index_oids_1); } +static const char *expected_index_cs[] = { + "B", "D", "F", "H", "J", "L/1", "L/B", "L/D", "L/a", "L/c", + "a", "c", "e", "g", "i", "k/1", "k/B", "k/D", "k/a", "k/c", +}; + +static const char *expected_index_ci[] = { + "a", "B", "c", "D", "e", "F", "g", "H", "i", "J", + "k/1", "k/a", "k/B", "k/c", "k/D", "L/1", "L/a", "L/B", "L/c", "L/D", +}; + +void test_diff_iterator__index_case_folding(void) +{ + git_buf path = GIT_BUF_INIT; + int fs_is_ci = 0; + + cl_git_pass(git_buf_joinpath(&path, cl_fixture("icase"), ".gitted/CoNfIg")); + fs_is_ci = git_path_exists(path.ptr); + git_buf_free(&path); + + index_iterator_test( + "icase", NULL, NULL, 0, ARRAY_SIZE(expected_index_cs), + fs_is_ci ? expected_index_ci : expected_index_cs, NULL); + + cl_git_sandbox_cleanup(); + + index_iterator_test( + "icase", NULL, NULL, GIT_ITERATOR_IGNORE_CASE, + ARRAY_SIZE(expected_index_ci), expected_index_ci, NULL); + + cl_git_sandbox_cleanup(); + + index_iterator_test( + "icase", NULL, NULL, GIT_ITERATOR_DONT_IGNORE_CASE, + ARRAY_SIZE(expected_index_cs), expected_index_cs, NULL); +} /* -- WORKDIR ITERATOR TESTS -- */ From 40ed499039f887ebcb0b5badf0157519148398b8 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Tue, 11 Feb 2014 14:45:37 -0800 Subject: [PATCH 07/20] Add diff threading tests and attr file cache locks This adds a basic test of doing simultaneous diffs on multiple threads and adds basic locking for the attr file cache because that was the immediate problem that arose from these tests. --- src/attr.c | 217 +++++++++++++++++++++++++++---------------- src/attr_file.c | 13 ++- src/attr_file.h | 1 + src/attrcache.h | 8 +- src/config_file.c | 20 ++-- src/diff_driver.c | 2 +- src/repository.h | 4 +- src/sortedcache.c | 5 +- src/strmap.h | 4 +- src/submodule.c | 3 +- tests/clar_libgit2.h | 2 +- tests/core/strmap.c | 72 +++++++------- tests/threads/diff.c | 152 ++++++++++++++++++++++++++++++ 13 files changed, 362 insertions(+), 141 deletions(-) create mode 100644 tests/threads/diff.c diff --git a/src/attr.c b/src/attr.c index d8a171d0f..f52a8a97b 100644 --- a/src/attr.c +++ b/src/attr.c @@ -33,6 +33,7 @@ static int collect_attr_files( const char *path, git_vector *files); +static void release_attr_files(git_vector *files); int git_attr_get( const char **value, @@ -76,7 +77,7 @@ int git_attr_get( } cleanup: - git_vector_free(&files); + release_attr_files(&files); git_attr_path__free(&path); return error; @@ -152,7 +153,7 @@ int git_attr_get_many( } cleanup: - git_vector_free(&files); + release_attr_files(&files); git_attr_path__free(&path); git__free(info); @@ -181,12 +182,10 @@ int git_attr_foreach( if (git_attr_path__init(&path, pathname, git_repository_workdir(repo)) < 0) return -1; - if ((error = collect_attr_files(repo, flags, pathname, &files)) < 0) + if ((error = collect_attr_files(repo, flags, pathname, &files)) < 0 || + (error = git_strmap_alloc(&seen)) < 0) goto cleanup; - seen = git_strmap_alloc(); - GITERR_CHECK_ALLOC(seen); - git_vector_foreach(&files, i, file) { git_attr_file__foreach_matching_rule(file, &path, j, rule) { @@ -211,7 +210,7 @@ int git_attr_foreach( cleanup: git_strmap_free(seen); - git_vector_free(&files); + release_attr_files(&files); git_attr_path__free(&path); return error; @@ -350,12 +349,21 @@ static int load_attr_from_cache( if (git_buf_printf(&cache_key, "%d#%s", (int)source, relative_path) < 0) return -1; + if (git_mutex_lock(&cache->lock) < 0) { + giterr_set(GITERR_OS, "Could not get cache attr lock"); + git_buf_free(&cache_key); + return -1; + } + cache_pos = git_strmap_lookup_index(cache->files, cache_key.ptr); - git_buf_free(&cache_key); - - if (git_strmap_valid_index(cache->files, cache_pos)) + if (git_strmap_valid_index(cache->files, cache_pos)) { *file = git_strmap_value_at(cache->files, cache_pos); + GIT_REFCOUNT_INC(*file); + } + + git_mutex_unlock(&cache->lock); + git_buf_free(&cache_key); return 0; } @@ -367,20 +375,26 @@ int git_attr_cache__internal_file( { int error = 0; git_attr_cache *cache = git_repository_attr_cache(repo); - khiter_t cache_pos = git_strmap_lookup_index(cache->files, filename); + khiter_t cache_pos; + + if (git_mutex_lock(&cache->lock) < 0) { + giterr_set(GITERR_OS, "Unable to get attr cache lock"); + return -1; + } + + cache_pos = git_strmap_lookup_index(cache->files, filename); if (git_strmap_valid_index(cache->files, cache_pos)) { *file = git_strmap_value_at(cache->files, cache_pos); - return 0; + } + else if (!(error = git_attr_file__new(file, 0, filename, &cache->pool))) { + + git_strmap_insert(cache->files, (*file)->key + 2, *file, error); + if (error > 0) + error = 0; } - if (git_attr_file__new(file, 0, filename, &cache->pool) < 0) - return -1; - - git_strmap_insert(cache->files, (*file)->key + 2, *file, error); - if (error > 0) - error = 0; - + git_mutex_unlock(&cache->lock); return error; } @@ -452,9 +466,17 @@ int git_attr_cache__push_file( if (parse && (error = parse(repo, parsedata, content, file)) < 0) goto finish; - git_strmap_insert(cache->files, file->key, file, error); //-V595 - if (error > 0) - error = 0; + if (git_mutex_lock(&cache->lock) < 0) { + giterr_set(GITERR_OS, "Unable to get attr cache lock"); + error = -1; + } else { + git_strmap_insert(cache->files, file->key, file, error); /* -V595 */ + if (error > 0) { /* > 0 means inserting for the first time */ + error = 0; + GIT_REFCOUNT_INC(file); + } + git_mutex_unlock(&cache->lock); + } /* remember "cache buster" file signature */ if (blob) @@ -481,7 +503,8 @@ finish: } #define push_attr_file(R,S,B,F) \ - git_attr_cache__push_file((R),(B),(F),GIT_ATTR_FILE_FROM_FILE,git_attr_file__parse_buffer,NULL,(S)) + git_attr_cache__push_file \ + ((R),(B),(F),GIT_ATTR_FILE_FROM_FILE,git_attr_file__parse_buffer,NULL,(S)) typedef struct { git_repository *repo; @@ -535,6 +558,18 @@ static int push_one_attr(void *ref, git_buf *path) return error; } +static void release_attr_files(git_vector *files) +{ + size_t i; + git_attr_file *file; + + git_vector_foreach(files, i, file) { + git_attr_file__free(file); + files->contents[i] = NULL; + } + git_vector_free(files); +} + static int collect_attr_files( git_repository *repo, uint32_t flags, @@ -600,7 +635,7 @@ static int collect_attr_files( cleanup: if (error < 0) - git_vector_free(files); + release_attr_files(files); git_buf_free(&dir); return error; @@ -637,61 +672,11 @@ static int attr_cache__lookup_path( return error; } -int git_attr_cache__init(git_repository *repo) +static void attr_cache__free(git_attr_cache *cache) { - int ret; - git_attr_cache *cache = git_repository_attr_cache(repo); - git_config *cfg; - - if (cache->initialized) - return 0; - - /* cache config settings for attributes and ignores */ - if (git_repository_config__weakptr(&cfg, repo) < 0) - return -1; - - ret = attr_cache__lookup_path( - &cache->cfg_attr_file, cfg, GIT_ATTR_CONFIG, GIT_ATTR_FILE_XDG); - if (ret < 0) - return ret; - - ret = attr_cache__lookup_path( - &cache->cfg_excl_file, cfg, GIT_IGNORE_CONFIG, GIT_IGNORE_FILE_XDG); - if (ret < 0) - return ret; - - /* allocate hashtable for attribute and ignore file contents */ - if (cache->files == NULL) { - cache->files = git_strmap_alloc(); - GITERR_CHECK_ALLOC(cache->files); - } - - /* allocate hashtable for attribute macros */ - if (cache->macros == NULL) { - cache->macros = git_strmap_alloc(); - GITERR_CHECK_ALLOC(cache->macros); - } - - /* allocate string pool */ - if (git_pool_init(&cache->pool, 1, 0) < 0) - return -1; - - cache->initialized = 1; - - /* insert default macros */ - return git_attr_add_macro(repo, "binary", "-diff -crlf -text"); -} - -void git_attr_cache_flush( - git_repository *repo) -{ - git_attr_cache *cache; - - if (!repo) + if (!cache) return; - cache = git_repository_attr_cache(repo); - if (cache->files != NULL) { git_attr_file *file; @@ -720,19 +705,93 @@ void git_attr_cache_flush( git__free(cache->cfg_excl_file); cache->cfg_excl_file = NULL; - cache->initialized = 0; + git_mutex_free(&cache->lock); + + git__free(cache); +} + +int git_attr_cache__init(git_repository *repo) +{ + int ret = 0; + git_attr_cache *cache = git_repository_attr_cache(repo); + git_config *cfg; + + if (cache) + return 0; + + if ((ret = git_repository_config__weakptr(&cfg, repo)) < 0) + return ret; + + cache = git__calloc(1, sizeof(git_attr_cache)); + GITERR_CHECK_ALLOC(cache); + + /* set up lock */ + if (git_mutex_init(&cache->lock) < 0) { + giterr_set(GITERR_OS, "Unable to initialize lock for attr cache"); + git__free(cache); + return -1; + } + + /* cache config settings for attributes and ignores */ + ret = attr_cache__lookup_path( + &cache->cfg_attr_file, cfg, GIT_ATTR_CONFIG, GIT_ATTR_FILE_XDG); + if (ret < 0) + goto cancel; + + ret = attr_cache__lookup_path( + &cache->cfg_excl_file, cfg, GIT_IGNORE_CONFIG, GIT_IGNORE_FILE_XDG); + if (ret < 0) + goto cancel; + + /* allocate hashtable for attribute and ignore file contents, + * hashtable for attribute macros, and string pool + */ + if ((ret = git_strmap_alloc(&cache->files)) < 0 || + (ret = git_strmap_alloc(&cache->macros)) < 0 || + (ret = git_pool_init(&cache->pool, 1, 0)) < 0) + goto cancel; + + cache = git__compare_and_swap(&repo->attrcache, NULL, cache); + if (cache) + goto cancel; /* raced with another thread, free this but no error */ + + /* insert default macros */ + return git_attr_add_macro(repo, "binary", "-diff -crlf -text"); + +cancel: + attr_cache__free(cache); + return ret; +} + +void git_attr_cache_flush(git_repository *repo) +{ + git_attr_cache *cache; + + /* this could be done less expensively, but for now, we'll just free + * the entire attrcache and let the next use reinitialize it... + */ + if (repo && (cache = git__swap(repo->attrcache, NULL)) != NULL) + attr_cache__free(cache); } int git_attr_cache__insert_macro(git_repository *repo, git_attr_rule *macro) { - git_strmap *macros = git_repository_attr_cache(repo)->macros; + git_attr_cache *cache = git_repository_attr_cache(repo); + git_strmap *macros = cache->macros; int error; /* TODO: generate warning log if (macro->assigns.length == 0) */ if (macro->assigns.length == 0) return 0; - git_strmap_insert(macros, macro->match.pattern, macro, error); + if (git_mutex_lock(&cache->lock) < 0) { + giterr_set(GITERR_OS, "Unable to get attr cache lock"); + error = -1; + } else { + git_strmap_insert(macros, macro->match.pattern, macro, error); + git_mutex_unlock(&cache->lock); + } + return (error < 0) ? -1 : 0; } diff --git a/src/attr_file.c b/src/attr_file.c index ea92336f7..695f661a8 100644 --- a/src/attr_file.c +++ b/src/attr_file.c @@ -23,6 +23,7 @@ int git_attr_file__new( attrs = git__calloc(1, sizeof(git_attr_file)); GITERR_CHECK_ALLOC(attrs); + GIT_REFCOUNT_INC(attrs); if (pool) attrs->pool = pool; @@ -152,11 +153,8 @@ void git_attr_file__clear_rules(git_attr_file *file) git_vector_free(&file->rules); } -void git_attr_file__free(git_attr_file *file) +static void attr_file_free(git_attr_file *file) { - if (!file) - return; - git_attr_file__clear_rules(file); if (file->pool_is_allocated) { @@ -168,6 +166,13 @@ void git_attr_file__free(git_attr_file *file) git__free(file); } +void git_attr_file__free(git_attr_file *file) +{ + if (!file) + return; + GIT_REFCOUNT_DEC(file, attr_file_free); +} + uint32_t git_attr_file__name_hash(const char *name) { uint32_t h = 5381; diff --git a/src/attr_file.h b/src/attr_file.h index 3bc7c6cb8..dbd6696c9 100644 --- a/src/attr_file.h +++ b/src/attr_file.h @@ -64,6 +64,7 @@ typedef struct { } git_attr_assignment; typedef struct { + git_refcount rc; char *key; /* cache "source#path" this was loaded from */ git_vector rules; /* vector of or */ git_pool *pool; diff --git a/src/attrcache.h b/src/attrcache.h index 077633b87..4f9cff6bb 100644 --- a/src/attrcache.h +++ b/src/attrcache.h @@ -11,12 +11,12 @@ #include "strmap.h" typedef struct { - int initialized; - git_pool pool; - git_strmap *files; /* hash path to git_attr_file of rules */ - git_strmap *macros; /* hash name to vector */ char *cfg_attr_file; /* cached value of core.attributesfile */ char *cfg_excl_file; /* cached value of core.excludesfile */ + git_strmap *files; /* hash path to git_attr_file of rules */ + git_strmap *macros; /* hash name to vector */ + git_mutex lock; + git_pool pool; } git_attr_cache; extern int git_attr_cache__init(git_repository *repo); diff --git a/src/config_file.c b/src/config_file.c index aedf2cb12..bb26aa8a3 100644 --- a/src/config_file.c +++ b/src/config_file.c @@ -180,11 +180,15 @@ static int config_open(git_config_backend *cfg, git_config_level_t level) b->level = level; - b->values = git_strmap_alloc(); - GITERR_CHECK_ALLOC(b->values); + if ((res = git_strmap_alloc(&b->values)) < 0) + return res; git_array_init(b->readers); reader = git_array_alloc(b->readers); + if (!reader) { + git_strmap_free(b->values); + return -1; + } memset(reader, 0, sizeof(struct reader)); reader->file_path = git__strdup(b->file_path); @@ -205,6 +209,7 @@ static int config_open(git_config_backend *cfg, git_config_level_t level) reader = git_array_get(b->readers, 0); git_buf_free(&reader->buffer); + return res; } @@ -218,8 +223,10 @@ static int config_refresh(git_config_backend *cfg) for (i = 0; i < git_array_size(b->readers); i++) { reader = git_array_get(b->readers, i); + res = git_futils_readbuffer_updated( - &reader->buffer, reader->file_path, &reader->file_mtime, &reader->file_size, &updated); + &reader->buffer, reader->file_path, + &reader->file_mtime, &reader->file_size, &updated); if (res < 0) return (res == GIT_ENOTFOUND) ? 0 : res; @@ -233,10 +240,9 @@ static int config_refresh(git_config_backend *cfg) /* need to reload - store old values and prep for reload */ old_values = b->values; - b->values = git_strmap_alloc(); - GITERR_CHECK_ALLOC(b->values); - - if ((res = config_parse(b, reader, b->level, 0)) < 0) { + if ((res = git_strmap_alloc(&b->values)) < 0) { + b->values = old_values; + } else if ((res = config_parse(b, reader, b->level, 0)) < 0) { free_vars(b->values); b->values = old_values; } else { diff --git a/src/diff_driver.c b/src/diff_driver.c index 4c9a0af65..8136e0dd9 100644 --- a/src/diff_driver.c +++ b/src/diff_driver.c @@ -66,7 +66,7 @@ git_diff_driver_registry *git_diff_driver_registry_new() if (!reg) return NULL; - if ((reg->drivers = git_strmap_alloc()) == NULL) { + if (git_strmap_alloc(®->drivers) < 0) { git_diff_driver_registry_free(reg); return NULL; } diff --git a/src/repository.h b/src/repository.h index 99923b63b..86db488fd 100644 --- a/src/repository.h +++ b/src/repository.h @@ -108,7 +108,7 @@ struct git_repository { git_submodule_cache *_submodules; git_cache objects; - git_attr_cache attrcache; + git_attr_cache *attrcache; git_diff_driver_registry *diff_drivers; char *path_repository; @@ -123,7 +123,7 @@ struct git_repository { GIT_INLINE(git_attr_cache *) git_repository_attr_cache(git_repository *repo) { - return &repo->attrcache; + return repo->attrcache; } int git_repository_head_tree(git_tree **tree, git_repository *repo); diff --git a/src/sortedcache.c b/src/sortedcache.c index 13f0921f1..625322034 100644 --- a/src/sortedcache.c +++ b/src/sortedcache.c @@ -20,7 +20,7 @@ int git_sortedcache_new( if (git_pool_init(&sc->pool, 1, 0) < 0 || git_vector_init(&sc->items, 4, item_cmp) < 0 || - (sc->map = git_strmap_alloc()) == NULL) + git_strmap_alloc(&sc->map) < 0) goto fail; if (git_rwlock_init(&sc->lock)) { @@ -39,8 +39,7 @@ int git_sortedcache_new( return 0; fail: - if (sc->map) - git_strmap_free(sc->map); + git_strmap_free(sc->map); git_vector_free(&sc->items); git_pool_clear(&sc->pool); git__free(sc); diff --git a/src/strmap.h b/src/strmap.h index 8276ab468..8985aaf7e 100644 --- a/src/strmap.h +++ b/src/strmap.h @@ -22,7 +22,9 @@ typedef khiter_t git_strmap_iter; #define GIT__USE_STRMAP \ __KHASH_IMPL(str, static kh_inline, const char *, void *, 1, kh_str_hash_func, kh_str_hash_equal) -#define git_strmap_alloc() kh_init(str) +#define git_strmap_alloc(hp) \ + ((*(hp) = kh_init(str)) == NULL) ? giterr_set_oom(), -1 : 0 + #define git_strmap_free(h) kh_destroy(str, h), h = NULL #define git_strmap_clear(h) kh_clear(str, h) diff --git a/src/submodule.c b/src/submodule.c index bea096df5..95d3d0d9c 100644 --- a/src/submodule.c +++ b/src/submodule.c @@ -1638,8 +1638,7 @@ static int submodule_cache_alloc( return -1; } - cache->submodules = git_strmap_alloc(); - if (!cache->submodules) { + if (git_strmap_alloc(&cache->submodules) < 0) { submodule_cache_free(cache); return -1; } diff --git a/tests/clar_libgit2.h b/tests/clar_libgit2.h index c2489db38..d395bd66f 100644 --- a/tests/clar_libgit2.h +++ b/tests/clar_libgit2.h @@ -11,7 +11,7 @@ * * Use this wrapper around all `git_` library calls that return error codes! */ -#define cl_git_pass(expr) cl_git_pass_(expr, __FILE__, __LINE__) +#define cl_git_pass(expr) cl_git_pass_((expr), __FILE__, __LINE__) #define cl_git_pass_(expr, file, line) do { \ int _lg2_error; \ diff --git a/tests/core/strmap.c b/tests/core/strmap.c index f34a4f89f..a120f1feb 100644 --- a/tests/core/strmap.c +++ b/tests/core/strmap.c @@ -3,12 +3,22 @@ GIT__USE_STRMAP; +git_strmap *g_table; + +void test_core_strmap__initialize(void) +{ + cl_git_pass(git_strmap_alloc(&g_table)); + cl_assert(g_table != NULL); +} + +void test_core_strmap__cleanup(void) +{ + git_strmap_free(g_table); +} + void test_core_strmap__0(void) { - git_strmap *table = git_strmap_alloc(); - cl_assert(table != NULL); - cl_assert(git_strmap_num_entries(table) == 0); - git_strmap_free(table); + cl_assert(git_strmap_num_entries(g_table) == 0); } static void insert_strings(git_strmap *table, int count) @@ -37,21 +47,17 @@ void test_core_strmap__1(void) { int i; char *str; - git_strmap *table = git_strmap_alloc(); - cl_assert(table != NULL); - insert_strings(table, 20); + insert_strings(g_table, 20); - cl_assert(git_strmap_exists(table, "aaaaaaaaa")); - cl_assert(git_strmap_exists(table, "ggggggggg")); - cl_assert(!git_strmap_exists(table, "aaaaaaaab")); - cl_assert(!git_strmap_exists(table, "abcdefghi")); + cl_assert(git_strmap_exists(g_table, "aaaaaaaaa")); + cl_assert(git_strmap_exists(g_table, "ggggggggg")); + cl_assert(!git_strmap_exists(g_table, "aaaaaaaab")); + cl_assert(!git_strmap_exists(g_table, "abcdefghi")); i = 0; - git_strmap_foreach_value(table, str, { i++; free(str); }); + git_strmap_foreach_value(g_table, str, { i++; free(str); }); cl_assert(i == 20); - - git_strmap_free(table); } void test_core_strmap__2(void) @@ -59,44 +65,36 @@ void test_core_strmap__2(void) khiter_t pos; int i; char *str; - git_strmap *table = git_strmap_alloc(); - cl_assert(table != NULL); - insert_strings(table, 20); + insert_strings(g_table, 20); - cl_assert(git_strmap_exists(table, "aaaaaaaaa")); - cl_assert(git_strmap_exists(table, "ggggggggg")); - cl_assert(!git_strmap_exists(table, "aaaaaaaab")); - cl_assert(!git_strmap_exists(table, "abcdefghi")); + cl_assert(git_strmap_exists(g_table, "aaaaaaaaa")); + cl_assert(git_strmap_exists(g_table, "ggggggggg")); + cl_assert(!git_strmap_exists(g_table, "aaaaaaaab")); + cl_assert(!git_strmap_exists(g_table, "abcdefghi")); - cl_assert(git_strmap_exists(table, "bbbbbbbbb")); - pos = git_strmap_lookup_index(table, "bbbbbbbbb"); - cl_assert(git_strmap_valid_index(table, pos)); - cl_assert_equal_s(git_strmap_value_at(table, pos), "bbbbbbbbb"); - free(git_strmap_value_at(table, pos)); - git_strmap_delete_at(table, pos); + cl_assert(git_strmap_exists(g_table, "bbbbbbbbb")); + pos = git_strmap_lookup_index(g_table, "bbbbbbbbb"); + cl_assert(git_strmap_valid_index(g_table, pos)); + cl_assert_equal_s(git_strmap_value_at(g_table, pos), "bbbbbbbbb"); + free(git_strmap_value_at(g_table, pos)); + git_strmap_delete_at(g_table, pos); - cl_assert(!git_strmap_exists(table, "bbbbbbbbb")); + cl_assert(!git_strmap_exists(g_table, "bbbbbbbbb")); i = 0; - git_strmap_foreach_value(table, str, { i++; free(str); }); + git_strmap_foreach_value(g_table, str, { i++; free(str); }); cl_assert(i == 19); - - git_strmap_free(table); } void test_core_strmap__3(void) { int i; char *str; - git_strmap *table = git_strmap_alloc(); - cl_assert(table != NULL); - insert_strings(table, 10000); + insert_strings(g_table, 10000); i = 0; - git_strmap_foreach_value(table, str, { i++; free(str); }); + git_strmap_foreach_value(g_table, str, { i++; free(str); }); cl_assert(i == 10000); - - git_strmap_free(table); } diff --git a/tests/threads/diff.c b/tests/threads/diff.c new file mode 100644 index 000000000..33afc58ac --- /dev/null +++ b/tests/threads/diff.c @@ -0,0 +1,152 @@ +#include "clar_libgit2.h" +#include "thread-utils.h" + +static git_repository *g_repo; +static git_tree *a, *b; +static git_atomic counts[4]; + +void test_threads_diff__cleanup(void) +{ + cl_git_sandbox_cleanup(); +} + +static void run_in_parallel( + int repeats, int threads, void *(*func)(void *), + void (*before_test)(void), void (*after_test)(void)) +{ + int r, t, *id = git__calloc(threads, sizeof(int)); +#ifdef GIT_THREADS + git_thread *th = git__calloc(threads, sizeof(git_thread)); +#else + void *th = NULL; +#endif + + cl_assert(id != NULL && th != NULL); + + for (r = 0; r < repeats; ++r) { + g_repo = cl_git_sandbox_reopen(); /* reopen sandbox to flush caches */ + + if (before_test) before_test(); + + for (t = 0; t < threads; ++t) { + id[t] = t; +#ifdef GIT_THREADS + cl_git_pass(git_thread_create(&th[t], NULL, func, &id[t])); +#else + cl_assert(func(&id[t]) == &id[t]); +#endif + } + +#ifdef GIT_THREADS + for (t = 0; t < threads; ++t) + cl_git_pass(git_thread_join(th[t], NULL)); + memset(th, 0, threads * sizeof(git_thread)); +#endif + + if (after_test) after_test(); + } + + git__free(id); + git__free(th); +} + +static void setup_trees(void) +{ + cl_git_pass(git_revparse_single( + (git_object **)&a, g_repo, "0017bd4ab1^{tree}")); + cl_git_pass(git_revparse_single( + (git_object **)&b, g_repo, "26a125ee1b^{tree}")); + + memset(counts, 0, sizeof(counts)); +} + +#define THREADS 20 + +static void free_trees(void) +{ + git_tree_free(a); a = NULL; + git_tree_free(b); b = NULL; + + cl_assert_equal_i(288, git_atomic_get(&counts[0])); + cl_assert_equal_i(112, git_atomic_get(&counts[1])); + cl_assert_equal_i( 80, git_atomic_get(&counts[2])); + cl_assert_equal_i( 96, git_atomic_get(&counts[3])); +} + +static void *run_index_diffs(void *arg) +{ + int thread = *(int *)arg; + git_diff_options opts = GIT_DIFF_OPTIONS_INIT; + git_diff *diff = NULL; + size_t i; + int exp[4] = { 0, 0, 0, 0 }; + +// fprintf(stderr, "%d >>>\n", thread); + + switch (thread & 0x03) { + case 0: /* diff index to workdir */; + cl_git_pass(git_diff_index_to_workdir(&diff, g_repo, NULL, &opts)); + break; + case 1: /* diff tree 'a' to index */; + cl_git_pass(git_diff_tree_to_index(&diff, g_repo, a, NULL, &opts)); + break; + case 2: /* diff tree 'b' to index */; + cl_git_pass(git_diff_tree_to_index(&diff, g_repo, b, NULL, &opts)); + break; + case 3: /* diff index to workdir (explicit index) */; + { + git_index *idx; + cl_git_pass(git_repository_index(&idx, g_repo)); + cl_git_pass(git_diff_index_to_workdir(&diff, g_repo, idx, &opts)); + git_index_free(idx); + break; + } + } + +// fprintf(stderr, "%d <<<\n", thread); + + /* keep some diff stats to make sure results are as expected */ + + i = git_diff_num_deltas(diff); + git_atomic_add(&counts[0], (int32_t)i); + exp[0] = (int)i; + + while (i > 0) { + switch (git_diff_get_delta(diff, --i)->status) { + case GIT_DELTA_MODIFIED: exp[1]++; git_atomic_inc(&counts[1]); break; + case GIT_DELTA_ADDED: exp[2]++; git_atomic_inc(&counts[2]); break; + case GIT_DELTA_DELETED: exp[3]++; git_atomic_inc(&counts[3]); break; + default: break; + } + } + +// fprintf(stderr, "%2d: [%d] total %d (M %d A %d D %d)\n", +// thread, (int)(thread & 0x03), exp[0], exp[1], exp[2], exp[3]); + + switch (thread & 0x03) { + case 0: case 3: + cl_assert_equal_i(8, exp[0]); cl_assert_equal_i(4, exp[1]); + cl_assert_equal_i(0, exp[2]); cl_assert_equal_i(4, exp[3]); + break; + case 1: + cl_assert_equal_i(12, exp[0]); cl_assert_equal_i(3, exp[1]); + cl_assert_equal_i(7, exp[2]); cl_assert_equal_i(2, exp[3]); + break; + case 2: + cl_assert_equal_i(8, exp[0]); cl_assert_equal_i(3, exp[1]); + cl_assert_equal_i(3, exp[2]); cl_assert_equal_i(2, exp[3]); + break; + } + + git_diff_free(diff); + + return arg; +} + +void test_threads_diff__concurrent_diffs(void) +{ + g_repo = cl_git_sandbox_init("status"); + + run_in_parallel( + 20, 32, run_index_diffs, setup_trees, free_trees); +} From 8a2834d34173220c56bd1898397c0e6d200f327d Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Fri, 14 Mar 2014 13:20:51 -0700 Subject: [PATCH 08/20] Index locking and entry allocation changes This makes the lock management on the index a little bit broader, having a number of routines hold the lock across looking up the item to be modified and actually making the modification. Still not true thread safety, but more pure index modifications are now safe which allows the simple cases (such as starting up a diff while index modifications are underway) safe enough to get the snapshot without hitting allocation problems. As part of this, I simplified the allocation of index entries to use a flex array and just put the path at the end of the index entry. This makes every entry self-contained and makes it a little easier to feel sure that pointers to strings aren't being accidentally copied and freed while other references are still being held. --- src/common.h | 1 + src/index.c | 369 +++++++++++++++++++++++------------------- src/merge.c | 10 +- src/thread-utils.h | 7 + tests/diff/iterator.c | 3 +- tests/threads/diff.c | 119 ++++++++++---- 6 files changed, 308 insertions(+), 201 deletions(-) diff --git a/src/common.h b/src/common.h index d389cf85d..9c8bdc18a 100644 --- a/src/common.h +++ b/src/common.h @@ -46,6 +46,7 @@ # include # ifdef GIT_THREADS # include +# include # endif #define GIT_STDLIB_CALL diff --git a/src/index.c b/src/index.c index d733e4e48..094328b95 100644 --- a/src/index.c +++ b/src/index.c @@ -90,13 +90,24 @@ struct entry_long { struct entry_srch_key { const char *path; - size_t path_len; + size_t pathlen; int stage; }; +struct entry_internal { + git_index_entry entry; + size_t pathlen; + char path[GIT_FLEX_ARRAY]; +}; + +struct reuc_entry_internal { + git_index_reuc_entry entry; + size_t pathlen; + char path[GIT_FLEX_ARRAY]; +}; + /* local declarations */ static size_t read_extension(git_index *index, const char *buffer, size_t buffer_size); -static size_t read_entry(git_index_entry *dest, const void *buffer, size_t buffer_size); static int read_header(struct index_header *dest, const void *buffer); static int parse_index(git_index *index, const char *buffer, size_t buffer_size); @@ -109,12 +120,12 @@ static void index_entry_reuc_free(git_index_reuc_entry *reuc); int git_index_entry_srch(const void *key, const void *array_member) { const struct entry_srch_key *srch_key = key; - const git_index_entry *entry = array_member; + const struct entry_internal *entry = array_member; int cmp; size_t len1, len2, len; - len1 = srch_key->path_len; - len2 = strlen(entry->path); + len1 = srch_key->pathlen; + len2 = entry->pathlen; len = len1 < len2 ? len1 : len2; cmp = memcmp(srch_key->path, entry->path, len); @@ -126,7 +137,7 @@ int git_index_entry_srch(const void *key, const void *array_member) return 1; if (srch_key->stage != GIT_INDEX_STAGE_ANY) - return srch_key->stage - GIT_IDXENTRY_STAGE(entry); + return srch_key->stage - GIT_IDXENTRY_STAGE(&entry->entry); return 0; } @@ -134,12 +145,12 @@ int git_index_entry_srch(const void *key, const void *array_member) int git_index_entry_isrch(const void *key, const void *array_member) { const struct entry_srch_key *srch_key = key; - const git_index_entry *entry = array_member; + const struct entry_internal *entry = array_member; int cmp; size_t len1, len2, len; - len1 = srch_key->path_len; - len2 = strlen(entry->path); + len1 = srch_key->pathlen; + len2 = entry->pathlen; len = len1 < len2 ? len1 : len2; cmp = strncasecmp(srch_key->path, entry->path, len); @@ -152,7 +163,7 @@ int git_index_entry_isrch(const void *key, const void *array_member) return 1; if (srch_key->stage != GIT_INDEX_STAGE_ANY) - return srch_key->stage - GIT_IDXENTRY_STAGE(entry); + return srch_key->stage - GIT_IDXENTRY_STAGE(&entry->entry); return 0; } @@ -276,17 +287,11 @@ static int reuc_icmp(const void *a, const void *b) static void index_entry_reuc_free(git_index_reuc_entry *reuc) { - if (!reuc) - return; - git__free(reuc->path); git__free(reuc); } static void index_entry_free(git_index_entry *entry) { - if (!entry) - return; - git__free(entry->path); git__free(entry); } @@ -447,18 +452,22 @@ void git_index_free(git_index *index) /* call with locked index */ static void index_free_deleted(git_index *index) { + int readers = (int)git_atomic_get(&index->readers); size_t i; - if (git_atomic_get(&index->readers) > 0) + if (readers > 0) return; - for (i = 0; i < index->deleted.length; ++i) - index_entry_free(git__swap(index->deleted.contents[i], NULL)); + for (i = 0; i < index->deleted.length; ++i) { + git_index_entry *ie = git__swap(index->deleted.contents[i], NULL); + index_entry_free(ie); + } git_vector_clear(&index->deleted); } -static int index_remove_entry(git_index *index, size_t pos, bool need_lock) +/* call with locked index */ +static int index_remove_entry(git_index *index, size_t pos) { int error = 0; git_index_entry *entry = git_vector_get(&index->entries, pos); @@ -466,22 +475,16 @@ static int index_remove_entry(git_index *index, size_t pos, bool need_lock) if (entry != NULL) git_tree_cache_invalidate_path(index->tree, entry->path); - if (need_lock && git_mutex_lock(&index->lock) < 0) { - giterr_set(GITERR_OS, "Unable to lock index"); - return -1; - } - error = git_vector_remove(&index->entries, pos); if (!error) { - if (!git_atomic_get(&index->readers)) - index_entry_free(entry); - else - error = git_vector_insert(&index->deleted, entry); - } + int readers = (int)git_atomic_get(&index->readers); - if (need_lock) - git_mutex_unlock(&index->lock); + if (readers > 0) + error = git_vector_insert(&index->deleted, entry); + else + index_entry_free(entry); + } return error; } @@ -501,7 +504,7 @@ int git_index_clear(git_index *index) } while (!error && index->entries.length > 0) - error = index_remove_entry(index, index->entries.length - 1, false); + error = index_remove_entry(index, index->entries.length - 1); index_free_deleted(index); git_mutex_unlock(&index->lock); @@ -684,6 +687,16 @@ size_t git_index_entrycount(const git_index *index) return index->entries.length; } +GIT_INLINE(int) git_index__find_internal( + size_t *out, git_index *index, const char *path, size_t path_len, int stage, + bool need_lock) +{ + if (index_sort_if_needed(index, need_lock) < 0) + return -1; + return git_index__find_in_entries( + out, &index->entries, index->entries_search, path, path_len, stage); +} + const git_index_entry *git_index_get_byindex( git_index *index, size_t n) { @@ -700,7 +713,7 @@ const git_index_entry *git_index_get_bypath( assert(index); - if (git_index__find(&pos, index, path, 0, stage) < 0) { + if (git_index__find_internal(&pos, index, path, 0, stage, true) < 0) { giterr_set(GITERR_INDEX, "Index does not contain %s", path); return NULL; } @@ -724,6 +737,21 @@ void git_index_entry__init_from_stat( entry->file_size = st->st_size; } +static git_index_entry *index_entry_alloc(const char *path) +{ + size_t pathlen = strlen(path); + struct entry_internal *entry = + git__calloc(sizeof(struct entry_internal) + pathlen + 1, 1); + if (!entry) + return NULL; + + entry->pathlen = pathlen; + memcpy(entry->path, path, pathlen); + entry->entry.path = entry->path; + + return (git_index_entry *)entry; +} + static int index_entry_init( git_index_entry **entry_out, git_index *index, const char *rel_path) { @@ -743,22 +771,31 @@ static int index_entry_init( if (error < 0) return error; - entry = git__calloc(1, sizeof(git_index_entry)); + entry = index_entry_alloc(rel_path); GITERR_CHECK_ALLOC(entry); + entry->id = oid; git_index_entry__init_from_stat(entry, &st, !index->distrust_filemode); - entry->id = oid; - entry->path = git__strdup(rel_path); - if (!entry->path) { - git__free(entry); - return -1; - } - - *entry_out = entry; + *entry_out = (git_index_entry *)entry; return 0; } +static git_index_reuc_entry *reuc_entry_alloc(const char *path) +{ + size_t pathlen = strlen(path); + struct reuc_entry_internal *entry = + git__calloc(sizeof(struct reuc_entry_internal) + pathlen + 1, 1); + if (!entry) + return NULL; + + entry->pathlen = pathlen; + memcpy(entry->path, path, pathlen); + entry->entry.path = entry->path; + + return (git_index_reuc_entry *)entry; +} + static int index_entry_reuc_init(git_index_reuc_entry **reuc_out, const char *path, int ancestor_mode, const git_oid *ancestor_oid, @@ -769,17 +806,9 @@ static int index_entry_reuc_init(git_index_reuc_entry **reuc_out, assert(reuc_out && path); - *reuc_out = NULL; - - reuc = git__calloc(1, sizeof(git_index_reuc_entry)); + *reuc_out = reuc = reuc_entry_alloc(path); GITERR_CHECK_ALLOC(reuc); - reuc->path = git__strdup(path); - if (reuc->path == NULL) { - git__free(reuc); - return -1; - } - if ((reuc->mode[0] = ancestor_mode) > 0) git_oid_cpy(&reuc->oid[0], ancestor_oid); @@ -789,10 +818,16 @@ static int index_entry_reuc_init(git_index_reuc_entry **reuc_out, if ((reuc->mode[2] = their_mode) > 0) git_oid_cpy(&reuc->oid[2], their_oid); - *reuc_out = reuc; return 0; } +static void index_entry_cpy(git_index_entry *tgt, const git_index_entry *src) +{ + char *tgt_path = tgt->path; + memcpy(tgt, src, sizeof(*tgt)); + tgt->path = tgt_path; /* reset to existing path data */ +} + static int index_entry_dup(git_index_entry **out, const git_index_entry *src) { git_index_entry *entry; @@ -802,19 +837,10 @@ static int index_entry_dup(git_index_entry **out, const git_index_entry *src) return 0; } - entry = git__malloc(sizeof(git_index_entry)); + *out = entry = index_entry_alloc(src->path); GITERR_CHECK_ALLOC(entry); - memcpy(entry, src, sizeof(git_index_entry)); - - /* duplicate the path string so we own it */ - entry->path = git__strdup(entry->path); - if (!entry->path) { - git__free(entry); - return -1; - } - - *out = entry; + index_entry_cpy(entry, src); return 0; } @@ -827,13 +853,13 @@ static int has_file_name(git_index *index, const char *name = entry->path; while (pos < index->entries.length) { - git_index_entry *p = index->entries.contents[pos++]; + struct entry_internal *p = index->entries.contents[pos++]; - if (len >= strlen(p->path)) + if (len >= p->pathlen) break; if (memcmp(name, p->path, len)) break; - if (GIT_IDXENTRY_STAGE(p) != stage) + if (GIT_IDXENTRY_STAGE(&p->entry) != stage) continue; if (p->path[len] != '/') continue; @@ -841,7 +867,7 @@ static int has_file_name(git_index *index, if (!ok_to_replace) break; - if (index_remove_entry(index, --pos, true) < 0) + if (index_remove_entry(index, --pos) < 0) break; } return retval; @@ -860,7 +886,7 @@ static int has_dir_name(git_index *index, const char *slash = name + strlen(name); for (;;) { - size_t len, position; + size_t len, pos; for (;;) { if (*--slash == '/') @@ -870,12 +896,12 @@ static int has_dir_name(git_index *index, } len = slash - name; - if (git_index__find(&position, index, name, len, stage) == 0) { + if (!git_index__find_internal(&pos, index, name, len, stage, false)) { retval = -1; if (!ok_to_replace) break; - if (index_remove_entry(index, position, true) < 0) + if (index_remove_entry(index, pos) < 0) break; continue; } @@ -885,20 +911,19 @@ static int has_dir_name(git_index *index, * already matches the sub-directory, then we know * we're ok, and we can exit. */ - while (position < index->entries.length) { - git_index_entry *p = index->entries.contents[position]; + for (; pos < index->entries.length; ++pos) { + struct entry_internal *p = index->entries.contents[pos]; - if ((strlen(p->path) <= len) || - (p->path[len] != '/') || + if (p->pathlen <= len || + p->path[len] != '/' || memcmp(p->path, name, len)) break; /* not our subdirectory */ - if (GIT_IDXENTRY_STAGE(p) == stage) + if (GIT_IDXENTRY_STAGE(&p->entry) == stage) return retval; - - position++; } } + return retval; } @@ -909,7 +934,8 @@ static int check_file_directory_collision(git_index *index, retval = retval + has_dir_name(index, entry, ok_to_replace); if (retval) { - giterr_set(GITERR_INDEX, "'%s' appears as both a file an a directory", entry->path); + giterr_set(GITERR_INDEX, + "'%s' appears as both a file and a directory", entry->path); return -1; } @@ -931,10 +957,9 @@ static int index_insert( assert(index && entry_ptr); entry = *entry_ptr; - assert(entry && entry->path); /* make sure that the path length flag is correct */ - path_length = strlen(entry->path); + path_length = ((struct entry_internal *)entry)->pathlen; entry->flags &= ~GIT_IDXENTRY_NAMEMASK; @@ -943,43 +968,43 @@ static int index_insert( else entry->flags |= GIT_IDXENTRY_NAMEMASK; + if ((error = git_mutex_lock(&index->lock)) < 0) { + giterr_set(GITERR_OS, "Unable to acquire index lock"); + return error; + } + + git_vector_sort(&index->entries); + /* look if an entry with this path already exists */ - if (!git_index__find( - &position, index, entry->path, 0, GIT_IDXENTRY_STAGE(entry))) { + if (!git_index__find_internal( + &position, index, entry->path, 0, GIT_IDXENTRY_STAGE(entry), false)) { existing = index->entries.contents[position]; /* update filemode to existing values if stat is not trusted */ entry->mode = index_merge_mode(index, existing, entry->mode); } + /* look for tree / blob name collisions, removing conflicts if requested */ error = check_file_directory_collision(index, entry, position, replace); if (error < 0) - goto done; + /* skip changes */; /* if we are replacing an existing item, overwrite the existing entry * and return it in place of the passed in one. */ - if (existing && replace) { - git__free(entry->path); - entry->path = existing->path; - - memcpy(existing, entry, sizeof(*entry)); + else if (existing && replace) { + index_entry_cpy(existing, entry); + index_entry_free(entry); *entry_ptr = existing; - - git__free(entry); - return 0; } - - /* if replacing is not requested or no existing entry exists, just - * insert entry at the end; the index is no longer sorted - */ - if ((error = git_mutex_lock(&index->lock)) < 0) { - giterr_set(GITERR_OS, "Unable to acquire index lock"); - } else { + else { + /* if replacing is not requested or no existing entry exists, just + * insert entry at the end; the index is no longer sorted + */ error = git_vector_insert(&index->entries, entry); - git_mutex_unlock(&index->lock); } -done: + git_mutex_unlock(&index->lock); + if (error < 0) { index_entry_free(*entry_ptr); *entry_ptr = NULL; @@ -1053,7 +1078,7 @@ int git_index_add(git_index *index, const git_index_entry *source_entry) git_index_entry *entry = NULL; int ret; - assert(index && source_entry); + assert(index && source_entry && source_entry->path); if ((ret = index_entry_dup(&entry, source_entry)) < 0 || (ret = index_insert(index, &entry, 1)) < 0) @@ -1065,15 +1090,24 @@ int git_index_add(git_index *index, const git_index_entry *source_entry) int git_index_remove(git_index *index, const char *path, int stage) { + int error; size_t position; - if (git_index__find(&position, index, path, 0, stage) < 0) { - giterr_set(GITERR_INDEX, "Index does not contain %s at stage %d", - path, stage); - return GIT_ENOTFOUND; + if (git_mutex_lock(&index->lock) < 0) { + giterr_set(GITERR_OS, "Failed to lock index"); + return -1; } - return index_remove_entry(index, position, true); + if (git_index__find_internal(&position, index, path, 0, stage, false) < 0) { + giterr_set( + GITERR_INDEX, "Index does not contain %s at stage %d", path, stage); + error = GIT_ENOTFOUND; + } else { + error = index_remove_entry(index, position); + } + + git_mutex_unlock(&index->lock); + return error; } int git_index_remove_directory(git_index *index, const char *dir, int stage) @@ -1083,12 +1117,17 @@ int git_index_remove_directory(git_index *index, const char *dir, int stage) size_t pos; git_index_entry *entry; - if (git_buf_sets(&pfx, dir) < 0 || git_path_to_dir(&pfx) < 0) + if (git_mutex_lock(&index->lock) < 0) { + giterr_set(GITERR_OS, "Failed to lock index"); return -1; + } - git_index__find(&pos, index, pfx.ptr, pfx.size, GIT_INDEX_STAGE_ANY); + if (!(error = git_buf_sets(&pfx, dir)) && + !(error = git_path_to_dir(&pfx))) + git_index__find_internal( + &pos, index, pfx.ptr, pfx.size, GIT_INDEX_STAGE_ANY, false); - while (1) { + while (!error) { entry = git_vector_get(&index->entries, pos); if (!entry || git__prefixcmp(entry->path, pfx.ptr) != 0) break; @@ -1098,12 +1137,12 @@ int git_index_remove_directory(git_index *index, const char *dir, int stage) continue; } - if ((error = index_remove_entry(index, pos, true)) < 0) - break; + error = index_remove_entry(index, pos); - /* removed entry at 'pos' so we don't need to increment it */ + /* removed entry at 'pos' so we don't need to increment */ } + git_mutex_unlock(&index->lock); git_buf_free(&pfx); return error; @@ -1115,7 +1154,7 @@ int git_index__find_in_entries( { struct entry_srch_key srch_key; srch_key.path = path; - srch_key.path_len = !path_len ? strlen(path) : path_len; + srch_key.pathlen = !path_len ? strlen(path) : path_len; srch_key.stage = stage; return git_vector_bsearch2(out, entries, entry_srch, &srch_key); } @@ -1124,12 +1163,7 @@ int git_index__find( size_t *out, git_index *index, const char *path, size_t path_len, int stage) { assert(index && path); - - if (index_sort_if_needed(index, true) < 0) - return -1; - - return git_index__find_in_entries( - out, &index->entries, index->entries_search, path, path_len, stage); + return git_index__find_internal(out, index, path, path_len, stage, true); } int git_index_find(size_t *at_pos, git_index *index, const char *path) @@ -1138,7 +1172,13 @@ int git_index_find(size_t *at_pos, git_index *index, const char *path) assert(index && path); + if (git_mutex_lock(&index->lock) < 0) { + giterr_set(GITERR_OS, "Failed to lock index"); + return -1; + } + if (git_vector_bsearch2(&pos, &index->entries, index->entries_search_path, path) < 0) { + git_mutex_unlock(&index->lock); giterr_set(GITERR_INDEX, "Index does not contain %s", path); return GIT_ENOTFOUND; } @@ -1158,6 +1198,7 @@ int git_index_find(size_t *at_pos, git_index *index, const char *path) if (at_pos) *at_pos = pos; + git_mutex_unlock(&index->lock); return 0; } @@ -1288,6 +1329,11 @@ int git_index_conflict_remove(git_index *index, const char *path) if (git_index_find(&pos, index, path) < 0) return GIT_ENOTFOUND; + if (git_mutex_lock(&index->lock) < 0) { + giterr_set(GITERR_OS, "Unable to lock index"); + return -1; + } + while ((conflict_entry = git_vector_get(&index->entries, pos)) != NULL) { if (index->entries_cmp_path(conflict_entry->path, path) != 0) @@ -1298,10 +1344,12 @@ int git_index_conflict_remove(git_index *index, const char *path) continue; } - if ((error = index_remove_entry(index, pos, true)) < 0) + if ((error = index_remove_entry(index, pos)) < 0) break; } + git_mutex_unlock(&index->lock); + return error; } @@ -1311,7 +1359,7 @@ static int index_conflicts_match(const git_vector *v, size_t idx, void *p) git_index_entry *entry = git_vector_get(v, idx); if (GIT_IDXENTRY_STAGE(entry) > 0 && - !index_remove_entry(index, idx, false)) + !index_remove_entry(index, idx)) return 1; return 0; @@ -1488,7 +1536,6 @@ static int index_reuc_insert( return git_vector_insert(&index->reuc, reuc); /* exists, replace it */ - git__free((*existing)->path); git__free(*existing); *existing = reuc; @@ -1596,13 +1643,9 @@ static int read_reuc(git_index *index, const char *buffer, size_t size) if (size <= len) return index_error_invalid("reading reuc entries"); - lost = git__calloc(1, sizeof(git_index_reuc_entry)); + lost = reuc_entry_alloc(buffer); GITERR_CHECK_ALLOC(lost); - /* read NUL-terminated pathname for entry */ - lost->path = git__strdup(buffer); - GITERR_CHECK_ALLOC(lost->path); - size -= len; buffer += len; @@ -1700,41 +1743,41 @@ static int read_conflict_names(git_index *index, const char *buffer, size_t size return 0; } -static size_t read_entry(git_index_entry *dest, const void *buffer, size_t buffer_size) +static size_t read_entry( + git_index_entry **out, const void *buffer, size_t buffer_size) { size_t path_length, entry_size; uint16_t flags_raw; const char *path_ptr; const struct entry_short *source = buffer; + git_index_entry entry = {{0}}; if (INDEX_FOOTER_SIZE + minimal_entry_size > buffer_size) return 0; - memset(dest, 0x0, sizeof(git_index_entry)); + entry.ctime.seconds = (git_time_t)ntohl(source->ctime.seconds); + entry.ctime.nanoseconds = ntohl(source->ctime.nanoseconds); + entry.mtime.seconds = (git_time_t)ntohl(source->mtime.seconds); + entry.mtime.nanoseconds = ntohl(source->mtime.nanoseconds); + entry.dev = ntohl(source->dev); + entry.ino = ntohl(source->ino); + entry.mode = ntohl(source->mode); + entry.uid = ntohl(source->uid); + entry.gid = ntohl(source->gid); + entry.file_size = ntohl(source->file_size); + git_oid_cpy(&entry.id, &source->oid); + entry.flags = ntohs(source->flags); - dest->ctime.seconds = (git_time_t)ntohl(source->ctime.seconds); - dest->ctime.nanoseconds = ntohl(source->ctime.nanoseconds); - dest->mtime.seconds = (git_time_t)ntohl(source->mtime.seconds); - dest->mtime.nanoseconds = ntohl(source->mtime.nanoseconds); - dest->dev = ntohl(source->dev); - dest->ino = ntohl(source->ino); - dest->mode = ntohl(source->mode); - dest->uid = ntohl(source->uid); - dest->gid = ntohl(source->gid); - dest->file_size = ntohl(source->file_size); - git_oid_cpy(&dest->id, &source->oid); - dest->flags = ntohs(source->flags); - - if (dest->flags & GIT_IDXENTRY_EXTENDED) { + if (entry.flags & GIT_IDXENTRY_EXTENDED) { const struct entry_long *source_l = (const struct entry_long *)source; path_ptr = source_l->path; flags_raw = ntohs(source_l->flags_extended); - memcpy(&dest->flags_extended, &flags_raw, 2); + memcpy(&entry.flags_extended, &flags_raw, 2); } else path_ptr = source->path; - path_length = dest->flags & GIT_IDXENTRY_NAMEMASK; + path_length = entry.flags & GIT_IDXENTRY_NAMEMASK; /* if this is a very long string, we must find its * real length without overflowing */ @@ -1748,7 +1791,7 @@ static size_t read_entry(git_index_entry *dest, const void *buffer, size_t buffe path_length = path_end - path_ptr; } - if (dest->flags & GIT_IDXENTRY_EXTENDED) + if (entry.flags & GIT_IDXENTRY_EXTENDED) entry_size = long_entry_size(path_length); else entry_size = short_entry_size(path_length); @@ -1756,8 +1799,10 @@ static size_t read_entry(git_index_entry *dest, const void *buffer, size_t buffe if (INDEX_FOOTER_SIZE + entry_size > buffer_size) return 0; - dest->path = git__strdup(path_ptr); - assert(dest->path); + entry.path = (char *)path_ptr; + + if (index_entry_dup(out, &entry) < 0) + return 0; return entry_size; } @@ -1853,20 +1898,12 @@ static int parse_index(git_index *index, const char *buffer, size_t buffer_size) return -1; } - git_vector_clear(&index->entries); + assert(!index->entries.length); /* Parse all the entries */ for (i = 0; i < header.entry_count && buffer_size > INDEX_FOOTER_SIZE; ++i) { - size_t entry_size; git_index_entry *entry; - - entry = git__malloc(sizeof(git_index_entry)); - if (!entry) { - error = -1; - goto done; - } - - entry_size = read_entry(entry, buffer, buffer_size); + size_t entry_size = read_entry(&entry, buffer, buffer_size); /* 0 bytes read means an object corruption */ if (entry_size == 0) { @@ -1874,8 +1911,10 @@ static int parse_index(git_index *index, const char *buffer, size_t buffer_size) goto done; } - if ((error = git_vector_insert(&index->entries, entry)) < 0) + if ((error = git_vector_insert(&index->entries, entry)) < 0) { + index_entry_free(entry); goto done; + } seek_forward(entry_size); } @@ -1953,7 +1992,7 @@ static int write_disk_entry(git_filebuf *file, git_index_entry *entry) size_t path_len, disk_size; char *path; - path_len = strlen(entry->path); + path_len = ((struct entry_internal *)entry)->pathlen; if (entry->flags & GIT_IDXENTRY_EXTENDED) disk_size = long_entry_size(path_len); @@ -2222,7 +2261,7 @@ static int read_tree_cb( if (git_buf_joinpath(&path, root, tentry->filename) < 0) return -1; - entry = git__calloc(1, sizeof(git_index_entry)); + entry = index_entry_alloc(path.ptr); GITERR_CHECK_ALLOC(entry); entry->mode = tentry->attr; @@ -2236,7 +2275,9 @@ static int read_tree_cb( entry->mode == old_entry->mode && git_oid_equal(&entry->id, &old_entry->id)) { + char *oldpath = entry->path; memcpy(entry, old_entry, sizeof(*entry)); + entry->path = oldpath; entry->flags_extended = 0; } @@ -2245,7 +2286,6 @@ static int read_tree_cb( else entry->flags = GIT_IDXENTRY_NAMEMASK; - entry->path = git_buf_detach(&path); git_buf_free(&path); if (git_vector_insert(data->new_entries, entry) < 0) { @@ -2543,10 +2583,11 @@ int git_index__snapshot(git_vector *entries, git_index *index) void git_index__release_snapshot(git_index *index) { git_atomic_dec(&index->readers); - git_index_free(index); if (!git_mutex_lock(&index->lock)) { index_free_deleted(index); /* try to free pending deleted items */ git_mutex_unlock(&index->lock); } + + git_index_free(index); } diff --git a/src/merge.c b/src/merge.c index 68105d483..2e40b6db8 100644 --- a/src/merge.c +++ b/src/merge.c @@ -1230,7 +1230,7 @@ done: return error; } -GIT_INLINE(int) index_entry_dup( +GIT_INLINE(int) index_entry_dup_pool( git_index_entry *out, git_pool *pool, const git_index_entry *src) @@ -1276,9 +1276,9 @@ static git_merge_diff *merge_diff_from_index_entries( if ((conflict = git_pool_malloc(pool, sizeof(git_merge_diff))) == NULL) return NULL; - if (index_entry_dup(&conflict->ancestor_entry, pool, entries[TREE_IDX_ANCESTOR]) < 0 || - index_entry_dup(&conflict->our_entry, pool, entries[TREE_IDX_OURS]) < 0 || - index_entry_dup(&conflict->their_entry, pool, entries[TREE_IDX_THEIRS]) < 0) + if (index_entry_dup_pool(&conflict->ancestor_entry, pool, entries[TREE_IDX_ANCESTOR]) < 0 || + index_entry_dup_pool(&conflict->our_entry, pool, entries[TREE_IDX_OURS]) < 0 || + index_entry_dup_pool(&conflict->their_entry, pool, entries[TREE_IDX_THEIRS]) < 0) return NULL; conflict->our_status = merge_delta_type_from_index_entries( @@ -1318,7 +1318,7 @@ static int merge_diff_list_insert_unmodified( entry = git_pool_malloc(&diff_list->pool, sizeof(git_index_entry)); GITERR_CHECK_ALLOC(entry); - if ((error = index_entry_dup(entry, &diff_list->pool, tree_items[0])) >= 0) + if ((error = index_entry_dup_pool(entry, &diff_list->pool, tree_items[0])) >= 0) error = git_vector_insert(&diff_list->staged, entry); return error; diff --git a/src/thread-utils.h b/src/thread-utils.h index 914c1357d..50d8610a3 100644 --- a/src/thread-utils.h +++ b/src/thread-utils.h @@ -47,6 +47,12 @@ typedef git_atomic git_atomic_ssize; #define git_thread_exit(status) pthread_exit(status) #define git_thread_join(id, status) pthread_join(id, status) +#if defined(GIT_WIN32) +#define git_thread_yield() Sleep(0) +#else +#define git_thread_yield() sched_yield() +#endif + /* Pthreads Mutex */ #define git_mutex pthread_mutex_t #define git_mutex_init(a) pthread_mutex_init(a, NULL) @@ -176,6 +182,7 @@ GIT_INLINE(int64_t) git_atomic64_add(git_atomic64 *a, int64_t addend) #define git_thread_kill(thread) (void)0 #define git_thread_exit(status) (void)0 #define git_thread_join(id, status) (void)0 +#define git_thread_yield() (void)0 /* Pthreads Mutex */ #define git_mutex unsigned int diff --git a/tests/diff/iterator.c b/tests/diff/iterator.c index 19a9a0077..cdc64eb1d 100644 --- a/tests/diff/iterator.c +++ b/tests/diff/iterator.c @@ -363,9 +363,8 @@ static void index_iterator_test( git_index *index; git_iterator *i; const git_index_entry *entry; - int error, count = 0; + int error, count = 0, caps; git_repository *repo = cl_git_sandbox_init(sandbox); - unsigned int caps; cl_git_pass(git_repository_index(&index, repo)); caps = git_index_caps(index); diff --git a/tests/threads/diff.c b/tests/threads/diff.c index 33afc58ac..7f10a699d 100644 --- a/tests/threads/diff.c +++ b/tests/threads/diff.c @@ -1,9 +1,10 @@ #include "clar_libgit2.h" #include "thread-utils.h" -static git_repository *g_repo; -static git_tree *a, *b; -static git_atomic counts[4]; +static git_repository *_repo; +static git_tree *_a, *_b; +static git_atomic _counts[4]; +static int _check_counts; void test_threads_diff__cleanup(void) { @@ -24,7 +25,7 @@ static void run_in_parallel( cl_assert(id != NULL && th != NULL); for (r = 0; r < repeats; ++r) { - g_repo = cl_git_sandbox_reopen(); /* reopen sandbox to flush caches */ + _repo = cl_git_sandbox_reopen(); /* reopen sandbox to flush caches */ if (before_test) before_test(); @@ -53,24 +54,26 @@ static void run_in_parallel( static void setup_trees(void) { cl_git_pass(git_revparse_single( - (git_object **)&a, g_repo, "0017bd4ab1^{tree}")); + (git_object **)&_a, _repo, "0017bd4ab1^{tree}")); cl_git_pass(git_revparse_single( - (git_object **)&b, g_repo, "26a125ee1b^{tree}")); + (git_object **)&_b, _repo, "26a125ee1b^{tree}")); - memset(counts, 0, sizeof(counts)); + memset(_counts, 0, sizeof(_counts)); } #define THREADS 20 static void free_trees(void) { - git_tree_free(a); a = NULL; - git_tree_free(b); b = NULL; + git_tree_free(_a); _a = NULL; + git_tree_free(_b); _b = NULL; - cl_assert_equal_i(288, git_atomic_get(&counts[0])); - cl_assert_equal_i(112, git_atomic_get(&counts[1])); - cl_assert_equal_i( 80, git_atomic_get(&counts[2])); - cl_assert_equal_i( 96, git_atomic_get(&counts[3])); + if (_check_counts) { + cl_assert_equal_i(288, git_atomic_get(&_counts[0])); + cl_assert_equal_i(112, git_atomic_get(&_counts[1])); + cl_assert_equal_i( 80, git_atomic_get(&_counts[2])); + cl_assert_equal_i( 96, git_atomic_get(&_counts[3])); + } } static void *run_index_diffs(void *arg) @@ -81,48 +84,41 @@ static void *run_index_diffs(void *arg) size_t i; int exp[4] = { 0, 0, 0, 0 }; -// fprintf(stderr, "%d >>>\n", thread); - switch (thread & 0x03) { case 0: /* diff index to workdir */; - cl_git_pass(git_diff_index_to_workdir(&diff, g_repo, NULL, &opts)); + cl_git_pass(git_diff_index_to_workdir(&diff, _repo, NULL, &opts)); break; case 1: /* diff tree 'a' to index */; - cl_git_pass(git_diff_tree_to_index(&diff, g_repo, a, NULL, &opts)); + cl_git_pass(git_diff_tree_to_index(&diff, _repo, _a, NULL, &opts)); break; case 2: /* diff tree 'b' to index */; - cl_git_pass(git_diff_tree_to_index(&diff, g_repo, b, NULL, &opts)); + cl_git_pass(git_diff_tree_to_index(&diff, _repo, _b, NULL, &opts)); break; case 3: /* diff index to workdir (explicit index) */; { git_index *idx; - cl_git_pass(git_repository_index(&idx, g_repo)); - cl_git_pass(git_diff_index_to_workdir(&diff, g_repo, idx, &opts)); + cl_git_pass(git_repository_index(&idx, _repo)); + cl_git_pass(git_diff_index_to_workdir(&diff, _repo, idx, &opts)); git_index_free(idx); break; } } -// fprintf(stderr, "%d <<<\n", thread); - /* keep some diff stats to make sure results are as expected */ i = git_diff_num_deltas(diff); - git_atomic_add(&counts[0], (int32_t)i); + git_atomic_add(&_counts[0], (int32_t)i); exp[0] = (int)i; while (i > 0) { switch (git_diff_get_delta(diff, --i)->status) { - case GIT_DELTA_MODIFIED: exp[1]++; git_atomic_inc(&counts[1]); break; - case GIT_DELTA_ADDED: exp[2]++; git_atomic_inc(&counts[2]); break; - case GIT_DELTA_DELETED: exp[3]++; git_atomic_inc(&counts[3]); break; + case GIT_DELTA_MODIFIED: exp[1]++; git_atomic_inc(&_counts[1]); break; + case GIT_DELTA_ADDED: exp[2]++; git_atomic_inc(&_counts[2]); break; + case GIT_DELTA_DELETED: exp[3]++; git_atomic_inc(&_counts[3]); break; default: break; } } -// fprintf(stderr, "%2d: [%d] total %d (M %d A %d D %d)\n", -// thread, (int)(thread & 0x03), exp[0], exp[1], exp[2], exp[3]); - switch (thread & 0x03) { case 0: case 3: cl_assert_equal_i(8, exp[0]); cl_assert_equal_i(4, exp[1]); @@ -145,8 +141,71 @@ static void *run_index_diffs(void *arg) void test_threads_diff__concurrent_diffs(void) { - g_repo = cl_git_sandbox_init("status"); + _repo = cl_git_sandbox_init("status"); + _check_counts = 1; run_in_parallel( 20, 32, run_index_diffs, setup_trees, free_trees); } + +static void *run_index_diffs_with_modifier(void *arg) +{ + int thread = *(int *)arg; + git_diff_options opts = GIT_DIFF_OPTIONS_INIT; + git_diff *diff = NULL; + git_index *idx = NULL; + + cl_git_pass(git_repository_index(&idx, _repo)); + + /* have first thread altering the index as we go */ + if (thread == 0) { + int i; + + for (i = 0; i < 300; ++i) { + switch (i & 0x03) { + case 0: (void)git_index_add_bypath(idx, "new_file"); break; + case 1: (void)git_index_remove_bypath(idx, "modified_file"); break; + case 2: (void)git_index_remove_bypath(idx, "new_file"); break; + case 3: (void)git_index_add_bypath(idx, "modified_file"); break; + } + git_thread_yield(); + } + + git_index_free(idx); + return arg; + } + + /* only use explicit index in this test to prevent reloading */ + + switch (thread & 0x03) { + case 0: /* diff index to workdir */; + cl_git_pass(git_diff_index_to_workdir(&diff, _repo, idx, &opts)); + break; + case 1: /* diff tree 'a' to index */; + cl_git_pass(git_diff_tree_to_index(&diff, _repo, _a, idx, &opts)); + break; + case 2: /* diff tree 'b' to index */; + cl_git_pass(git_diff_tree_to_index(&diff, _repo, _b, idx, &opts)); + break; + case 3: /* diff index to workdir reversed */; + opts.flags |= GIT_DIFF_REVERSE; + cl_git_pass(git_diff_index_to_workdir(&diff, _repo, idx, &opts)); + break; + } + + /* results will be unpredictable with index modifier thread running */ + + git_diff_free(diff); + git_index_free(idx); + + return arg; +} + +void test_threads_diff__with_concurrent_index_modified(void) +{ + _repo = cl_git_sandbox_init("status"); + _check_counts = 0; + + run_in_parallel( + 20, 32, run_index_diffs_with_modifier, setup_trees, free_trees); +} From 52bb0476a8ad462c56b1d2c68b6d5f20f947ee50 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Fri, 14 Mar 2014 13:53:15 -0700 Subject: [PATCH 09/20] Clean up index snapshot function naming Clear up some of the various "find" functions and the snapshot API naming to be things I like more. --- src/checkout.c | 6 ++-- src/index.c | 86 +++++++++++++++++++++++++++----------------------- src/index.h | 16 ++++++---- src/iterator.c | 7 ++-- src/pathspec.c | 2 +- 5 files changed, 63 insertions(+), 54 deletions(-) diff --git a/src/checkout.c b/src/checkout.c index a251c08c0..0b385226b 100644 --- a/src/checkout.c +++ b/src/checkout.c @@ -279,17 +279,17 @@ static int checkout_action_wd_only( if (data->index != NULL) { size_t pos; - error = git_index__find( + error = git_index__find_pos( &pos, data->index, wd->path, 0, GIT_INDEX_STAGE_ANY); if (wd->mode != GIT_FILEMODE_TREE) { - if (!error) { /* found by git_index__find call */ + if (!error) { /* found by git_index__find_pos call */ notify = GIT_CHECKOUT_NOTIFY_DIRTY; remove = ((data->strategy & GIT_CHECKOUT_FORCE) != 0); } else if (error != GIT_ENOTFOUND) return error; else - error = 0; /* git_index__find does not set error msg */ + error = 0; /* git_index__find_pos does not set error msg */ } else { /* for tree entries, we have to see if there are any index * entries that are contained inside that tree diff --git a/src/index.c b/src/index.c index 094328b95..10fb1fea7 100644 --- a/src/index.c +++ b/src/index.c @@ -342,6 +342,28 @@ static int index_sort_if_needed(git_index *index, bool need_lock) return 0; } +GIT_INLINE(int) index_find_in_entries( + size_t *out, git_vector *entries, git_vector_cmp entry_srch, + const char *path, size_t path_len, int stage) +{ + struct entry_srch_key srch_key; + srch_key.path = path; + srch_key.pathlen = !path_len ? strlen(path) : path_len; + srch_key.stage = stage; + return git_vector_bsearch2(out, entries, entry_srch, &srch_key); +} + +GIT_INLINE(int) index_find( + size_t *out, git_index *index, + const char *path, size_t path_len, int stage, bool need_lock) +{ + if (index_sort_if_needed(index, need_lock) < 0) + return -1; + + return index_find_in_entries( + out, &index->entries, index->entries_search, path, path_len, stage); +} + void git_index__set_ignore_case(git_index *index, bool ignore_case) { index->ignore_case = ignore_case; @@ -687,16 +709,6 @@ size_t git_index_entrycount(const git_index *index) return index->entries.length; } -GIT_INLINE(int) git_index__find_internal( - size_t *out, git_index *index, const char *path, size_t path_len, int stage, - bool need_lock) -{ - if (index_sort_if_needed(index, need_lock) < 0) - return -1; - return git_index__find_in_entries( - out, &index->entries, index->entries_search, path, path_len, stage); -} - const git_index_entry *git_index_get_byindex( git_index *index, size_t n) { @@ -713,7 +725,7 @@ const git_index_entry *git_index_get_bypath( assert(index); - if (git_index__find_internal(&pos, index, path, 0, stage, true) < 0) { + if (index_find(&pos, index, path, 0, stage, true) < 0) { giterr_set(GITERR_INDEX, "Index does not contain %s", path); return NULL; } @@ -896,7 +908,7 @@ static int has_dir_name(git_index *index, } len = slash - name; - if (!git_index__find_internal(&pos, index, name, len, stage, false)) { + if (!index_find(&pos, index, name, len, stage, false)) { retval = -1; if (!ok_to_replace) break; @@ -976,7 +988,7 @@ static int index_insert( git_vector_sort(&index->entries); /* look if an entry with this path already exists */ - if (!git_index__find_internal( + if (!index_find( &position, index, entry->path, 0, GIT_IDXENTRY_STAGE(entry), false)) { existing = index->entries.contents[position]; /* update filemode to existing values if stat is not trusted */ @@ -1098,7 +1110,7 @@ int git_index_remove(git_index *index, const char *path, int stage) return -1; } - if (git_index__find_internal(&position, index, path, 0, stage, false) < 0) { + if (index_find(&position, index, path, 0, stage, false) < 0) { giterr_set( GITERR_INDEX, "Index does not contain %s at stage %d", path, stage); error = GIT_ENOTFOUND; @@ -1124,8 +1136,7 @@ int git_index_remove_directory(git_index *index, const char *dir, int stage) if (!(error = git_buf_sets(&pfx, dir)) && !(error = git_path_to_dir(&pfx))) - git_index__find_internal( - &pos, index, pfx.ptr, pfx.size, GIT_INDEX_STAGE_ANY, false); + index_find(&pos, index, pfx.ptr, pfx.size, GIT_INDEX_STAGE_ANY, false); while (!error) { entry = git_vector_get(&index->entries, pos); @@ -1148,22 +1159,11 @@ int git_index_remove_directory(git_index *index, const char *dir, int stage) return error; } -int git_index__find_in_entries( - size_t *out, git_vector *entries, git_vector_cmp entry_srch, - const char *path, size_t path_len, int stage) -{ - struct entry_srch_key srch_key; - srch_key.path = path; - srch_key.pathlen = !path_len ? strlen(path) : path_len; - srch_key.stage = stage; - return git_vector_bsearch2(out, entries, entry_srch, &srch_key); -} - -int git_index__find( +int git_index__find_pos( size_t *out, git_index *index, const char *path, size_t path_len, int stage) { assert(index && path); - return git_index__find_internal(out, index, path, path_len, stage, true); + return index_find(out, index, path, path_len, stage, true); } int git_index_find(size_t *at_pos, git_index *index, const char *path) @@ -1177,7 +1177,8 @@ int git_index_find(size_t *at_pos, git_index *index, const char *path) return -1; } - if (git_vector_bsearch2(&pos, &index->entries, index->entries_search_path, path) < 0) { + if (git_vector_bsearch2( + &pos, &index->entries, index->entries_search_path, path) < 0) { git_mutex_unlock(&index->lock); giterr_set(GITERR_INDEX, "Index does not contain %s", path); return GIT_ENOTFOUND; @@ -1186,13 +1187,11 @@ int git_index_find(size_t *at_pos, git_index *index, const char *path) /* Since our binary search only looked at path, we may be in the * middle of a list of stages. */ - while (pos > 0) { - const git_index_entry *prev = git_vector_get(&index->entries, pos-1); + for (; pos > 0; --pos) { + const git_index_entry *prev = git_vector_get(&index->entries, pos - 1); if (index->entries_cmp_path(prev->path, path) != 0) break; - - --pos; } if (at_pos) @@ -2269,7 +2268,7 @@ static int read_tree_cb( /* look for corresponding old entry and copy data to new entry */ if (data->old_entries != NULL && - !git_index__find_in_entries( + !index_find_in_entries( &pos, data->old_entries, data->entry_cmp, path.ptr, 0, 0) && (old_entry = git_vector_get(data->old_entries, pos)) != NULL && entry->mode == old_entry->mode && @@ -2394,7 +2393,7 @@ int git_index_add_all( /* skip ignored items that are not already in the index */ if ((flags & GIT_INDEX_ADD_FORCE) == 0 && git_iterator_current_is_ignored(wditer) && - git_index__find(&existing, index, wd->path, 0, 0) < 0) + index_find(&existing, index, wd->path, 0, 0, true) < 0) continue; /* issue notification callback if requested */ @@ -2556,7 +2555,7 @@ int git_index_update_all( return error; } -int git_index__snapshot(git_vector *entries, git_index *index) +int git_index_snapshot_new(git_vector *snap, git_index *index) { int error; @@ -2570,7 +2569,7 @@ int git_index__snapshot(git_vector *entries, git_index *index) git_atomic_inc(&index->readers); git_vector_sort(&index->entries); - error = git_vector_dup(entries, &index->entries, index->entries._cmp); + error = git_vector_dup(snap, &index->entries, index->entries._cmp); git_mutex_unlock(&index->lock); @@ -2580,8 +2579,10 @@ int git_index__snapshot(git_vector *entries, git_index *index) return error; } -void git_index__release_snapshot(git_index *index) +void git_index_snapshot_release(git_vector *snap, git_index *index) { + git_vector_free(snap); + git_atomic_dec(&index->readers); if (!git_mutex_lock(&index->lock)) { @@ -2591,3 +2592,10 @@ void git_index__release_snapshot(git_index *index) git_index_free(index); } + +int git_index_snapshot_find( + size_t *out, git_vector *entries, git_vector_cmp entry_srch, + const char *path, size_t path_len, int stage) +{ + return index_find_in_entries(out, entries, entry_srch, path, path_len, stage); +} diff --git a/src/index.h b/src/index.h index cb4425885..50a0b4b6c 100644 --- a/src/index.h +++ b/src/index.h @@ -61,11 +61,13 @@ extern int git_index_entry_icmp(const void *a, const void *b); extern int git_index_entry_srch(const void *a, const void *b); extern int git_index_entry_isrch(const void *a, const void *b); -/* Search index for `path`, returning GIT_ENOTFOUND if it does not exist. +/* Search index for `path`, returning GIT_ENOTFOUND if it does not exist + * (but not setting an error message). + * * `at_pos` is set to the position where it is or would be inserted. * Pass `path_len` as strlen of path or 0 to call strlen internally. */ -extern int git_index__find( +extern int git_index__find_pos( size_t *at_pos, git_index *index, const char *path, size_t path_len, int stage); extern void git_index__set_ignore_case(git_index *index, bool ignore_case); @@ -82,13 +84,13 @@ extern int git_index__changed_relative_to(git_index *index, const git_futils_fil /* Copy the current entries vector *and* increment the index refcount. * Call `git_index__release_snapshot` when done. */ -extern int git_index__snapshot(git_vector *entries, git_index *index); -extern void git_index__release_snapshot(git_index *index); +extern int git_index_snapshot_new(git_vector *snap, git_index *index); +extern void git_index_snapshot_release(git_vector *snap, git_index *index); /* Allow searching in a snapshot; entries must already be sorted! */ -extern int git_index__find_in_entries( - size_t *at_pos, - git_vector *entries, git_vector_cmp entry_srch, +extern int git_index_snapshot_find( + size_t *at_pos, git_vector *snap, git_vector_cmp entry_srch, const char *path, size_t path_len, int stage); + #endif diff --git a/src/iterator.c b/src/iterator.c index 53ef278d1..63c14f962 100644 --- a/src/iterator.c +++ b/src/iterator.c @@ -820,7 +820,7 @@ static int index_iterator__reset( ii->current = 0; if (ii->base.start) - git_index__find_in_entries( + git_index_snapshot_find( &ii->current, &ii->entries, ii->entry_srch, ii->base.start, 0, 0); if ((ie = index_iterator__skip_conflicts(ii)) == NULL) @@ -846,9 +846,8 @@ static int index_iterator__reset( static void index_iterator__free(git_iterator *self) { index_iterator *ii = (index_iterator *)self; - git_index__release_snapshot(ii->index); + git_index_snapshot_release(&ii->entries, ii->index); ii->index = NULL; - git_vector_free(&ii->entries); git_buf_free(&ii->partial); } @@ -863,7 +862,7 @@ int git_iterator_for_index( index_iterator *ii = git__calloc(1, sizeof(index_iterator)); GITERR_CHECK_ALLOC(ii); - if ((error = git_index__snapshot(&ii->entries, index)) < 0) { + if ((error = git_index_snapshot_new(&ii->entries, index)) < 0) { git__free(ii); return error; } diff --git a/src/pathspec.c b/src/pathspec.c index 471488495..09650de7c 100644 --- a/src/pathspec.c +++ b/src/pathspec.c @@ -445,7 +445,7 @@ static int pathspec_match_from_iterator( /* check if path is ignored and untracked */ if (index != NULL && git_iterator_current_is_ignored(iter) && - git_index__find(NULL, index, entry->path, 0, GIT_INDEX_STAGE_ANY) < 0) + git_index__find_pos(NULL, index, entry->path, 0, GIT_INDEX_STAGE_ANY) < 0) continue; /* mark the matched pattern as used */ From 3816debc13e8d5b96ad40be95a09fe67f6fa5a96 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Fri, 14 Mar 2014 14:51:04 -0700 Subject: [PATCH 10/20] Fix threading tests when threads disabled --- src/global.c | 20 ++++++++++---------- tests/threads/diff.c | 3 ++- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/global.c b/src/global.c index c26b4b311..15baf1eb8 100644 --- a/src/global.c +++ b/src/global.c @@ -16,8 +16,9 @@ git_mutex git__mwindow_mutex; #define MAX_SHUTDOWN_CB 8 -git_global_shutdown_fn git__shutdown_callbacks[MAX_SHUTDOWN_CB]; -git_atomic git__n_shutdown_callbacks; +static git_global_shutdown_fn git__shutdown_callbacks[MAX_SHUTDOWN_CB]; +static git_atomic git__n_shutdown_callbacks; +static git_atomic git__n_inits; void git__on_shutdown(git_global_shutdown_fn callback) { @@ -74,7 +75,6 @@ static void git__shutdown(void) static DWORD _tls_index; static DWORD _mutex = 0; -static DWORD _n_inits = 0; static int synchronized_threads_init() { @@ -101,7 +101,7 @@ int git_threads_init(void) while (InterlockedCompareExchange(&_mutex, 1, 0)) { Sleep(0); } /* Only do work on a 0 -> 1 transition of the refcount */ - if (1 == ++_n_inits) + if (1 == git_atomic_inc(&git__n_inits)) error = synchronized_threads_init(); /* Exit the lock */ @@ -124,7 +124,7 @@ void git_threads_shutdown(void) while (InterlockedCompareExchange(&_mutex, 1, 0)) { Sleep(0); } /* Only do work on a 1 -> 0 transition of the refcount */ - if (0 == --_n_inits) + if (0 == git_atomic_dec(&git__n_inits)) synchronized_threads_shutdown(); /* Exit the lock */ @@ -135,7 +135,7 @@ git_global_st *git__global_state(void) { void *ptr; - assert(_n_inits); + assert(git_atomic_get(&git__n_inits) > 0); if ((ptr = TlsGetValue(_tls_index)) != NULL) return ptr; @@ -153,7 +153,6 @@ git_global_st *git__global_state(void) static pthread_key_t _tls_key; static pthread_once_t _once_init = PTHREAD_ONCE_INIT; -static git_atomic git__n_inits; int init_error = 0; static void cb__free_status(void *st) @@ -204,7 +203,7 @@ git_global_st *git__global_state(void) { void *ptr; - assert(git__n_inits.val); + assert(git_atomic_get(&git__n_inits) > 0); if ((ptr = pthread_getspecific(_tls_key)) != NULL) return ptr; @@ -224,14 +223,15 @@ static git_global_st __state; int git_threads_init(void) { - /* noop */ + git_atomic_inc(&git__n_inits); return 0; } void git_threads_shutdown(void) { /* Shut down any subsystems that have global state */ - git__shutdown(); + if (0 == git_atomic_dec(&git__n_inits)) + git__shutdown(); } git_global_st *git__global_state(void) diff --git a/tests/threads/diff.c b/tests/threads/diff.c index 7f10a699d..5565c4bf1 100644 --- a/tests/threads/diff.c +++ b/tests/threads/diff.c @@ -18,11 +18,12 @@ static void run_in_parallel( int r, t, *id = git__calloc(threads, sizeof(int)); #ifdef GIT_THREADS git_thread *th = git__calloc(threads, sizeof(git_thread)); + cl_assert(th != NULL); #else void *th = NULL; #endif - cl_assert(id != NULL && th != NULL); + cl_assert(id != NULL); for (r = 0; r < repeats; ++r) { _repo = cl_git_sandbox_reopen(); /* reopen sandbox to flush caches */ From b87776151ad40caaec8bfe1becec4db429c9ea96 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Fri, 14 Mar 2014 15:37:42 -0700 Subject: [PATCH 11/20] Fix refcount issues with mutex protected ignores Some ignore files were not being freed from the cache. --- src/ignore.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/ignore.c b/src/ignore.c index 5cf4fca5c..9b3c6a8bc 100644 --- a/src/ignore.c +++ b/src/ignore.c @@ -204,9 +204,23 @@ int git_ignore__pop_dir(git_ignores *ign) void git_ignore__free(git_ignores *ignores) { - /* don't need to free ignores->ign_internal since it is in cache */ + unsigned int i; + git_attr_file *file; + + /* don't need to free ignores->ign_internal it is cached exactly once */ + + git_vector_foreach(&ignores->ign_path, i, file) { + git_attr_file__free(file); + ignores->ign_path.contents[i] = NULL; + } git_vector_free(&ignores->ign_path); + + git_vector_foreach(&ignores->ign_global, i, file) { + git_attr_file__free(file); + ignores->ign_global.contents[i] = NULL; + } git_vector_free(&ignores->ign_global); + git_buf_free(&ignores->dir); } From cef170abf2704a3e3aa109020f7041e43b5e4f71 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Fri, 14 Mar 2014 16:45:46 -0700 Subject: [PATCH 12/20] Fix leak when using push and pop with ignores The iterator pushes and pops ignores incrementally onto a list as it traverses the directory structure so that it doesn't have to constantly recheck which ignore files apply. With the new ref counting, it wasn't decrementing the refcount on the ignores that it removed from the vector. --- src/ignore.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ignore.c b/src/ignore.c index 9b3c6a8bc..0fb042a34 100644 --- a/src/ignore.c +++ b/src/ignore.c @@ -191,7 +191,10 @@ int git_ignore__pop_dir(git_ignores *ign) if (ign->dir.size >= keylen && !memcmp(ign->dir.ptr + ign->dir.size - keylen, start, keylen)) + { + git_attr_file__free(git_vector_last(&ign->ign_path)); git_vector_pop(&ign->ign_path); + } } if (--ign->depth > 0) { From aba6b5edbd1d1f999086669eb8e2f553af0ac300 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Fri, 14 Mar 2014 21:59:26 -0700 Subject: [PATCH 13/20] Fix leak in git_index_conflict_cleanup I introduced a leak into conflict cleanup by removing items from inside the git_vector_remove_matching call. This simplifies the code to just use one common way for the two conflict cleanup APIs. When an index has an active snapshot, removing an item can cause an error (inserting into the deferred deletion vector), so I made the git_index_conflict_cleanup API return an error code. I felt like this wasn't so bad since it is just like the other APIs. I fixed up a couple of comments while I was changing the header. --- include/git2/index.h | 19 ++++++++++++------- src/index.c | 32 ++++++++++---------------------- 2 files changed, 22 insertions(+), 29 deletions(-) diff --git a/include/git2/index.h b/include/git2/index.h index 4d33f13d2..05e58a632 100644 --- a/include/git2/index.h +++ b/include/git2/index.h @@ -575,8 +575,7 @@ GIT_EXTERN(int) git_index_update_all( * @param at_pos the address to which the position of the index entry is written (optional) * @param index an existing index object * @param path path to search - * @return a zero-based position in the index if found; - * GIT_ENOTFOUND otherwise + * @return a zero-based position in the index if found; GIT_ENOTFOUND otherwise */ GIT_EXTERN(int) git_index_find(size_t *at_pos, git_index *index, const char *path); @@ -620,6 +619,7 @@ GIT_EXTERN(int) git_index_conflict_add( * @param their_out Pointer to store the their entry * @param index an existing index object * @param path path to search + * @return 0 or an error code */ GIT_EXTERN(int) git_index_conflict_get( const git_index_entry **ancestor_out, @@ -632,16 +632,18 @@ GIT_EXTERN(int) git_index_conflict_get( * Removes the index entries that represent a conflict of a single file. * * @param index an existing index object - * @param path to search + * @param path path to remove conflicts for + * @return 0 or an error code */ GIT_EXTERN(int) git_index_conflict_remove(git_index *index, const char *path); /** - * Remove all conflicts in the index (entries with a stage greater than 0.) + * Remove all conflicts in the index (entries with a stage greater than 0). * * @param index an existing index object + * @return 0 or an error code */ -GIT_EXTERN(void) git_index_conflict_cleanup(git_index *index); +GIT_EXTERN(int) git_index_conflict_cleanup(git_index *index); /** * Determine if the index contains entries representing file conflicts. @@ -651,9 +653,12 @@ GIT_EXTERN(void) git_index_conflict_cleanup(git_index *index); GIT_EXTERN(int) git_index_has_conflicts(const git_index *index); /** - * Create an iterator for the conflicts in the index. You may not modify the - * index while iterating, the results are undefined. + * Create an iterator for the conflicts in the index. * + * The index must not be modified while iterating; the results are undefined. + * + * @param iterator_out The newly created conflict iterator + * @param index The index to scan * @return 0 or an error code */ GIT_EXTERN(int) git_index_conflict_iterator_new( diff --git a/src/index.c b/src/index.c index 10fb1fea7..27a557cfb 100644 --- a/src/index.c +++ b/src/index.c @@ -1317,15 +1317,13 @@ int git_index_conflict_get( return 0; } -int git_index_conflict_remove(git_index *index, const char *path) +static int index_conflict_remove(git_index *index, const char *path) { - size_t pos; + size_t pos = 0; git_index_entry *conflict_entry; int error = 0; - assert(index && path); - - if (git_index_find(&pos, index, path) < 0) + if (path != NULL && git_index_find(&pos, index, path) < 0) return GIT_ENOTFOUND; if (git_mutex_lock(&index->lock) < 0) { @@ -1335,7 +1333,8 @@ int git_index_conflict_remove(git_index *index, const char *path) while ((conflict_entry = git_vector_get(&index->entries, pos)) != NULL) { - if (index->entries_cmp_path(conflict_entry->path, path) != 0) + if (path != NULL && + index->entries_cmp_path(conflict_entry->path, path) != 0) break; if (GIT_IDXENTRY_STAGE(conflict_entry) == 0) { @@ -1352,27 +1351,16 @@ int git_index_conflict_remove(git_index *index, const char *path) return error; } -static int index_conflicts_match(const git_vector *v, size_t idx, void *p) +int git_index_conflict_remove(git_index *index, const char *path) { - git_index *index = p; - git_index_entry *entry = git_vector_get(v, idx); - - if (GIT_IDXENTRY_STAGE(entry) > 0 && - !index_remove_entry(index, idx)) - return 1; - - return 0; + assert(index && path); + return index_conflict_remove(index, path); } -void git_index_conflict_cleanup(git_index *index) +int git_index_conflict_cleanup(git_index *index) { assert(index); - - if (git_mutex_lock(&index->lock) < 0) - return; - git_vector_remove_matching(&index->entries, index_conflicts_match, index); - index_free_deleted(index); - git_mutex_unlock(&index->lock); + return index_conflict_remove(index, NULL); } int git_index_has_conflicts(const git_index *index) From 1fa17b5c92cb92a2785fba403b87525169b205c0 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Fri, 14 Mar 2014 22:01:30 -0700 Subject: [PATCH 14/20] Minor tree cache speedups While I was looking at the conflict cleanup code, I looked over at the tree cache code, since we clear the tree cache for each entry that gets removed and there is some redundancy there. I made some small tweaks to avoid extra calls to strchr and strlen in a few circumstances. --- src/tree-cache.c | 27 ++++++++++----------------- src/tree-cache.h | 1 + 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/tree-cache.c b/src/tree-cache.c index 1d3997154..49afd6e49 100644 --- a/src/tree-cache.c +++ b/src/tree-cache.c @@ -7,23 +7,16 @@ #include "tree-cache.h" -static git_tree_cache *find_child(const git_tree_cache *tree, const char *path) +static git_tree_cache *find_child( + const git_tree_cache *tree, const char *path, const char *end) { - size_t i, dirlen; - const char *end; - - end = strchr(path, '/'); - if (end == NULL) { - end = strrchr(path, '\0'); - } - - dirlen = end - path; + size_t i, dirlen = end ? (size_t)(end - path) : strlen(path); for (i = 0; i < tree->children_count; ++i) { - const char *childname = tree->children[i]->name; + git_tree_cache *child = tree->children[i]; - if (strlen(childname) == dirlen && !memcmp(path, childname, dirlen)) - return tree->children[i]; + if (child->namelen == dirlen && !memcmp(path, child->name, dirlen)) + return child; } return NULL; @@ -44,7 +37,7 @@ void git_tree_cache_invalidate_path(git_tree_cache *tree, const char *path) if (end == NULL) /* End of path */ break; - tree = find_child(tree, ptr); + tree = find_child(tree, ptr, end); if (tree == NULL) /* We don't have that tree */ return; @@ -64,10 +57,9 @@ const git_tree_cache *git_tree_cache_get(const git_tree_cache *tree, const char while (1) { end = strchr(ptr, '/'); - tree = find_child(tree, ptr); - if (tree == NULL) { /* Can't find it */ + tree = find_child(tree, ptr, end); + if (tree == NULL) /* Can't find it */ return NULL; - } if (end == NULL || *end + 1 == '\0') return tree; @@ -100,6 +92,7 @@ static int read_tree_internal(git_tree_cache **out, tree->parent = parent; /* NUL-terminated tree name */ + tree->namelen = name_len; memcpy(tree->name, name_start, name_len); tree->name[name_len] = '\0'; diff --git a/src/tree-cache.h b/src/tree-cache.h index 805483a78..90c82dbbf 100644 --- a/src/tree-cache.h +++ b/src/tree-cache.h @@ -18,6 +18,7 @@ struct git_tree_cache { ssize_t entries; git_oid oid; + size_t namelen; char name[GIT_FLEX_ARRAY]; }; From 7d4908724fd7d4d8e096b4faf2c652ba5b77644e Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Thu, 10 Apr 2014 22:31:01 -0700 Subject: [PATCH 15/20] Attribute file cache refactor This is a big refactoring of the attribute file cache to be a bit simpler which in turn makes it easier to enforce a lock around any updates to the cache so that it can be used in a threaded env. Tons of changes to the attributes and ignores code. --- src/attr.c | 478 +++------------------------------ src/attr.h | 34 +-- src/attr_file.c | 342 +++++++++++++---------- src/attr_file.h | 59 ++-- src/attrcache.c | 397 +++++++++++++++++++++++++++ src/attrcache.h | 55 +++- src/fileops.c | 4 +- src/fileops.h | 9 +- src/ignore.c | 128 +++++---- src/index.c | 17 +- src/sortedcache.c | 5 +- src/submodule.c | 2 - tests/attr/file.c | 16 +- tests/attr/lookup.c | 16 +- tests/threads/diff.c | 49 +--- tests/threads/iterator.c | 49 ++++ tests/threads/thread_helpers.c | 44 +++ tests/threads/thread_helpers.h | 8 + 18 files changed, 948 insertions(+), 764 deletions(-) create mode 100644 src/attrcache.c create mode 100644 tests/threads/iterator.c create mode 100644 tests/threads/thread_helpers.c create mode 100644 tests/threads/thread_helpers.h diff --git a/src/attr.c b/src/attr.c index f52a8a97b..c53a728de 100644 --- a/src/attr.c +++ b/src/attr.c @@ -2,7 +2,7 @@ #include "repository.h" #include "sysdir.h" #include "config.h" -#include "attr.h" +#include "attr_file.h" #include "ignore.h" #include "git2/oid.h" #include @@ -216,7 +216,6 @@ cleanup: return error; } - int git_attr_add_macro( git_repository *repo, const char *name, @@ -251,261 +250,6 @@ int git_attr_add_macro( return error; } -bool git_attr_cache__is_cached( - git_repository *repo, git_attr_file_source source, const char *path) -{ - git_buf cache_key = GIT_BUF_INIT; - git_strmap *files = git_repository_attr_cache(repo)->files; - const char *workdir = git_repository_workdir(repo); - bool rval; - - if (workdir && git__prefixcmp(path, workdir) == 0) - path += strlen(workdir); - if (git_buf_printf(&cache_key, "%d#%s", (int)source, path) < 0) - return false; - - rval = git_strmap_exists(files, git_buf_cstr(&cache_key)); - - git_buf_free(&cache_key); - - return rval; -} - -static int load_attr_file( - const char **data, - git_futils_filestamp *stamp, - const char *filename) -{ - int error; - git_buf content = GIT_BUF_INIT; - - error = git_futils_filestamp_check(stamp, filename); - if (error < 0) - return error; - - /* if error == 0, then file is up to date. By returning GIT_ENOTFOUND, - * we tell the caller not to reparse this file... - */ - if (!error) - return GIT_ENOTFOUND; - - error = git_futils_readbuffer(&content, filename); - if (error < 0) { - /* convert error into ENOTFOUND so failed permissions / invalid - * file type don't actually stop the operation in progress. - */ - return GIT_ENOTFOUND; - - /* TODO: once warnings are available, issue a warning callback */ - } - - *data = git_buf_detach(&content); - - return 0; -} - -static int load_attr_blob_from_index( - const char **content, - git_blob **blob, - git_repository *repo, - const git_oid *old_oid, - const char *relfile) -{ - int error; - size_t pos; - git_index *index; - const git_index_entry *entry; - - if ((error = git_repository_index__weakptr(&index, repo)) < 0 || - (error = git_index_find(&pos, index, relfile)) < 0) - return error; - - entry = git_index_get_byindex(index, pos); - - if (old_oid && git_oid__cmp(old_oid, &entry->id) == 0) - return GIT_ENOTFOUND; - - if ((error = git_blob_lookup(blob, repo, &entry->id)) < 0) - return error; - - *content = git_blob_rawcontent(*blob); - return 0; -} - -static int load_attr_from_cache( - git_attr_file **file, - git_attr_cache *cache, - git_attr_file_source source, - const char *relative_path) -{ - git_buf cache_key = GIT_BUF_INIT; - khiter_t cache_pos; - - *file = NULL; - - if (!cache || !cache->files) - return 0; - - if (git_buf_printf(&cache_key, "%d#%s", (int)source, relative_path) < 0) - return -1; - - if (git_mutex_lock(&cache->lock) < 0) { - giterr_set(GITERR_OS, "Could not get cache attr lock"); - git_buf_free(&cache_key); - return -1; - } - - cache_pos = git_strmap_lookup_index(cache->files, cache_key.ptr); - - if (git_strmap_valid_index(cache->files, cache_pos)) { - *file = git_strmap_value_at(cache->files, cache_pos); - GIT_REFCOUNT_INC(*file); - } - - git_mutex_unlock(&cache->lock); - git_buf_free(&cache_key); - - return 0; -} - -int git_attr_cache__internal_file( - git_repository *repo, - const char *filename, - git_attr_file **file) -{ - int error = 0; - git_attr_cache *cache = git_repository_attr_cache(repo); - khiter_t cache_pos; - - if (git_mutex_lock(&cache->lock) < 0) { - giterr_set(GITERR_OS, "Unable to get attr cache lock"); - return -1; - } - - cache_pos = git_strmap_lookup_index(cache->files, filename); - - if (git_strmap_valid_index(cache->files, cache_pos)) { - *file = git_strmap_value_at(cache->files, cache_pos); - } - else if (!(error = git_attr_file__new(file, 0, filename, &cache->pool))) { - - git_strmap_insert(cache->files, (*file)->key + 2, *file, error); - if (error > 0) - error = 0; - } - - git_mutex_unlock(&cache->lock); - return error; -} - -int git_attr_cache__push_file( - git_repository *repo, - const char *base, - const char *filename, - git_attr_file_source source, - git_attr_file_parser parse, - void* parsedata, - git_vector *stack) -{ - int error = 0; - git_buf path = GIT_BUF_INIT; - const char *workdir = git_repository_workdir(repo); - const char *relfile, *content = NULL; - git_attr_cache *cache = git_repository_attr_cache(repo); - git_attr_file *file = NULL; - git_blob *blob = NULL; - git_futils_filestamp stamp; - - assert(filename && stack); - - /* join base and path as needed */ - if (base != NULL && git_path_root(filename) < 0) { - if (git_buf_joinpath(&path, base, filename) < 0) - return -1; - filename = path.ptr; - } - - relfile = filename; - if (workdir && git__prefixcmp(relfile, workdir) == 0) - relfile += strlen(workdir); - - /* check cache */ - if (load_attr_from_cache(&file, cache, source, relfile) < 0) - return -1; - - /* if not in cache, load data, parse, and cache */ - - if (source == GIT_ATTR_FILE_FROM_FILE) { - git_futils_filestamp_set( - &stamp, file ? &file->cache_data.stamp : NULL); - - error = load_attr_file(&content, &stamp, filename); - } else { - error = load_attr_blob_from_index(&content, &blob, - repo, file ? &file->cache_data.oid : NULL, relfile); - } - - if (error) { - /* not finding a file is not an error for this function */ - if (error == GIT_ENOTFOUND) { - giterr_clear(); - error = 0; - } - goto finish; - } - - /* if we got here, we have to parse and/or reparse the file */ - if (file) - git_attr_file__clear_rules(file); - else { - error = git_attr_file__new(&file, source, relfile, &cache->pool); - if (error < 0) - goto finish; - } - - if (parse && (error = parse(repo, parsedata, content, file)) < 0) - goto finish; - - if (git_mutex_lock(&cache->lock) < 0) { - giterr_set(GITERR_OS, "Unable to get attr cache lock"); - error = -1; - } else { - git_strmap_insert(cache->files, file->key, file, error); /* -V595 */ - if (error > 0) { /* > 0 means inserting for the first time */ - error = 0; - GIT_REFCOUNT_INC(file); - } - git_mutex_unlock(&cache->lock); - } - - /* remember "cache buster" file signature */ - if (blob) - git_oid_cpy(&file->cache_data.oid, git_object_id((git_object *)blob)); - else - git_futils_filestamp_set(&file->cache_data.stamp, &stamp); - -finish: - /* push file onto vector if we found one*/ - if (!error && file != NULL) - error = git_vector_insert(stack, file); - - if (error != 0) - git_attr_file__free(file); - - if (blob) - git_blob_free(blob); - else - git__free((void *)content); - - git_buf_free(&path); - - return error; -} - -#define push_attr_file(R,S,B,F) \ - git_attr_cache__push_file \ - ((R),(B),(F),GIT_ATTR_FILE_FROM_FILE,git_attr_file__parse_buffer,NULL,(S)) - typedef struct { git_repository *repo; uint32_t flags; @@ -514,46 +258,64 @@ typedef struct { git_vector *files; } attr_walk_up_info; -int git_attr_cache__decide_sources( - uint32_t flags, bool has_wd, bool has_index, git_attr_file_source *srcs) +static int attr_decide_sources( + uint32_t flags, bool has_wd, bool has_index, git_attr_cache_source *srcs) { int count = 0; switch (flags & 0x03) { case GIT_ATTR_CHECK_FILE_THEN_INDEX: if (has_wd) - srcs[count++] = GIT_ATTR_FILE_FROM_FILE; + srcs[count++] = GIT_ATTR_CACHE__FROM_FILE; if (has_index) - srcs[count++] = GIT_ATTR_FILE_FROM_INDEX; + srcs[count++] = GIT_ATTR_CACHE__FROM_INDEX; break; case GIT_ATTR_CHECK_INDEX_THEN_FILE: if (has_index) - srcs[count++] = GIT_ATTR_FILE_FROM_INDEX; + srcs[count++] = GIT_ATTR_CACHE__FROM_INDEX; if (has_wd) - srcs[count++] = GIT_ATTR_FILE_FROM_FILE; + srcs[count++] = GIT_ATTR_CACHE__FROM_FILE; break; case GIT_ATTR_CHECK_INDEX_ONLY: if (has_index) - srcs[count++] = GIT_ATTR_FILE_FROM_INDEX; + srcs[count++] = GIT_ATTR_CACHE__FROM_INDEX; break; } return count; } +static int push_attr_file( + git_repository *repo, + git_vector *list, + git_attr_cache_source source, + const char *base, + const char *filename) +{ + int error = 0; + git_attr_file *file = NULL; + + if ((error = git_attr_cache__get( + &file, repo, source, base, filename, + git_attr_file__parse_buffer, NULL)) < 0 || + (error = git_vector_insert(list, file)) < 0) + git_attr_file__free(file); + + return error; +} + static int push_one_attr(void *ref, git_buf *path) { int error = 0, n_src, i; attr_walk_up_info *info = (attr_walk_up_info *)ref; - git_attr_file_source src[2]; + git_attr_cache_source src[2]; - n_src = git_attr_cache__decide_sources( + n_src = attr_decide_sources( info->flags, info->workdir != NULL, info->index != NULL, src); for (i = 0; !error && i < n_src; ++i) - error = git_attr_cache__push_file( - info->repo, path->ptr, GIT_ATTR_FILE, src[i], - git_attr_file__parse_buffer, NULL, info->files); + error = push_attr_file( + info->repo, info->files, src[i], path->ptr, GIT_ATTR_FILE); return error; } @@ -601,7 +363,8 @@ static int collect_attr_files( */ error = push_attr_file( - repo, files, git_repository_path(repo), GIT_ATTR_FILE_INREPO); + repo, files, GIT_ATTR_CACHE__FROM_FILE, + git_repository_path(repo), GIT_ATTR_FILE_INREPO); if (error < 0) goto cleanup; @@ -618,7 +381,8 @@ static int collect_attr_files( if (git_repository_attr_cache(repo)->cfg_attr_file != NULL) { error = push_attr_file( - repo, files, NULL, git_repository_attr_cache(repo)->cfg_attr_file); + repo, files, GIT_ATTR_CACHE__FROM_FILE, + NULL, git_repository_attr_cache(repo)->cfg_attr_file); if (error < 0) goto cleanup; } @@ -626,7 +390,8 @@ static int collect_attr_files( if ((flags & GIT_ATTR_CHECK_NO_SYSTEM) == 0) { error = git_sysdir_find_system_file(&dir, GIT_ATTR_FILE_SYSTEM); if (!error) - error = push_attr_file(repo, files, NULL, dir.ptr); + error = push_attr_file( + repo, files, GIT_ATTR_CACHE__FROM_FILE, NULL, dir.ptr); else if (error == GIT_ENOTFOUND) { giterr_clear(); error = 0; @@ -640,172 +405,3 @@ static int collect_attr_files( return error; } - -static int attr_cache__lookup_path( - char **out, git_config *cfg, const char *key, const char *fallback) -{ - git_buf buf = GIT_BUF_INIT; - int error; - const git_config_entry *entry = NULL; - - *out = NULL; - - if ((error = git_config__lookup_entry(&entry, cfg, key, false)) < 0) - return error; - - if (entry) { - const char *cfgval = entry->value; - - /* expand leading ~/ as needed */ - if (cfgval && cfgval[0] == '~' && cfgval[1] == '/' && - !git_sysdir_find_global_file(&buf, &cfgval[2])) - *out = git_buf_detach(&buf); - else if (cfgval) - *out = git__strdup(cfgval); - - } - else if (!git_sysdir_find_xdg_file(&buf, fallback)) - *out = git_buf_detach(&buf); - - git_buf_free(&buf); - - return error; -} - -static void attr_cache__free(git_attr_cache *cache) -{ - if (!cache) - return; - - if (cache->files != NULL) { - git_attr_file *file; - - git_strmap_foreach_value(cache->files, file, { - git_attr_file__free(file); - }); - - git_strmap_free(cache->files); - } - - if (cache->macros != NULL) { - git_attr_rule *rule; - - git_strmap_foreach_value(cache->macros, rule, { - git_attr_rule__free(rule); - }); - - git_strmap_free(cache->macros); - } - - git_pool_clear(&cache->pool); - - git__free(cache->cfg_attr_file); - cache->cfg_attr_file = NULL; - - git__free(cache->cfg_excl_file); - cache->cfg_excl_file = NULL; - - git_mutex_free(&cache->lock); - - git__free(cache); -} - -int git_attr_cache__init(git_repository *repo) -{ - int ret = 0; - git_attr_cache *cache = git_repository_attr_cache(repo); - git_config *cfg; - - if (cache) - return 0; - - if ((ret = git_repository_config__weakptr(&cfg, repo)) < 0) - return ret; - - cache = git__calloc(1, sizeof(git_attr_cache)); - GITERR_CHECK_ALLOC(cache); - - /* set up lock */ - if (git_mutex_init(&cache->lock) < 0) { - giterr_set(GITERR_OS, "Unable to initialize lock for attr cache"); - git__free(cache); - return -1; - } - - /* cache config settings for attributes and ignores */ - ret = attr_cache__lookup_path( - &cache->cfg_attr_file, cfg, GIT_ATTR_CONFIG, GIT_ATTR_FILE_XDG); - if (ret < 0) - goto cancel; - - ret = attr_cache__lookup_path( - &cache->cfg_excl_file, cfg, GIT_IGNORE_CONFIG, GIT_IGNORE_FILE_XDG); - if (ret < 0) - goto cancel; - - /* allocate hashtable for attribute and ignore file contents, - * hashtable for attribute macros, and string pool - */ - if ((ret = git_strmap_alloc(&cache->files)) < 0 || - (ret = git_strmap_alloc(&cache->macros)) < 0 || - (ret = git_pool_init(&cache->pool, 1, 0)) < 0) - goto cancel; - - cache = git__compare_and_swap(&repo->attrcache, NULL, cache); - if (cache) - goto cancel; /* raced with another thread, free this but no error */ - - /* insert default macros */ - return git_attr_add_macro(repo, "binary", "-diff -crlf -text"); - -cancel: - attr_cache__free(cache); - return ret; -} - -void git_attr_cache_flush(git_repository *repo) -{ - git_attr_cache *cache; - - /* this could be done less expensively, but for now, we'll just free - * the entire attrcache and let the next use reinitialize it... - */ - if (repo && (cache = git__swap(repo->attrcache, NULL)) != NULL) - attr_cache__free(cache); -} - -int git_attr_cache__insert_macro(git_repository *repo, git_attr_rule *macro) -{ - git_attr_cache *cache = git_repository_attr_cache(repo); - git_strmap *macros = cache->macros; - int error; - - /* TODO: generate warning log if (macro->assigns.length == 0) */ - if (macro->assigns.length == 0) - return 0; - - if (git_mutex_lock(&cache->lock) < 0) { - giterr_set(GITERR_OS, "Unable to get attr cache lock"); - error = -1; - } else { - git_strmap_insert(macros, macro->match.pattern, macro, error); - git_mutex_unlock(&cache->lock); - } - - return (error < 0) ? -1 : 0; -} - -git_attr_rule *git_attr_cache__lookup_macro( - git_repository *repo, const char *name) -{ - git_strmap *macros = git_repository_attr_cache(repo)->macros; - khiter_t pos; - - pos = git_strmap_lookup_index(macros, name); - - if (!git_strmap_valid_index(macros, pos)) - return NULL; - - return (git_attr_rule *)git_strmap_value_at(macros, pos); -} - diff --git a/src/attr.h b/src/attr.h index 19c979bcd..f9f216d07 100644 --- a/src/attr.h +++ b/src/attr.h @@ -8,38 +8,6 @@ #define INCLUDE_attr_h__ #include "attr_file.h" - -#define GIT_ATTR_CONFIG "core.attributesfile" -#define GIT_IGNORE_CONFIG "core.excludesfile" - -typedef int (*git_attr_file_parser)( - git_repository *, void *, const char *, git_attr_file *); - -extern int git_attr_cache__insert_macro( - git_repository *repo, git_attr_rule *macro); - -extern git_attr_rule *git_attr_cache__lookup_macro( - git_repository *repo, const char *name); - -extern int git_attr_cache__push_file( - git_repository *repo, - const char *base, - const char *filename, - git_attr_file_source source, - git_attr_file_parser parse, - void *parsedata, /* passed through to parse function */ - git_vector *stack); - -extern int git_attr_cache__internal_file( - git_repository *repo, - const char *key, - git_attr_file **file_ptr); - -/* returns true if path is in cache */ -extern bool git_attr_cache__is_cached( - git_repository *repo, git_attr_file_source source, const char *path); - -extern int git_attr_cache__decide_sources( - uint32_t flags, bool has_wd, bool has_index, git_attr_file_source *srcs); +#include "attrcache.h" #endif diff --git a/src/attr_file.c b/src/attr_file.c index 695f661a8..86b3448ee 100644 --- a/src/attr_file.c +++ b/src/attr_file.c @@ -1,11 +1,144 @@ #include "common.h" #include "repository.h" #include "filebuf.h" -#include "attr.h" +#include "attr_file.h" #include "git2/blob.h" #include "git2/tree.h" +#include "index.h" #include +static void attr_file_free(git_attr_file *file) +{ + git_attr_file__clear_rules(file); + git_pool_clear(&file->pool); + git__memzero(file, sizeof(*file)); + git__free(file); +} + +int git_attr_file__new( + git_attr_file **out, + git_attr_cache_entry *ce, + git_attr_cache_source source) +{ + git_attr_file *attrs = git__calloc(1, sizeof(git_attr_file)); + GITERR_CHECK_ALLOC(attrs); + + if (git_pool_init(&attrs->pool, 1, 0) < 0 || + git_vector_init(&attrs->rules, 0, NULL) < 0) + { + attr_file_free(attrs); + return -1; + } + + GIT_REFCOUNT_INC(attrs); + attrs->ce = ce; + attrs->source = source; + *out = attrs; + return 0; +} + +void git_attr_file__clear_rules(git_attr_file *file) +{ + unsigned int i; + git_attr_rule *rule; + + git_vector_foreach(&file->rules, i, rule) + git_attr_rule__free(rule); + git_vector_free(&file->rules); +} + +void git_attr_file__free(git_attr_file *file) +{ + if (!file) + return; + GIT_REFCOUNT_DEC(file, attr_file_free); +} + +static int attr_file_oid_from_index( + git_oid *oid, git_repository *repo, const char *path) +{ + int error; + git_index *idx; + size_t pos; + const git_index_entry *entry; + + if ((error = git_repository_index__weakptr(&idx, repo)) < 0 || + (error = git_index__find_pos(&pos, idx, path, 0, 0)) < 0) + return error; + + if (!(entry = git_index_get_byindex(idx, pos))) + return GIT_ENOTFOUND; + + *oid = entry->id; + return 0; +} + +int git_attr_file__load( + git_attr_file **out, + git_repository *repo, + git_attr_cache_entry *ce, + git_attr_cache_source source, + git_attr_cache_parser parser, + void *payload) +{ + int error = 0; + git_blob *blob = NULL; + git_buf content = GIT_BUF_INIT; + const char *data = NULL; + git_attr_file *file; + + *out = NULL; + + if (source == GIT_ATTR_CACHE__FROM_INDEX) { + git_oid id; + + if ((error = attr_file_oid_from_index(&id, repo, ce->path)) < 0 || + (error = git_blob_lookup(&blob, repo, &id)) < 0) + return error; + + data = git_blob_rawcontent(blob); + } else { + if ((error = git_futils_readbuffer(&content, ce->fullpath)) < 0) + /* always return ENOTFOUND so item will just be skipped */ + /* TODO: issue a warning once warnings API is available */ + return GIT_ENOTFOUND; + data = content.ptr; + } + + if ((error = git_attr_file__new(&file, ce, source)) < 0) + goto cleanup; + + if (parser && (error = parser(repo, file, data, payload)) < 0) + git_attr_file__free(file); + else + *out = file; + +cleanup: + git_blob_free(blob); + git_buf_free(&content); + + return error; +} + +int git_attr_file__out_of_date(git_repository *repo, git_attr_file *file) +{ + if (!file) + return 1; + + if (file->source == GIT_ATTR_CACHE__FROM_INDEX) { + int error; + git_oid id; + + if ((error = attr_file_oid_from_index(&id, repo, file->ce->path)) < 0) + return error; + + return (git_oid__cmp(&file->cache_data.oid, &id) != 0); + } + + return git_futils_filestamp_check( + &file->cache_data.stamp, file->ce->fullpath); +} + static int sort_by_hash_and_name(const void *a_raw, const void *b_raw); static void git_attr_rule__clear(git_attr_rule *rule); static bool parse_optimized_patterns( @@ -13,74 +146,28 @@ static bool parse_optimized_patterns( git_pool *pool, const char *pattern); -int git_attr_file__new( - git_attr_file **attrs_ptr, - git_attr_file_source from, - const char *path, - git_pool *pool) -{ - git_attr_file *attrs = NULL; - - attrs = git__calloc(1, sizeof(git_attr_file)); - GITERR_CHECK_ALLOC(attrs); - GIT_REFCOUNT_INC(attrs); - - if (pool) - attrs->pool = pool; - else { - attrs->pool = git__calloc(1, sizeof(git_pool)); - if (!attrs->pool || git_pool_init(attrs->pool, 1, 0) < 0) - goto fail; - attrs->pool_is_allocated = true; - } - - if (path) { - size_t len = strlen(path); - - attrs->key = git_pool_malloc(attrs->pool, (uint32_t)len + 3); - GITERR_CHECK_ALLOC(attrs->key); - - attrs->key[0] = '0' + (char)from; - attrs->key[1] = '#'; - memcpy(&attrs->key[2], path, len); - attrs->key[len + 2] = '\0'; - } - - if (git_vector_init(&attrs->rules, 4, NULL) < 0) - goto fail; - - *attrs_ptr = attrs; - return 0; - -fail: - git_attr_file__free(attrs); - attrs_ptr = NULL; - return -1; -} - int git_attr_file__parse_buffer( - git_repository *repo, void *parsedata, const char *buffer, git_attr_file *attrs) + git_repository *repo, + git_attr_file *attrs, + const char *data, + void *payload) { int error = 0; - const char *scan = NULL, *context = NULL; + const char *scan = data, *context = NULL; git_attr_rule *rule = NULL; - GIT_UNUSED(parsedata); - - assert(buffer && attrs); - - scan = buffer; + GIT_UNUSED(payload); /* if subdir file path, convert context for file paths */ - if (attrs->key && - git_path_root(attrs->key + 2) < 0 && - git__suffixcmp(attrs->key, "/" GIT_ATTR_FILE) == 0) - context = attrs->key + 2; + if (attrs->ce && + git_path_root(attrs->ce->path) < 0 && + !git__suffixcmp(attrs->ce->path, "/" GIT_ATTR_FILE)) + context = attrs->ce->path; while (!error && *scan) { /* allocate rule if needed */ if (!rule) { - if (!(rule = git__calloc(1, sizeof(git_attr_rule)))) { + if (!(rule = git__calloc(1, sizeof(*rule)))) { error = -1; break; } @@ -90,9 +177,9 @@ int git_attr_file__parse_buffer( /* parse the next "pattern attr attr attr" line */ if (!(error = git_attr_fnmatch__parse( - &rule->match, attrs->pool, context, &scan)) && + &rule->match, &attrs->pool, context, &scan)) && !(error = git_attr_assignment__parse( - repo, attrs->pool, &rule->assigns, &scan))) + repo, &attrs->pool, &rule->assigns, &scan))) { if (rule->match.flags & GIT_ATTR_FNMATCH_MACRO) /* should generate error/warning if this is coming from any @@ -118,61 +205,6 @@ int git_attr_file__parse_buffer( return error; } -int git_attr_file__new_and_load( - git_attr_file **attrs_ptr, - const char *path) -{ - int error; - git_buf content = GIT_BUF_INIT; - - if ((error = git_attr_file__new(attrs_ptr, 0, path, NULL)) < 0) - return error; - - if (!(error = git_futils_readbuffer(&content, path))) - error = git_attr_file__parse_buffer( - NULL, NULL, git_buf_cstr(&content), *attrs_ptr); - - git_buf_free(&content); - - if (error) { - git_attr_file__free(*attrs_ptr); - *attrs_ptr = NULL; - } - - return error; -} - -void git_attr_file__clear_rules(git_attr_file *file) -{ - unsigned int i; - git_attr_rule *rule; - - git_vector_foreach(&file->rules, i, rule) - git_attr_rule__free(rule); - - git_vector_free(&file->rules); -} - -static void attr_file_free(git_attr_file *file) -{ - git_attr_file__clear_rules(file); - - if (file->pool_is_allocated) { - git_pool_clear(file->pool); - git__free(file->pool); - } - file->pool = NULL; - - git__free(file); -} - -void git_attr_file__free(git_attr_file *file) -{ - if (!file) - return; - GIT_REFCOUNT_DEC(file, attr_file_free); -} - uint32_t git_attr_file__name_hash(const char *name) { uint32_t h = 5381; @@ -183,7 +215,6 @@ uint32_t git_attr_file__name_hash(const char *name) return h; } - int git_attr_file__lookup_one( git_attr_file *file, const git_attr_path *path, @@ -212,25 +243,64 @@ int git_attr_file__lookup_one( return 0; } +int git_attr_file__load_standalone( + git_attr_file **out, + const char *path) +{ + int error; + git_attr_file *file; + git_buf content = GIT_BUF_INIT; + + error = git_attr_file__new(&file, NULL, GIT_ATTR_CACHE__FROM_FILE); + if (error < 0) + return error; + + error = git_attr_cache_entry__new(&file->ce, NULL, path, &file->pool); + if (error < 0) { + git_attr_file__free(file); + return error; + } + /* because the cache entry is allocated from the file's own pool, we + * don't have to free it - freeing file+pool will free cache entry, too. + */ + + if (!(error = git_futils_readbuffer(&content, path))) { + error = git_attr_file__parse_buffer(NULL, file, content.ptr, NULL); + git_buf_free(&content); + } + + if (error < 0) + git_attr_file__free(file); + else + *out = file; + + return error; +} bool git_attr_fnmatch__match( git_attr_fnmatch *match, const git_attr_path *path) { - int fnm; - int icase_flags = (match->flags & GIT_ATTR_FNMATCH_ICASE) ? FNM_CASEFOLD : 0; + const char *filename; + int flags = 0; - if (match->flags & GIT_ATTR_FNMATCH_DIRECTORY && !path->is_dir) + if ((match->flags & GIT_ATTR_FNMATCH_DIRECTORY) && !path->is_dir) return false; - if (match->flags & GIT_ATTR_FNMATCH_FULLPATH) - fnm = p_fnmatch(match->pattern, path->path, FNM_PATHNAME | icase_flags); - else if (path->is_dir) - fnm = p_fnmatch(match->pattern, path->basename, FNM_LEADING_DIR | icase_flags); - else - fnm = p_fnmatch(match->pattern, path->basename, icase_flags); + if (match->flags & GIT_ATTR_FNMATCH_ICASE) + flags |= FNM_CASEFOLD; - return (fnm == FNM_NOMATCH) ? false : true; + if (match->flags & GIT_ATTR_FNMATCH_FULLPATH) { + filename = path->path; + flags |= FNM_PATHNAME; + } else { + filename = path->basename; + + if (path->is_dir) + flags |= FNM_LEADING_DIR; + } + + return (p_fnmatch(match->pattern, filename, flags) != FNM_NOMATCH); } bool git_attr_rule__match( @@ -245,7 +315,6 @@ bool git_attr_rule__match( return matched; } - git_attr_assignment *git_attr_rule__lookup_assignment( git_attr_rule *rule, const char *name) { @@ -344,7 +413,7 @@ void git_attr_path__free(git_attr_path *info) int git_attr_fnmatch__parse( git_attr_fnmatch *spec, git_pool *pool, - const char *source, + const char *context, const char **base) { const char *pattern, *scan; @@ -412,21 +481,21 @@ int git_attr_fnmatch__parse( } if ((spec->flags & GIT_ATTR_FNMATCH_FULLPATH) != 0 && - source != NULL && git_path_root(pattern) < 0) + context != NULL && git_path_root(pattern) < 0) { - /* use context path minus the trailing filename */ - char *slash = strrchr(source, '/'); - size_t sourcelen = slash ? slash - source + 1 : 0; + /* use context path minus the trailing filename */ + char *slash = strrchr(context, '/'); + size_t contextlen = slash ? slash - context + 1 : 0; /* given an unrooted fullpath match from a file inside a repo, * prefix the pattern with the relative directory of the source file */ spec->pattern = git_pool_malloc( - pool, (uint32_t)(sourcelen + spec->length + 1)); + pool, (uint32_t)(contextlen + spec->length + 1)); if (spec->pattern) { - memcpy(spec->pattern, source, sourcelen); - memcpy(spec->pattern + sourcelen, pattern, spec->length); - spec->length += sourcelen; + memcpy(spec->pattern, context, contextlen); + memcpy(spec->pattern + contextlen, pattern, spec->length); + spec->length += contextlen; spec->pattern[spec->length] = '\0'; } } else { @@ -439,6 +508,7 @@ int git_attr_fnmatch__parse( } else { /* strip '\' that might have be used for internal whitespace */ spec->length = git__unescape(spec->pattern); + /* TODO: convert remaining '\' into '/' for POSIX ??? */ } return 0; diff --git a/src/attr_file.h b/src/attr_file.h index dbd6696c9..f92ce3c96 100644 --- a/src/attr_file.h +++ b/src/attr_file.h @@ -13,6 +13,7 @@ #include "pool.h" #include "buffer.h" #include "fileops.h" +#include "attrcache.h" #define GIT_ATTR_FILE ".gitattributes" #define GIT_ATTR_FILE_INREPO "info/attributes" @@ -45,10 +46,10 @@ typedef struct { unsigned int flags; } git_attr_fnmatch; -typedef struct { +struct git_attr_rule { git_attr_fnmatch match; git_vector assigns; /* vector of */ -} git_attr_rule; +}; typedef struct { git_refcount unused; @@ -63,17 +64,17 @@ typedef struct { const char *value; } git_attr_assignment; -typedef struct { +struct git_attr_file { git_refcount rc; - char *key; /* cache "source#path" this was loaded from */ - git_vector rules; /* vector of or */ - git_pool *pool; - bool pool_is_allocated; + git_attr_cache_entry *ce; + git_attr_cache_source source; + git_vector rules; /* vector of or */ + git_pool pool; union { git_oid oid; git_futils_filestamp stamp; } cache_data; -} git_attr_file; +}; typedef struct { git_buf full; @@ -82,29 +83,41 @@ typedef struct { int is_dir; } git_attr_path; -typedef enum { - GIT_ATTR_FILE_FROM_FILE = 0, - GIT_ATTR_FILE_FROM_INDEX = 1 -} git_attr_file_source; - /* * git_attr_file API */ -extern int git_attr_file__new( - git_attr_file **attrs_ptr, git_attr_file_source src, const char *path, git_pool *pool); +int git_attr_file__new( + git_attr_file **out, + git_attr_cache_entry *ce, + git_attr_cache_source source); -extern int git_attr_file__new_and_load( - git_attr_file **attrs_ptr, const char *path); +void git_attr_file__free(git_attr_file *file); -extern void git_attr_file__free(git_attr_file *file); +int git_attr_file__load( + git_attr_file **out, + git_repository *repo, + git_attr_cache_entry *ce, + git_attr_cache_source source, + git_attr_cache_parser parser, + void *payload); -extern void git_attr_file__clear_rules(git_attr_file *file); +int git_attr_file__load_standalone( + git_attr_file **out, + const char *path); -extern int git_attr_file__parse_buffer( - git_repository *repo, void *parsedata, const char *buf, git_attr_file *file); +int git_attr_file__out_of_date( + git_repository *repo, git_attr_file *file); -extern int git_attr_file__lookup_one( +int git_attr_file__parse_buffer( + git_repository *repo, + git_attr_file *attrs, + const char *data, + void *payload); + +void git_attr_file__clear_rules(git_attr_file *file); + +int git_attr_file__lookup_one( git_attr_file *file, const git_attr_path *path, const char *attr, @@ -115,7 +128,7 @@ extern int git_attr_file__lookup_one( git_vector_rforeach(&(file)->rules, (iter), (rule)) \ if (git_attr_rule__match((rule), (path))) -extern uint32_t git_attr_file__name_hash(const char *name); +uint32_t git_attr_file__name_hash(const char *name); /* diff --git a/src/attrcache.c b/src/attrcache.c new file mode 100644 index 000000000..6d097234c --- /dev/null +++ b/src/attrcache.c @@ -0,0 +1,397 @@ +#include "common.h" +#include "repository.h" +#include "attr_file.h" +#include "config.h" +#include "sysdir.h" +#include "ignore.h" + +GIT__USE_STRMAP; + +GIT_INLINE(int) attr_cache_lock(git_attr_cache *cache) +{ + GIT_UNUSED(cache); /* avoid warning if threading is off */ + + if (git_mutex_lock(&cache->lock) < 0) { + giterr_set(GITERR_OS, "Unable to get attr cache lock"); + return -1; + } + return 0; +} + +GIT_INLINE(void) attr_cache_unlock(git_attr_cache *cache) +{ + GIT_UNUSED(cache); /* avoid warning if threading is off */ + git_mutex_unlock(&cache->lock); +} + +GIT_INLINE(git_attr_cache_entry *) attr_cache_lookup_entry( + git_attr_cache *cache, const char *path) +{ + khiter_t pos = git_strmap_lookup_index(cache->files, path); + + if (git_strmap_valid_index(cache->files, pos)) + return git_strmap_value_at(cache->files, pos); + else + return NULL; +} + +int git_attr_cache_entry__new( + git_attr_cache_entry **out, + const char *base, + const char *path, + git_pool *pool) +{ + size_t baselen = base ? strlen(base) : 0, pathlen = strlen(path); + size_t cachesize = sizeof(git_attr_cache_entry) + baselen + pathlen + 1; + git_attr_cache_entry *ce; + + ce = git_pool_mallocz(pool, cachesize); + GITERR_CHECK_ALLOC(ce); + + if (baselen) + memcpy(ce->fullpath, base, baselen); + memcpy(&ce->fullpath[baselen], path, pathlen); + ce->path = &ce->fullpath[baselen]; + *out = ce; + + return 0; +} + +/* call with attrcache locked */ +static int attr_cache_make_entry( + git_attr_cache_entry **out, git_repository *repo, const char *path) +{ + int error = 0; + git_attr_cache *cache = git_repository_attr_cache(repo); + git_attr_cache_entry *ce = NULL; + + error = git_attr_cache_entry__new( + &ce, git_repository_workdir(repo), path, &cache->pool); + + if (!error) { + git_strmap_insert(cache->files, ce->path, ce, error); + if (error > 0) + error = 0; + } + + *out = ce; + return error; +} + +/* insert entry or replace existing if we raced with another thread */ +static int attr_cache_upsert(git_attr_cache *cache, git_attr_file *file) +{ + git_attr_cache_entry *ce; + git_attr_file *old; + + if (attr_cache_lock(cache) < 0) + return -1; + + ce = attr_cache_lookup_entry(cache, file->ce->path); + + old = ce->file[file->source]; + + GIT_REFCOUNT_OWN(file, ce); + GIT_REFCOUNT_INC(file); + ce->file[file->source] = file; + + if (old) { + GIT_REFCOUNT_OWN(old, NULL); + git_attr_file__free(old); + } + + attr_cache_unlock(cache); + return 0; +} + +static int attr_cache_remove(git_attr_cache *cache, git_attr_file *file) +{ + int error = 0; + git_attr_cache_entry *ce; + bool found = false; + + if (!file) + return 0; + if ((error = attr_cache_lock(cache)) < 0) + return error; + + if ((ce = attr_cache_lookup_entry(cache, file->ce->path)) != NULL && + ce->file[file->source] == file) + { + ce->file[file->source] = NULL; + found = true; + } + + attr_cache_unlock(cache); + + if (found) + git_attr_file__free(file); + + return error; +} + +int git_attr_cache__get( + git_attr_file **out, + git_repository *repo, + git_attr_cache_source source, + const char *base, + const char *filename, + git_attr_cache_parser parser, + void *payload) +{ + int error = 0; + git_buf path = GIT_BUF_INIT; + const char *wd = git_repository_workdir(repo), *relfile; + git_attr_cache *cache = git_repository_attr_cache(repo); + git_attr_cache_entry *ce = NULL; + git_attr_file *file = NULL; + + /* join base and path as needed */ + if (base != NULL && git_path_root(filename) < 0) { + if (git_buf_joinpath(&path, base, filename) < 0) + return -1; + filename = path.ptr; + } + + relfile = filename; + if (wd && !git__prefixcmp(relfile, wd)) + relfile += strlen(wd); + + /* check cache for existing entry */ + if ((error = attr_cache_lock(cache)) < 0) + goto cleanup; + + ce = attr_cache_lookup_entry(cache, relfile); + if (!ce) { + if ((error = attr_cache_make_entry(&ce, repo, relfile)) < 0) + goto cleanup; + } else if (ce->file[source] != NULL) { + file = ce->file[source]; + GIT_REFCOUNT_INC(file); + } + + attr_cache_unlock(cache); + + /* if this is not a file backed entry, just create a new empty one */ + if (!parser) { + error = git_attr_file__new(&file, ce, source); + goto cleanup; + } + + /* otherwise load and/or reload as needed */ + switch (git_attr_file__out_of_date(repo, file)) { + case 1: + if (!(error = git_attr_file__load( + &file, repo, ce, source, parser, payload))) + error = attr_cache_upsert(cache, file); + break; + case 0: + /* just use the file */ + break; + case GIT_ENOTFOUND: + /* did exist and now does not - remove from cache */ + error = attr_cache_remove(cache, file); + file = NULL; + break; + default: + /* other error (e.g. out of memory, can't read index) */ + giterr_clear(); + break; + } + +cleanup: + *out = error ? NULL : file; + git_buf_free(&path); + return error; +} + +bool git_attr_cache__is_cached( + git_repository *repo, + git_attr_cache_source source, + const char *filename) +{ + git_attr_cache *cache = git_repository_attr_cache(repo); + git_strmap *files; + khiter_t pos; + git_attr_cache_entry *ce; + + if (!(cache = git_repository_attr_cache(repo)) || + !(files = cache->files)) + return false; + + pos = git_strmap_lookup_index(files, filename); + if (!git_strmap_valid_index(files, pos)) + return false; + + ce = git_strmap_value_at(files, pos); + + return ce && (ce->file[source] != NULL); +} + + +static int attr_cache__lookup_path( + char **out, git_config *cfg, const char *key, const char *fallback) +{ + git_buf buf = GIT_BUF_INIT; + int error; + const git_config_entry *entry = NULL; + + *out = NULL; + + if ((error = git_config__lookup_entry(&entry, cfg, key, false)) < 0) + return error; + + if (entry) { + const char *cfgval = entry->value; + + /* expand leading ~/ as needed */ + if (cfgval && cfgval[0] == '~' && cfgval[1] == '/' && + !git_sysdir_find_global_file(&buf, &cfgval[2])) + *out = git_buf_detach(&buf); + else if (cfgval) + *out = git__strdup(cfgval); + + } + else if (!git_sysdir_find_xdg_file(&buf, fallback)) + *out = git_buf_detach(&buf); + + git_buf_free(&buf); + + return error; +} + +static void attr_cache__free(git_attr_cache *cache) +{ + if (!cache) + return; + + if (cache->files != NULL) { + git_attr_file *file; + + git_strmap_foreach_value(cache->files, file, { + git_attr_file__free(file); + }); + git_strmap_free(cache->files); + } + + if (cache->macros != NULL) { + git_attr_rule *rule; + + git_strmap_foreach_value(cache->macros, rule, { + git_attr_rule__free(rule); + }); + git_strmap_free(cache->macros); + } + + git_pool_clear(&cache->pool); + + git__free(cache->cfg_attr_file); + cache->cfg_attr_file = NULL; + + git__free(cache->cfg_excl_file); + cache->cfg_excl_file = NULL; + + git_mutex_free(&cache->lock); + + git__free(cache); +} + +int git_attr_cache__init(git_repository *repo) +{ + int ret = 0; + git_attr_cache *cache = git_repository_attr_cache(repo); + git_config *cfg; + + if (cache) + return 0; + + if ((ret = git_repository_config__weakptr(&cfg, repo)) < 0) + return ret; + + cache = git__calloc(1, sizeof(git_attr_cache)); + GITERR_CHECK_ALLOC(cache); + + /* set up lock */ + if (git_mutex_init(&cache->lock) < 0) { + giterr_set(GITERR_OS, "Unable to initialize lock for attr cache"); + git__free(cache); + return -1; + } + + /* cache config settings for attributes and ignores */ + ret = attr_cache__lookup_path( + &cache->cfg_attr_file, cfg, GIT_ATTR_CONFIG, GIT_ATTR_FILE_XDG); + if (ret < 0) + goto cancel; + + ret = attr_cache__lookup_path( + &cache->cfg_excl_file, cfg, GIT_IGNORE_CONFIG, GIT_IGNORE_FILE_XDG); + if (ret < 0) + goto cancel; + + /* allocate hashtable for attribute and ignore file contents, + * hashtable for attribute macros, and string pool + */ + if ((ret = git_strmap_alloc(&cache->files)) < 0 || + (ret = git_strmap_alloc(&cache->macros)) < 0 || + (ret = git_pool_init(&cache->pool, 1, 0)) < 0) + goto cancel; + + cache = git__compare_and_swap(&repo->attrcache, NULL, cache); + if (cache) + goto cancel; /* raced with another thread, free this but no error */ + + /* insert default macros */ + return git_attr_add_macro(repo, "binary", "-diff -crlf -text"); + +cancel: + attr_cache__free(cache); + return ret; +} + +void git_attr_cache_flush(git_repository *repo) +{ + git_attr_cache *cache; + + /* this could be done less expensively, but for now, we'll just free + * the entire attrcache and let the next use reinitialize it... + */ + if (repo && (cache = git__swap(repo->attrcache, NULL)) != NULL) + attr_cache__free(cache); +} + +int git_attr_cache__insert_macro(git_repository *repo, git_attr_rule *macro) +{ + git_attr_cache *cache = git_repository_attr_cache(repo); + git_strmap *macros = cache->macros; + int error; + + /* TODO: generate warning log if (macro->assigns.length == 0) */ + if (macro->assigns.length == 0) + return 0; + + if (git_mutex_lock(&cache->lock) < 0) { + giterr_set(GITERR_OS, "Unable to get attr cache lock"); + error = -1; + } else { + git_strmap_insert(macros, macro->match.pattern, macro, error); + git_mutex_unlock(&cache->lock); + } + + return (error < 0) ? -1 : 0; +} + +git_attr_rule *git_attr_cache__lookup_macro( + git_repository *repo, const char *name) +{ + git_strmap *macros = git_repository_attr_cache(repo)->macros; + khiter_t pos; + + pos = git_strmap_lookup_index(macros, name); + + if (!git_strmap_valid_index(macros, pos)) + return NULL; + + return (git_attr_rule *)git_strmap_value_at(macros, pos); +} + diff --git a/src/attrcache.h b/src/attrcache.h index 4f9cff6bb..8e7f022b0 100644 --- a/src/attrcache.h +++ b/src/attrcache.h @@ -9,11 +9,15 @@ #include "pool.h" #include "strmap.h" +#include "buffer.h" + +#define GIT_ATTR_CONFIG "core.attributesfile" +#define GIT_IGNORE_CONFIG "core.excludesfile" typedef struct { char *cfg_attr_file; /* cached value of core.attributesfile */ char *cfg_excl_file; /* cached value of core.excludesfile */ - git_strmap *files; /* hash path to git_attr_file of rules */ + git_strmap *files; /* hash path to git_attr_cache_entry records */ git_strmap *macros; /* hash name to vector */ git_mutex lock; git_pool pool; @@ -21,4 +25,53 @@ typedef struct { extern int git_attr_cache__init(git_repository *repo); +typedef enum { + GIT_ATTR_CACHE__FROM_FILE = 0, + GIT_ATTR_CACHE__FROM_INDEX = 1, + + GIT_ATTR_CACHE_NUM_SOURCES = 2 +} git_attr_cache_source; + +typedef struct git_attr_file git_attr_file; +typedef struct git_attr_rule git_attr_rule; + +typedef struct { + git_attr_file *file[GIT_ATTR_CACHE_NUM_SOURCES]; + const char *path; /* points into fullpath */ + char fullpath[GIT_FLEX_ARRAY]; +} git_attr_cache_entry; + +typedef int (*git_attr_cache_parser)( + git_repository *repo, + git_attr_file *file, + const char *data, + void *payload); + +/* get file - loading and reload as needed */ +extern int git_attr_cache__get( + git_attr_file **file, + git_repository *repo, + git_attr_cache_source source, + const char *base, + const char *filename, + git_attr_cache_parser parser, + void *payload); + +extern bool git_attr_cache__is_cached( + git_repository *repo, + git_attr_cache_source source, + const char *path); + +extern int git_attr_cache__insert_macro( + git_repository *repo, git_attr_rule *macro); + +extern git_attr_rule *git_attr_cache__lookup_macro( + git_repository *repo, const char *name); + +extern int git_attr_cache_entry__new( + git_attr_cache_entry **out, + const char *base, + const char *path, + git_pool *pool); + #endif diff --git a/src/fileops.c b/src/fileops.c index 5709499b0..d8d819151 100644 --- a/src/fileops.c +++ b/src/fileops.c @@ -804,10 +804,8 @@ int git_futils_filestamp_check( if (stamp == NULL) return 1; - if (p_stat(path, &st) < 0) { - giterr_set(GITERR_OS, "Could not stat '%s'", path); + if (p_stat(path, &st) < 0) return GIT_ENOTFOUND; - } if (stamp->mtime == (git_time_t)st.st_mtime && stamp->size == (git_off_t)st.st_size && diff --git a/src/fileops.h b/src/fileops.h index 6a65235de..cfc2ce701 100644 --- a/src/fileops.h +++ b/src/fileops.h @@ -292,13 +292,14 @@ typedef struct { * Compare stat information for file with reference info. * * This function updates the file stamp to current data for the given path - * and returns 0 if the file is up-to-date relative to the prior setting or - * 1 if the file has been changed. (This also may return GIT_ENOTFOUND if - * the file doesn't exist.) + * and returns 0 if the file is up-to-date relative to the prior setting, + * 1 if the file has been changed, or GIT_ENOTFOUND if the file doesn't + * exist. This will not call giterr_set, so you must set the error if you + * plan to return an error. * * @param stamp File stamp to be checked * @param path Path to stat and check if changed - * @return 0 if up-to-date, 1 if out-of-date, <0 on error + * @return 0 if up-to-date, 1 if out-of-date, GIT_ENOTFOUND if cannot stat */ extern int git_futils_filestamp_check( git_futils_filestamp *stamp, const char *path); diff --git a/src/ignore.c b/src/ignore.c index 0fb042a34..3ee7ba03a 100644 --- a/src/ignore.c +++ b/src/ignore.c @@ -1,7 +1,7 @@ #include "git2/ignore.h" #include "common.h" #include "ignore.h" -#include "attr.h" +#include "attr_file.h" #include "path.h" #include "config.h" @@ -10,26 +10,27 @@ #define GIT_IGNORE_DEFAULT_RULES ".\n..\n.git\n" static int parse_ignore_file( - git_repository *repo, void *parsedata, const char *buffer, git_attr_file *ignores) + git_repository *repo, + git_attr_file *attrs, + const char *data, + void *payload) { int error = 0; - git_attr_fnmatch *match = NULL; - const char *scan = NULL, *context = NULL; int ignore_case = false; + const char *scan = data, *context = NULL; + git_attr_fnmatch *match = NULL; - /* Prefer to have the caller pass in a git_ignores as the parsedata - * object. If they did not, then look up the value of ignore_case */ - if (parsedata != NULL) - ignore_case = ((git_ignores *)parsedata)->ignore_case; + /* either read ignore_case from ignores structure or use repo config */ + if (payload != NULL) + ignore_case = ((git_ignores *)payload)->ignore_case; else if (git_repository__cvar(&ignore_case, repo, GIT_CVAR_IGNORECASE) < 0) - return error; + giterr_clear(); - if (ignores->key && - git_path_root(ignores->key + 2) < 0 && - git__suffixcmp(ignores->key, "/" GIT_IGNORE_FILE) == 0) - context = ignores->key + 2; - - scan = buffer; + /* if subdir file path, convert context for file paths */ + if (attrs->ce && + git_path_root(attrs->ce->path) < 0 && + !git__suffixcmp(attrs->ce->path, "/" GIT_IGNORE_FILE)) + context = attrs->ce->path; while (!error && *scan) { if (!match) { @@ -40,7 +41,7 @@ static int parse_ignore_file( match->flags = GIT_ATTR_FNMATCH_ALLOWSPACE | GIT_ATTR_FNMATCH_ALLOWNEG; if (!(error = git_attr_fnmatch__parse( - match, ignores->pool, context, &scan))) + match, &attrs->pool, context, &scan))) { match->flags |= GIT_ATTR_FNMATCH_IGNORE; @@ -48,7 +49,7 @@ static int parse_ignore_file( match->flags |= GIT_ATTR_FNMATCH_ICASE; scan = git__next_line(scan); - error = git_vector_insert(&ignores->rules, match); + error = git_vector_insert(&attrs->rules, match); } if (error != 0) { @@ -67,28 +68,46 @@ static int parse_ignore_file( return error; } -#define push_ignore_file(R,IGN,S,B,F) \ - git_attr_cache__push_file((R),(B),(F),GIT_ATTR_FILE_FROM_FILE,parse_ignore_file,(IGN),(S)) +static int push_ignore_file( + git_ignores *ignores, + git_vector *which_list, + const char *base, + const char *filename) +{ + int error = 0; + git_attr_file *file = NULL; + + if ((error = git_attr_cache__get( + &file, ignores->repo, GIT_ATTR_CACHE__FROM_FILE, + base, filename, parse_ignore_file, ignores)) < 0 || + (error = git_vector_insert(which_list, file)) < 0) + git_attr_file__free(file); + + return error; +} static int push_one_ignore(void *payload, git_buf *path) { git_ignores *ign = payload; - ign->depth++; - - return push_ignore_file( - ign->repo, ign, &ign->ign_path, path->ptr, GIT_IGNORE_FILE); + return push_ignore_file(ign, &ign->ign_path, path->ptr, GIT_IGNORE_FILE); } -static int get_internal_ignores(git_attr_file **ign, git_repository *repo) +static int get_internal_ignores(git_attr_file **out, git_repository *repo) { int error; - if (!(error = git_attr_cache__init(repo))) - error = git_attr_cache__internal_file(repo, GIT_IGNORE_INTERNAL, ign); + if ((error = git_attr_cache__init(repo)) < 0) + return error; - if (!error && !(*ign)->rules.length) - error = parse_ignore_file(repo, NULL, GIT_IGNORE_DEFAULT_RULES, *ign); + /* get with NULL parser, gives existing or empty git_attr_file */ + error = git_attr_cache__get( + out, repo, GIT_ATTR_CACHE__FROM_FILE, + NULL, GIT_IGNORE_INTERNAL, NULL, NULL); + + /* if internal rules list is empty, insert default rules */ + if (!error && !(*out)->rules.length) + error = parse_ignore_file(repo, *out, GIT_IGNORE_DEFAULT_RULES, NULL); return error; } @@ -127,8 +146,7 @@ int git_ignore__for_path( goto cleanup; /* set up internals */ - error = get_internal_ignores(&ignores->ign_internal, repo); - if (error < 0) + if ((error = get_internal_ignores(&ignores->ign_internal, repo)) < 0) goto cleanup; /* load .gitignore up the path */ @@ -140,14 +158,16 @@ int git_ignore__for_path( } /* load .git/info/exclude */ - error = push_ignore_file(repo, ignores, &ignores->ign_global, + error = push_ignore_file( + ignores, &ignores->ign_global, git_repository_path(repo), GIT_IGNORE_FILE_INREPO); if (error < 0) goto cleanup; /* load core.excludesfile */ if (git_repository_attr_cache(repo)->cfg_excl_file != NULL) - error = push_ignore_file(repo, ignores, &ignores->ign_global, NULL, + error = push_ignore_file( + ignores, &ignores->ign_global, NULL, git_repository_attr_cache(repo)->cfg_excl_file); cleanup: @@ -165,35 +185,33 @@ int git_ignore__push_dir(git_ignores *ign, const char *dir) ign->depth++; return push_ignore_file( - ign->repo, ign, &ign->ign_path, ign->dir.ptr, GIT_IGNORE_FILE); + ign, &ign->ign_path, ign->dir.ptr, GIT_IGNORE_FILE); } int git_ignore__pop_dir(git_ignores *ign) { if (ign->ign_path.length > 0) { git_attr_file *file = git_vector_last(&ign->ign_path); - const char *start, *end, *scan; - size_t keylen; + const char *start = file->ce->path, *end; - /* - ign->dir looks something like "a/b/" (or "a/b/c/d/") - * - file->key looks something like "0#a/b/.gitignore + /* - ign->dir looks something like "/home/user/a/b/" (or "a/b/c/d/") + * - file->path looks something like "a/b/.gitignore * - * We are popping the last directory off ign->dir. We also want to - * remove the file from the vector if the directory part of the key - * matches the ign->dir path. We need to test if the "a/b" part of + * We are popping the last directory off ign->dir. We also want + * to remove the file from the vector if the popped directory + * matches the ignore path. We need to test if the "a/b" part of * the file key matches the path we are about to pop. */ - for (start = end = scan = &file->key[2]; *scan; ++scan) - if (*scan == '/') - end = scan; /* point 'end' to last '/' in key */ - keylen = (end - start) + 1; + if ((end = strrchr(start, '/')) != NULL) { + size_t dirlen = (end - start) + 1; - if (ign->dir.size >= keylen && - !memcmp(ign->dir.ptr + ign->dir.size - keylen, start, keylen)) - { - git_attr_file__free(git_vector_last(&ign->ign_path)); - git_vector_pop(&ign->ign_path); + if (ign->dir.size >= dirlen && + !memcmp(ign->dir.ptr + ign->dir.size - dirlen, start, dirlen)) + { + git_vector_pop(&ign->ign_path); + git_attr_file__free(file); + } } } @@ -210,7 +228,7 @@ void git_ignore__free(git_ignores *ignores) unsigned int i; git_attr_file *file; - /* don't need to free ignores->ign_internal it is cached exactly once */ + git_attr_file__free(ignores->ign_internal); git_vector_foreach(&ignores->ign_path, i, file) { git_attr_file__free(file); @@ -283,10 +301,12 @@ int git_ignore_add_rule( const char *rules) { int error; - git_attr_file *ign_internal; + git_attr_file *ign_internal = NULL; - if (!(error = get_internal_ignores(&ign_internal, repo))) + if (!(error = get_internal_ignores(&ign_internal, repo))) { error = parse_ignore_file(repo, NULL, rules, ign_internal); + git_attr_file__free(ign_internal); + } return error; } @@ -300,8 +320,10 @@ int git_ignore_clear_internal_rules( if (!(error = get_internal_ignores(&ign_internal, repo))) { git_attr_file__clear_rules(ign_internal); - return parse_ignore_file( + error = parse_ignore_file( repo, NULL, GIT_IGNORE_DEFAULT_RULES, ign_internal); + + git_attr_file__free(ign_internal); } return error; diff --git a/src/index.c b/src/index.c index 27a557cfb..d7d937f63 100644 --- a/src/index.c +++ b/src/index.c @@ -607,8 +607,15 @@ int git_index_read(git_index *index, int force) } updated = git_futils_filestamp_check(&stamp, index->index_file_path); - if (updated < 0 || (!updated && !force)) + if (updated < 0) { + giterr_set( + GITERR_INDEX, + "Failed to read index: '%s' no longer exists", + index->index_file_path); return updated; + } + if (!updated && !force) + return 0; error = git_futils_readbuffer(&buffer, index->index_file_path); if (error < 0) @@ -667,11 +674,11 @@ int git_index_write(git_index *index) if ((error = git_filebuf_commit(&file)) < 0) return error; - error = git_futils_filestamp_check(&index->stamp, index->index_file_path); - if (error < 0) - return error; + if (git_futils_filestamp_check(&index->stamp, index->index_file_path) < 0) + /* index could not be read from disk! */; + else + index->on_disk = 1; - index->on_disk = 1; return 0; } diff --git a/src/sortedcache.c b/src/sortedcache.c index 625322034..c6b226153 100644 --- a/src/sortedcache.c +++ b/src/sortedcache.c @@ -232,9 +232,8 @@ unlock: void git_sortedcache_updated(git_sortedcache *sc) { - /* update filestamp to latest value */ - if (git_futils_filestamp_check(&sc->stamp, sc->path) < 0) - giterr_clear(); + /* update filestamp to latest value */ + git_futils_filestamp_check(&sc->stamp, sc->path); } /* release all items in sorted cache */ diff --git a/src/submodule.c b/src/submodule.c index 95d3d0d9c..5ddbfe828 100644 --- a/src/submodule.c +++ b/src/submodule.c @@ -1693,8 +1693,6 @@ static int submodule_cache_refresh(git_submodule_cache *cache, int refresh) update_gitmod = (wd != NULL) ? git_futils_filestamp_check(&cache->gitmodules_stamp, path.ptr) : (cache->gitmodules_stamp.mtime != 0); - if (update_gitmod < 0) - giterr_clear(); } /* clear submodule flags that are to be refreshed */ diff --git a/tests/attr/file.c b/tests/attr/file.c index 4eb1d22fe..e35957b51 100644 --- a/tests/attr/file.c +++ b/tests/attr/file.c @@ -11,9 +11,9 @@ void test_attr_file__simple_read(void) git_attr_assignment *assign; git_attr_rule *rule; - cl_git_pass(git_attr_file__new_and_load(&file, cl_fixture("attr/attr0"))); + cl_git_pass(git_attr_file__load_standalone(&file, cl_fixture("attr/attr0"))); - cl_assert_equal_s(cl_fixture("attr/attr0"), file->key + 2); + cl_assert_equal_s(cl_fixture("attr/attr0"), file->ce->path); cl_assert(file->rules.length == 1); rule = get_rule(0); @@ -37,9 +37,9 @@ void test_attr_file__match_variants(void) git_attr_rule *rule; git_attr_assignment *assign; - cl_git_pass(git_attr_file__new_and_load(&file, cl_fixture("attr/attr1"))); + cl_git_pass(git_attr_file__load_standalone(&file, cl_fixture("attr/attr1"))); - cl_assert_equal_s(cl_fixture("attr/attr1"), file->key + 2); + cl_assert_equal_s(cl_fixture("attr/attr1"), file->ce->path); cl_assert(file->rules.length == 10); /* let's do a thorough check of this rule, then just verify @@ -121,9 +121,9 @@ void test_attr_file__assign_variants(void) git_attr_rule *rule; git_attr_assignment *assign; - cl_git_pass(git_attr_file__new_and_load(&file, cl_fixture("attr/attr2"))); + cl_git_pass(git_attr_file__load_standalone(&file, cl_fixture("attr/attr2"))); - cl_assert_equal_s(cl_fixture("attr/attr2"), file->key + 2); + cl_assert_equal_s(cl_fixture("attr/attr2"), file->ce->path); cl_assert(file->rules.length == 11); check_one_assign(file, 0, 0, "pat0", "simple", EXPECT_TRUE, NULL); @@ -187,8 +187,8 @@ void test_attr_file__check_attr_examples(void) git_attr_rule *rule; git_attr_assignment *assign; - cl_git_pass(git_attr_file__new_and_load(&file, cl_fixture("attr/attr3"))); - cl_assert_equal_s(cl_fixture("attr/attr3"), file->key + 2); + cl_git_pass(git_attr_file__load_standalone(&file, cl_fixture("attr/attr3"))); + cl_assert_equal_s(cl_fixture("attr/attr3"), file->ce->path); cl_assert(file->rules.length == 3); rule = get_rule(0); diff --git a/tests/attr/lookup.c b/tests/attr/lookup.c index 200bdd2c7..099597efc 100644 --- a/tests/attr/lookup.c +++ b/tests/attr/lookup.c @@ -9,8 +9,8 @@ void test_attr_lookup__simple(void) git_attr_path path; const char *value = NULL; - cl_git_pass(git_attr_file__new_and_load(&file, cl_fixture("attr/attr0"))); - cl_assert_equal_s(cl_fixture("attr/attr0"), file->key + 2); + cl_git_pass(git_attr_file__load_standalone(&file, cl_fixture("attr/attr0"))); + cl_assert_equal_s(cl_fixture("attr/attr0"), file->ce->path); cl_assert(file->rules.length == 1); cl_git_pass(git_attr_path__init(&path, "test", NULL)); @@ -129,8 +129,8 @@ void test_attr_lookup__match_variants(void) { NULL, NULL, 0, NULL } }; - cl_git_pass(git_attr_file__new_and_load(&file, cl_fixture("attr/attr1"))); - cl_assert_equal_s(cl_fixture("attr/attr1"), file->key + 2); + cl_git_pass(git_attr_file__load_standalone(&file, cl_fixture("attr/attr1"))); + cl_assert_equal_s(cl_fixture("attr/attr1"), file->ce->path); cl_assert(file->rules.length == 10); cl_git_pass(git_attr_path__init(&path, "/testing/for/pat0", NULL)); @@ -190,7 +190,7 @@ void test_attr_lookup__assign_variants(void) { NULL, NULL, 0, NULL } }; - cl_git_pass(git_attr_file__new_and_load(&file, cl_fixture("attr/attr2"))); + cl_git_pass(git_attr_file__load_standalone(&file, cl_fixture("attr/attr2"))); cl_assert(file->rules.length == 11); run_test_cases(file, cases, 0); @@ -225,7 +225,7 @@ void test_attr_lookup__check_attr_examples(void) { NULL, NULL, 0, NULL } }; - cl_git_pass(git_attr_file__new_and_load(&file, cl_fixture("attr/attr3"))); + cl_git_pass(git_attr_file__load_standalone(&file, cl_fixture("attr/attr3"))); cl_assert(file->rules.length == 3); run_test_cases(file, cases, 0); @@ -250,9 +250,9 @@ void test_attr_lookup__from_buffer(void) { NULL, NULL, 0, NULL } }; - cl_git_pass(git_attr_file__new(&file, 0, NULL, NULL)); + cl_git_pass(git_attr_file__new(&file, NULL, 0)); - cl_git_pass(git_attr_file__parse_buffer(NULL, NULL, "a* foo\nabc bar\n* baz", file)); + cl_git_pass(git_attr_file__parse_buffer(NULL, file, "a* foo\nabc bar\n* baz", NULL)); cl_assert(file->rules.length == 3); diff --git a/tests/threads/diff.c b/tests/threads/diff.c index 5565c4bf1..562eec71c 100644 --- a/tests/threads/diff.c +++ b/tests/threads/diff.c @@ -1,59 +1,22 @@ #include "clar_libgit2.h" -#include "thread-utils.h" +#include "thread_helpers.h" static git_repository *_repo; static git_tree *_a, *_b; static git_atomic _counts[4]; static int _check_counts; +#define THREADS 20 + void test_threads_diff__cleanup(void) { cl_git_sandbox_cleanup(); } -static void run_in_parallel( - int repeats, int threads, void *(*func)(void *), - void (*before_test)(void), void (*after_test)(void)) -{ - int r, t, *id = git__calloc(threads, sizeof(int)); -#ifdef GIT_THREADS - git_thread *th = git__calloc(threads, sizeof(git_thread)); - cl_assert(th != NULL); -#else - void *th = NULL; -#endif - - cl_assert(id != NULL); - - for (r = 0; r < repeats; ++r) { - _repo = cl_git_sandbox_reopen(); /* reopen sandbox to flush caches */ - - if (before_test) before_test(); - - for (t = 0; t < threads; ++t) { - id[t] = t; -#ifdef GIT_THREADS - cl_git_pass(git_thread_create(&th[t], NULL, func, &id[t])); -#else - cl_assert(func(&id[t]) == &id[t]); -#endif - } - -#ifdef GIT_THREADS - for (t = 0; t < threads; ++t) - cl_git_pass(git_thread_join(th[t], NULL)); - memset(th, 0, threads * sizeof(git_thread)); -#endif - - if (after_test) after_test(); - } - - git__free(id); - git__free(th); -} - static void setup_trees(void) { + _repo = cl_git_sandbox_reopen(); /* reopen sandbox to flush caches */ + cl_git_pass(git_revparse_single( (git_object **)&_a, _repo, "0017bd4ab1^{tree}")); cl_git_pass(git_revparse_single( @@ -62,8 +25,6 @@ static void setup_trees(void) memset(_counts, 0, sizeof(_counts)); } -#define THREADS 20 - static void free_trees(void) { git_tree_free(_a); _a = NULL; diff --git a/tests/threads/iterator.c b/tests/threads/iterator.c new file mode 100644 index 000000000..4dd251fa5 --- /dev/null +++ b/tests/threads/iterator.c @@ -0,0 +1,49 @@ +#include "clar_libgit2.h" +#include "thread_helpers.h" +#include "iterator.h" + +static git_repository *_repo; + +void test_threads_iterator__cleanup(void) +{ + cl_git_sandbox_cleanup(); +} + +static void *run_workdir_iterator(void *arg) +{ + int error = 0, thread = *(int *)arg; + git_iterator *iter; + const git_index_entry *entry = NULL; + + cl_git_pass(git_iterator_for_workdir( + &iter, _repo, GIT_ITERATOR_DONT_AUTOEXPAND, NULL, NULL)); + + while (!error) { + if (entry && entry->mode == GIT_FILEMODE_TREE) { + error = git_iterator_advance_into(&entry, iter); + + if (error == GIT_ENOTFOUND) + error = git_iterator_advance(&entry, iter); + } else { + error = git_iterator_advance(&entry, iter); + } + + if (!error) + (void)git_iterator_current_is_ignored(iter); + } + + cl_assert_equal_i(GIT_ITEROVER, error); + + git_iterator_free(iter); + + return arg; +} + + +void test_threads_iterator__workdir(void) +{ + _repo = cl_git_sandbox_init("status"); + + run_in_parallel( + 1, 20, run_workdir_iterator, NULL, NULL); +} diff --git a/tests/threads/thread_helpers.c b/tests/threads/thread_helpers.c new file mode 100644 index 000000000..25370dddb --- /dev/null +++ b/tests/threads/thread_helpers.c @@ -0,0 +1,44 @@ +#include "clar_libgit2.h" +#include "thread_helpers.h" + +void run_in_parallel( + int repeats, + int threads, + void *(*func)(void *), + void (*before_test)(void), + void (*after_test)(void)) +{ + int r, t, *id = git__calloc(threads, sizeof(int)); +#ifdef GIT_THREADS + git_thread *th = git__calloc(threads, sizeof(git_thread)); + cl_assert(th != NULL); +#else + void *th = NULL; +#endif + + cl_assert(id != NULL); + + for (r = 0; r < repeats; ++r) { + if (before_test) before_test(); + + for (t = 0; t < threads; ++t) { + id[t] = t; +#ifdef GIT_THREADS + cl_git_pass(git_thread_create(&th[t], NULL, func, &id[t])); +#else + cl_assert(func(&id[t]) == &id[t]); +#endif + } + +#ifdef GIT_THREADS + for (t = 0; t < threads; ++t) + cl_git_pass(git_thread_join(th[t], NULL)); + memset(th, 0, threads * sizeof(git_thread)); +#endif + + if (after_test) after_test(); + } + + git__free(id); + git__free(th); +} diff --git a/tests/threads/thread_helpers.h b/tests/threads/thread_helpers.h new file mode 100644 index 000000000..3c13cfb6b --- /dev/null +++ b/tests/threads/thread_helpers.h @@ -0,0 +1,8 @@ +#include "thread-utils.h" + +void run_in_parallel( + int repeats, + int threads, + void *(*func)(void *), + void (*before_test)(void), + void (*after_test)(void)); From 2e9d813bd69ab014b970a75b5a4f7d24f241cc05 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Fri, 11 Apr 2014 12:12:47 -0700 Subject: [PATCH 16/20] Fix tests with new attr cache code --- src/attr.c | 20 ++++---- src/attr_file.c | 4 +- src/attrcache.c | 119 ++++++++++++++++++++++++++++++++++-------------- src/ignore.c | 27 +++++------ 4 files changed, 111 insertions(+), 59 deletions(-) diff --git a/src/attr.c b/src/attr.c index c53a728de..2e314998f 100644 --- a/src/attr.c +++ b/src/attr.c @@ -60,6 +60,7 @@ int git_attr_get( if ((error = collect_attr_files(repo, flags, pathname, &files)) < 0) goto cleanup; + memset(&attr, 0, sizeof(attr)); attr.name = name; attr.name_hash = git_attr_file__name_hash(name); @@ -295,11 +296,15 @@ static int push_attr_file( int error = 0; git_attr_file *file = NULL; - if ((error = git_attr_cache__get( - &file, repo, source, base, filename, - git_attr_file__parse_buffer, NULL)) < 0 || - (error = git_vector_insert(list, file)) < 0) - git_attr_file__free(file); + error = git_attr_cache__get( + &file, repo, source, base, filename, git_attr_file__parse_buffer, NULL); + if (error < 0) + return error; + + if (file != NULL) { + if ((error = git_vector_insert(list, file)) < 0) + git_attr_file__free(file); + } return error; } @@ -343,9 +348,8 @@ static int collect_attr_files( const char *workdir = git_repository_workdir(repo); attr_walk_up_info info = { NULL }; - if (git_attr_cache__init(repo) < 0 || - git_vector_init(files, 4, NULL) < 0) - return -1; + if ((error = git_attr_cache__init(repo)) < 0) + return error; /* Resolve path in a non-bare repo */ if (workdir != NULL) diff --git a/src/attr_file.c b/src/attr_file.c index 86b3448ee..66f71ad19 100644 --- a/src/attr_file.c +++ b/src/attr_file.c @@ -23,9 +23,7 @@ int git_attr_file__new( git_attr_file *attrs = git__calloc(1, sizeof(git_attr_file)); GITERR_CHECK_ALLOC(attrs); - if (git_pool_init(&attrs->pool, 1, 0) < 0 || - git_vector_init(&attrs->rules, 0, NULL) < 0) - { + if (git_pool_init(&attrs->pool, 1, 0) < 0) { attr_file_free(attrs); return -1; } diff --git a/src/attrcache.c b/src/attrcache.c index 6d097234c..a7dc0c887 100644 --- a/src/attrcache.c +++ b/src/attrcache.c @@ -41,16 +41,29 @@ int git_attr_cache_entry__new( const char *path, git_pool *pool) { - size_t baselen = base ? strlen(base) : 0, pathlen = strlen(path); - size_t cachesize = sizeof(git_attr_cache_entry) + baselen + pathlen + 1; + size_t baselen = 0, pathlen = strlen(path); + size_t cachesize = sizeof(git_attr_cache_entry) + pathlen + 1; git_attr_cache_entry *ce; + if (base != NULL && git_path_root(path) < 0) { + baselen = strlen(base); + cachesize += baselen; + + if (baselen && base[baselen - 1] != '/') + cachesize++; + } + ce = git_pool_mallocz(pool, cachesize); GITERR_CHECK_ALLOC(ce); - if (baselen) + if (baselen) { memcpy(ce->fullpath, base, baselen); + + if (base[baselen - 1] != '/') + ce->fullpath[baselen++] = '/'; + } memcpy(&ce->fullpath[baselen], path, pathlen); + ce->path = &ce->fullpath[baselen]; *out = ce; @@ -119,6 +132,7 @@ static int attr_cache_remove(git_attr_cache *cache, git_attr_file *file) ce->file[file->source] == file) { ce->file[file->source] = NULL; + GIT_REFCOUNT_OWN(file, NULL); found = true; } @@ -130,14 +144,13 @@ static int attr_cache_remove(git_attr_cache *cache, git_attr_file *file) return error; } -int git_attr_cache__get( - git_attr_file **out, +static int attr_cache_lookup( + git_attr_file **out_file, + git_attr_cache_entry **out_ce, git_repository *repo, git_attr_cache_source source, const char *base, - const char *filename, - git_attr_cache_parser parser, - void *payload) + const char *filename) { int error = 0; git_buf path = GIT_BUF_INIT; @@ -172,36 +185,61 @@ int git_attr_cache__get( attr_cache_unlock(cache); +cleanup: + *out_file = file; + *out_ce = ce; + + git_buf_free(&path); + return error; +} + +int git_attr_cache__get( + git_attr_file **out, + git_repository *repo, + git_attr_cache_source source, + const char *base, + const char *filename, + git_attr_cache_parser parser, + void *payload) +{ + int error = 0; + git_attr_cache *cache = git_repository_attr_cache(repo); + git_attr_cache_entry *ce = NULL; + git_attr_file *file = NULL; + + if ((error = attr_cache_lookup(&file, &ce, repo, source, base, filename)) < 0) + goto cleanup; + /* if this is not a file backed entry, just create a new empty one */ if (!parser) { - error = git_attr_file__new(&file, ce, source); - goto cleanup; + if (!file && !(error = git_attr_file__new(&file, ce, source))) + error = attr_cache_upsert(cache, file); + } + /* otherwise load and/or reload as needed */ + else if (!file || (error = git_attr_file__out_of_date(repo, file)) > 0) { + if (!(error = git_attr_file__load( + &file, repo, ce, source, parser, payload))) + error = attr_cache_upsert(cache, file); } - /* otherwise load and/or reload as needed */ - switch (git_attr_file__out_of_date(repo, file)) { - case 1: - if (!(error = git_attr_file__load( - &file, repo, ce, source, parser, payload))) - error = attr_cache_upsert(cache, file); - break; - case 0: - /* just use the file */ - break; - case GIT_ENOTFOUND: - /* did exist and now does not - remove from cache */ - error = attr_cache_remove(cache, file); - file = NULL; - break; - default: - /* other error (e.g. out of memory, can't read index) */ + /* GIT_ENOTFOUND is okay when probing for the file. If the file did + * exist and now does not, we have to remove it from cache, however. + */ + if (error == GIT_ENOTFOUND) { giterr_clear(); - break; + error = 0; + + if (file != NULL) + error = attr_cache_remove(cache, file); } cleanup: - *out = error ? NULL : file; - git_buf_free(&path); + if (error < 0 && file != NULL) { + git_attr_file__free(file); + file = NULL; + } + *out = file; + return error; } @@ -250,7 +288,6 @@ static int attr_cache__lookup_path( *out = git_buf_detach(&buf); else if (cfgval) *out = git__strdup(cfgval); - } else if (!git_sysdir_find_xdg_file(&buf, fallback)) *out = git_buf_detach(&buf); @@ -262,14 +299,24 @@ static int attr_cache__lookup_path( static void attr_cache__free(git_attr_cache *cache) { + bool unlock; + if (!cache) return; - if (cache->files != NULL) { - git_attr_file *file; + unlock = (git_mutex_lock(&cache->lock) == 0); - git_strmap_foreach_value(cache->files, file, { - git_attr_file__free(file); + if (cache->files != NULL) { + git_attr_cache_entry *ce; + int i; + + git_strmap_foreach_value(cache->files, ce, { + for (i = 0; i < GIT_ATTR_CACHE_NUM_SOURCES; ++i) { + if (ce->file[i]) { + GIT_REFCOUNT_OWN(ce->file[i], NULL); + git_attr_file__free(ce->file[i]); + } + } }); git_strmap_free(cache->files); } @@ -291,6 +338,8 @@ static void attr_cache__free(git_attr_cache *cache) git__free(cache->cfg_excl_file); cache->cfg_excl_file = NULL; + if (unlock) + git_mutex_unlock(&cache->lock); git_mutex_free(&cache->lock); git__free(cache); diff --git a/src/ignore.c b/src/ignore.c index 3ee7ba03a..1f9df8ab4 100644 --- a/src/ignore.c +++ b/src/ignore.c @@ -77,11 +77,16 @@ static int push_ignore_file( int error = 0; git_attr_file *file = NULL; - if ((error = git_attr_cache__get( - &file, ignores->repo, GIT_ATTR_CACHE__FROM_FILE, - base, filename, parse_ignore_file, ignores)) < 0 || - (error = git_vector_insert(which_list, file)) < 0) - git_attr_file__free(file); + error = git_attr_cache__get( + &file, ignores->repo, GIT_ATTR_CACHE__FROM_FILE, + base, filename, parse_ignore_file, ignores); + if (error < 0) + return error; + + if (file != NULL) { + if ((error = git_vector_insert(which_list, file)) < 0) + git_attr_file__free(file); + } return error; } @@ -122,19 +127,15 @@ int git_ignore__for_path( assert(ignores); + memset(ignores, 0, sizeof(*ignores)); ignores->repo = repo; - git_buf_init(&ignores->dir, 0); - ignores->ign_internal = NULL; - ignores->depth = 0; /* Read the ignore_case flag */ if ((error = git_repository__cvar( &ignores->ignore_case, repo, GIT_CVAR_IGNORECASE)) < 0) goto cleanup; - if ((error = git_vector_init(&ignores->ign_path, 8, NULL)) < 0 || - (error = git_vector_init(&ignores->ign_global, 2, NULL)) < 0 || - (error = git_attr_cache__init(repo)) < 0) + if ((error = git_attr_cache__init(repo)) < 0) goto cleanup; /* given a unrooted path in a non-bare repo, resolve it */ @@ -304,7 +305,7 @@ int git_ignore_add_rule( git_attr_file *ign_internal = NULL; if (!(error = get_internal_ignores(&ign_internal, repo))) { - error = parse_ignore_file(repo, NULL, rules, ign_internal); + error = parse_ignore_file(repo, ign_internal, rules, NULL); git_attr_file__free(ign_internal); } @@ -321,7 +322,7 @@ int git_ignore_clear_internal_rules( git_attr_file__clear_rules(ign_internal); error = parse_ignore_file( - repo, NULL, GIT_IGNORE_DEFAULT_RULES, ign_internal); + repo, ign_internal, GIT_IGNORE_DEFAULT_RULES, NULL); git_attr_file__free(ign_internal); } From ea642d61f9b3dd3d9e1670621198483541cf4f0d Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Mon, 14 Apr 2014 12:29:27 -0700 Subject: [PATCH 17/20] Fix race checking for existing index items In the threading tests, I was still seeing a race condition where the same item could end up being inserted multiple times into the index. Preserving the sorted-ness of the index outside of the `index_insert` call fixes the issue. --- src/index.c | 52 +++++++++++++++++++++++++++----------------- tests/threads/diff.c | 10 +++++++-- 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/src/index.c b/src/index.c index d7d937f63..083c01fe4 100644 --- a/src/index.c +++ b/src/index.c @@ -292,6 +292,7 @@ static void index_entry_reuc_free(git_index_reuc_entry *reuc) static void index_entry_free(git_index_entry *entry) { + memset(&entry->id, 0, sizeof(entry->id)); git__free(entry); } @@ -415,9 +416,9 @@ int git_index_open(git_index **index_out, const char *index_path) } if (git_vector_init(&index->entries, 32, git_index_entry_cmp) < 0 || - git_vector_init(&index->names, 32, conflict_name_cmp) < 0 || - git_vector_init(&index->reuc, 32, reuc_cmp) < 0 || - git_vector_init(&index->deleted, 2, git_index_entry_cmp) < 0) + git_vector_init(&index->names, 8, conflict_name_cmp) < 0 || + git_vector_init(&index->reuc, 8, reuc_cmp) < 0 || + git_vector_init(&index->deleted, 8, git_index_entry_cmp) < 0) goto fail; index->entries_cmp_path = git__strcmp_cb; @@ -477,7 +478,7 @@ static void index_free_deleted(git_index *index) int readers = (int)git_atomic_get(&index->readers); size_t i; - if (readers > 0) + if (readers > 0 || !index->deleted.length) return; for (i = 0; i < index->deleted.length; ++i) { @@ -500,12 +501,11 @@ static int index_remove_entry(git_index *index, size_t pos) error = git_vector_remove(&index->entries, pos); if (!error) { - int readers = (int)git_atomic_get(&index->readers); - - if (readers > 0) + if (git_atomic_get(&index->readers) > 0) { error = git_vector_insert(&index->deleted, entry); - else + } else { index_entry_free(entry); + } } return error; @@ -529,13 +529,13 @@ int git_index_clear(git_index *index) error = index_remove_entry(index, index->entries.length - 1); index_free_deleted(index); - git_mutex_unlock(&index->lock); - git_index_reuc_clear(index); git_index_name_clear(index); git_futils_filestamp_set(&index->stamp, NULL); + git_mutex_unlock(&index->lock); + return error; } @@ -860,6 +860,7 @@ static int index_entry_dup(git_index_entry **out, const git_index_entry *src) GITERR_CHECK_ALLOC(entry); index_entry_cpy(entry, src); + return 0; } @@ -961,6 +962,15 @@ static int check_file_directory_collision(git_index *index, return 0; } +static int index_no_dups(void **old, void *new) +{ + const git_index_entry *entry = new; + GIT_UNUSED(old); + giterr_set(GITERR_INDEX, "'%s' appears multiple times at stage %d", + entry->path, GIT_IDXENTRY_STAGE(entry)); + return GIT_EEXISTS; +} + /* index_insert takes ownership of the new entry - if it can't insert * it, then it will return an error **and also free the entry**. When * it replaces an existing entry, it will update the entry_ptr with the @@ -987,9 +997,9 @@ static int index_insert( else entry->flags |= GIT_IDXENTRY_NAMEMASK; - if ((error = git_mutex_lock(&index->lock)) < 0) { + if (git_mutex_lock(&index->lock) < 0) { giterr_set(GITERR_OS, "Unable to acquire index lock"); - return error; + return -1; } git_vector_sort(&index->entries); @@ -1010,25 +1020,27 @@ static int index_insert( /* if we are replacing an existing item, overwrite the existing entry * and return it in place of the passed in one. */ - else if (existing && replace) { - index_entry_cpy(existing, entry); + else if (existing) { + if (replace) + index_entry_cpy(existing, entry); index_entry_free(entry); - *entry_ptr = existing; + *entry_ptr = entry = existing; } else { - /* if replacing is not requested or no existing entry exists, just - * insert entry at the end; the index is no longer sorted + /* if replace is not requested or no existing entry exists, insert + * at the sorted position. (Since we re-sort after each insert to + * check for dups, this is actually cheaper in the long run.) */ - error = git_vector_insert(&index->entries, entry); + error = git_vector_insert_sorted(&index->entries, entry, index_no_dups); } - git_mutex_unlock(&index->lock); - if (error < 0) { index_entry_free(*entry_ptr); *entry_ptr = NULL; } + git_mutex_unlock(&index->lock); + return error; } diff --git a/tests/threads/diff.c b/tests/threads/diff.c index 562eec71c..d33e75f2c 100644 --- a/tests/threads/diff.c +++ b/tests/threads/diff.c @@ -15,8 +15,14 @@ void test_threads_diff__cleanup(void) static void setup_trees(void) { + git_index *idx; + _repo = cl_git_sandbox_reopen(); /* reopen sandbox to flush caches */ + /* avoid competing to load initial index */ + cl_git_pass(git_repository_index(&idx, _repo)); + git_index_free(idx); + cl_git_pass(git_revparse_single( (git_object **)&_a, _repo, "0017bd4ab1^{tree}")); cl_git_pass(git_revparse_single( @@ -107,7 +113,7 @@ void test_threads_diff__concurrent_diffs(void) _check_counts = 1; run_in_parallel( - 20, 32, run_index_diffs, setup_trees, free_trees); + 5, 32, run_index_diffs, setup_trees, free_trees); } static void *run_index_diffs_with_modifier(void *arg) @@ -169,5 +175,5 @@ void test_threads_diff__with_concurrent_index_modified(void) _check_counts = 0; run_in_parallel( - 20, 32, run_index_diffs_with_modifier, setup_trees, free_trees); + 5, 16, run_index_diffs_with_modifier, setup_trees, free_trees); } From e6e8530aa6c8773dd523fd6fe07629b9481a8fee Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Mon, 14 Apr 2014 12:31:17 -0700 Subject: [PATCH 18/20] Lock attribute file while reparsing data I don't love this approach, but achieving thread-safety for attribute and ignore data while reloading files would require a larger rewrite in order to avoid this. If an attribute or ignore file is out of date, this holds a lock on the file while we are reloading the data so that another thread won't try to reload the data at the same time. --- src/attr_file.c | 31 +++++++++++++++++++++++++++++-- src/attr_file.h | 4 +++- src/ignore.c | 31 +++++++++++++++++-------------- 3 files changed, 49 insertions(+), 17 deletions(-) diff --git a/src/attr_file.c b/src/attr_file.c index 66f71ad19..574de25bb 100644 --- a/src/attr_file.c +++ b/src/attr_file.c @@ -9,8 +9,13 @@ static void attr_file_free(git_attr_file *file) { - git_attr_file__clear_rules(file); + bool unlock = !git_mutex_lock(&file->lock); + git_attr_file__clear_rules(file, false); git_pool_clear(&file->pool); + if (unlock) + git_mutex_unlock(&file->lock); + git_mutex_free(&file->lock); + git__memzero(file, sizeof(*file)); git__free(file); } @@ -23,6 +28,12 @@ int git_attr_file__new( git_attr_file *attrs = git__calloc(1, sizeof(git_attr_file)); GITERR_CHECK_ALLOC(attrs); + if (git_mutex_init(&attrs->lock) < 0) { + giterr_set(GITERR_OS, "Failed to initialize lock"); + git__free(attrs); + return -1; + } + if (git_pool_init(&attrs->pool, 1, 0) < 0) { attr_file_free(attrs); return -1; @@ -35,14 +46,24 @@ int git_attr_file__new( return 0; } -void git_attr_file__clear_rules(git_attr_file *file) +int git_attr_file__clear_rules(git_attr_file *file, bool need_lock) { unsigned int i; git_attr_rule *rule; + if (need_lock && git_mutex_lock(&file->lock) < 0) { + giterr_set(GITERR_OS, "Failed to lock attribute file"); + return -1; + } + git_vector_foreach(&file->rules, i, rule) git_attr_rule__free(rule); git_vector_free(&file->rules); + + if (need_lock) + git_mutex_unlock(&file->lock); + + return 0; } void git_attr_file__free(git_attr_file *file) @@ -162,6 +183,11 @@ int git_attr_file__parse_buffer( !git__suffixcmp(attrs->ce->path, "/" GIT_ATTR_FILE)) context = attrs->ce->path; + if (git_mutex_lock(&attrs->lock) < 0) { + giterr_set(GITERR_OS, "Failed to lock attribute file"); + return -1; + } + while (!error && *scan) { /* allocate rule if needed */ if (!rule) { @@ -198,6 +224,7 @@ int git_attr_file__parse_buffer( } } + git_mutex_unlock(&attrs->lock); git_attr_rule__free(rule); return error; diff --git a/src/attr_file.h b/src/attr_file.h index f92ce3c96..9b4b8724b 100644 --- a/src/attr_file.h +++ b/src/attr_file.h @@ -66,6 +66,7 @@ typedef struct { struct git_attr_file { git_refcount rc; + git_mutex lock; git_attr_cache_entry *ce; git_attr_cache_source source; git_vector rules; /* vector of or */ @@ -115,7 +116,8 @@ int git_attr_file__parse_buffer( const char *data, void *payload); -void git_attr_file__clear_rules(git_attr_file *file); +int git_attr_file__clear_rules( + git_attr_file *file, bool need_lock); int git_attr_file__lookup_one( git_attr_file *file, diff --git a/src/ignore.c b/src/ignore.c index 1f9df8ab4..fbebd9ad3 100644 --- a/src/ignore.c +++ b/src/ignore.c @@ -32,6 +32,11 @@ static int parse_ignore_file( !git__suffixcmp(attrs->ce->path, "/" GIT_IGNORE_FILE)) context = attrs->ce->path; + if (git_mutex_lock(&attrs->lock) < 0) { + giterr_set(GITERR_OS, "Failed to lock attribute file"); + return -1; + } + while (!error && *scan) { if (!match) { match = git__calloc(1, sizeof(*match)); @@ -63,6 +68,7 @@ static int parse_ignore_file( } } + git_mutex_unlock(&attrs->lock); git__free(match); return error; @@ -247,12 +253,12 @@ void git_ignore__free(git_ignores *ignores) } static bool ignore_lookup_in_rules( - git_vector *rules, git_attr_path *path, int *ignored) + git_attr_file *file, git_attr_path *path, int *ignored) { size_t j; git_attr_fnmatch *match; - git_vector_rforeach(rules, j, match) { + git_vector_rforeach(&file->rules, j, match) { if (git_attr_fnmatch__match(match, path)) { *ignored = ((match->flags & GIT_ATTR_FNMATCH_NEGATIVE) == 0); return true; @@ -274,19 +280,18 @@ int git_ignore__lookup( return -1; /* first process builtins - success means path was found */ - if (ignore_lookup_in_rules( - &ignores->ign_internal->rules, &path, ignored)) + if (ignore_lookup_in_rules(ignores->ign_internal, &path, ignored)) goto cleanup; /* next process files in the path */ git_vector_foreach(&ignores->ign_path, i, file) { - if (ignore_lookup_in_rules(&file->rules, &path, ignored)) + if (ignore_lookup_in_rules(file, &path, ignored)) goto cleanup; } /* last process global ignores */ git_vector_foreach(&ignores->ign_global, i, file) { - if (ignore_lookup_in_rules(&file->rules, &path, ignored)) + if (ignore_lookup_in_rules(file, &path, ignored)) goto cleanup; } @@ -319,10 +324,9 @@ int git_ignore_clear_internal_rules( git_attr_file *ign_internal; if (!(error = get_internal_ignores(&ign_internal, repo))) { - git_attr_file__clear_rules(ign_internal); - - error = parse_ignore_file( - repo, ign_internal, GIT_IGNORE_DEFAULT_RULES, NULL); + if (!(error = git_attr_file__clear_rules(ign_internal, true))) + error = parse_ignore_file( + repo, ign_internal, GIT_IGNORE_DEFAULT_RULES, NULL); git_attr_file__free(ign_internal); } @@ -371,19 +375,18 @@ int git_ignore_path_is_ignored( break; /* first process builtins - success means path was found */ - if (ignore_lookup_in_rules( - &ignores.ign_internal->rules, &path, ignored)) + if (ignore_lookup_in_rules(ignores.ign_internal, &path, ignored)) goto cleanup; /* next process files in the path */ git_vector_foreach(&ignores.ign_path, i, file) { - if (ignore_lookup_in_rules(&file->rules, &path, ignored)) + if (ignore_lookup_in_rules(file, &path, ignored)) goto cleanup; } /* last process global ignores */ git_vector_foreach(&ignores.ign_global, i, file) { - if (ignore_lookup_in_rules(&file->rules, &path, ignored)) + if (ignore_lookup_in_rules(file, &path, ignored)) goto cleanup; } From 823c0e9cc142529912976f2e6abff3db456cb204 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Thu, 17 Apr 2014 11:53:13 -0700 Subject: [PATCH 19/20] Fix broken logic for attr cache invalidation The checks to see if files were out of date in the attibute cache was wrong because the cache-breaker data wasn't getting stored correctly. Additionally, when the cache-breaker triggered, the old file data was being leaked. --- CMakeLists.txt | 5 ++ src/attr.c | 24 ++++----- src/attr_file.c | 115 ++++++++++++++++++++++++++------------- src/attr_file.h | 51 +++++++++++------- src/attrcache.c | 122 +++++++++++++++++++++--------------------- src/attrcache.h | 47 +++++----------- src/fileops.c | 14 +++++ src/fileops.h | 6 +++ src/ignore.c | 62 +++++++++------------ tests/attr/file.c | 8 +-- tests/attr/lookup.c | 6 +-- tests/attr/repo.c | 9 ++-- tests/status/ignore.c | 6 ++- 13 files changed, 265 insertions(+), 210 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 23c5af1fc..918e5b8f7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -305,6 +305,11 @@ ELSE () ENDIF () IF (APPLE) # Apple deprecated OpenSSL SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-deprecated-declarations") + + # With clang, disable some annoying extra warnings + IF (NOT CMAKE_COMPILER_IS_GNUCC) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-const-variable -Wno-unused-function") + ENDIF() ENDIF () IF (PROFILE) SET(CMAKE_C_FLAGS "-pg ${CMAKE_C_FLAGS}") diff --git a/src/attr.c b/src/attr.c index 2e314998f..622874348 100644 --- a/src/attr.c +++ b/src/attr.c @@ -260,26 +260,26 @@ typedef struct { } attr_walk_up_info; static int attr_decide_sources( - uint32_t flags, bool has_wd, bool has_index, git_attr_cache_source *srcs) + uint32_t flags, bool has_wd, bool has_index, git_attr_file_source *srcs) { int count = 0; switch (flags & 0x03) { case GIT_ATTR_CHECK_FILE_THEN_INDEX: if (has_wd) - srcs[count++] = GIT_ATTR_CACHE__FROM_FILE; + srcs[count++] = GIT_ATTR_FILE__FROM_FILE; if (has_index) - srcs[count++] = GIT_ATTR_CACHE__FROM_INDEX; + srcs[count++] = GIT_ATTR_FILE__FROM_INDEX; break; case GIT_ATTR_CHECK_INDEX_THEN_FILE: if (has_index) - srcs[count++] = GIT_ATTR_CACHE__FROM_INDEX; + srcs[count++] = GIT_ATTR_FILE__FROM_INDEX; if (has_wd) - srcs[count++] = GIT_ATTR_CACHE__FROM_FILE; + srcs[count++] = GIT_ATTR_FILE__FROM_FILE; break; case GIT_ATTR_CHECK_INDEX_ONLY: if (has_index) - srcs[count++] = GIT_ATTR_CACHE__FROM_INDEX; + srcs[count++] = GIT_ATTR_FILE__FROM_INDEX; break; } @@ -289,7 +289,7 @@ static int attr_decide_sources( static int push_attr_file( git_repository *repo, git_vector *list, - git_attr_cache_source source, + git_attr_file_source source, const char *base, const char *filename) { @@ -297,7 +297,7 @@ static int push_attr_file( git_attr_file *file = NULL; error = git_attr_cache__get( - &file, repo, source, base, filename, git_attr_file__parse_buffer, NULL); + &file, repo, source, base, filename, git_attr_file__parse_buffer); if (error < 0) return error; @@ -313,7 +313,7 @@ static int push_one_attr(void *ref, git_buf *path) { int error = 0, n_src, i; attr_walk_up_info *info = (attr_walk_up_info *)ref; - git_attr_cache_source src[2]; + git_attr_file_source src[2]; n_src = attr_decide_sources( info->flags, info->workdir != NULL, info->index != NULL, src); @@ -367,7 +367,7 @@ static int collect_attr_files( */ error = push_attr_file( - repo, files, GIT_ATTR_CACHE__FROM_FILE, + repo, files, GIT_ATTR_FILE__FROM_FILE, git_repository_path(repo), GIT_ATTR_FILE_INREPO); if (error < 0) goto cleanup; @@ -385,7 +385,7 @@ static int collect_attr_files( if (git_repository_attr_cache(repo)->cfg_attr_file != NULL) { error = push_attr_file( - repo, files, GIT_ATTR_CACHE__FROM_FILE, + repo, files, GIT_ATTR_FILE__FROM_FILE, NULL, git_repository_attr_cache(repo)->cfg_attr_file); if (error < 0) goto cleanup; @@ -395,7 +395,7 @@ static int collect_attr_files( error = git_sysdir_find_system_file(&dir, GIT_ATTR_FILE_SYSTEM); if (!error) error = push_attr_file( - repo, files, GIT_ATTR_CACHE__FROM_FILE, NULL, dir.ptr); + repo, files, GIT_ATTR_FILE__FROM_FILE, NULL, dir.ptr); else if (error == GIT_ENOTFOUND) { giterr_clear(); error = 0; diff --git a/src/attr_file.c b/src/attr_file.c index 574de25bb..65bbf78e8 100644 --- a/src/attr_file.c +++ b/src/attr_file.c @@ -2,6 +2,7 @@ #include "repository.h" #include "filebuf.h" #include "attr_file.h" +#include "attrcache.h" #include "git2/blob.h" #include "git2/tree.h" #include "index.h" @@ -22,8 +23,8 @@ static void attr_file_free(git_attr_file *file) int git_attr_file__new( git_attr_file **out, - git_attr_cache_entry *ce, - git_attr_cache_source source) + git_attr_file_entry *entry, + git_attr_file_source source) { git_attr_file *attrs = git__calloc(1, sizeof(git_attr_file)); GITERR_CHECK_ALLOC(attrs); @@ -40,7 +41,7 @@ int git_attr_file__new( } GIT_REFCOUNT_INC(attrs); - attrs->ce = ce; + attrs->entry = entry; attrs->source = source; *out = attrs; return 0; @@ -95,42 +96,77 @@ static int attr_file_oid_from_index( int git_attr_file__load( git_attr_file **out, git_repository *repo, - git_attr_cache_entry *ce, - git_attr_cache_source source, - git_attr_cache_parser parser, - void *payload) + git_attr_file_entry *entry, + git_attr_file_source source, + git_attr_file_parser parser) { int error = 0; git_blob *blob = NULL; git_buf content = GIT_BUF_INIT; const char *data = NULL; git_attr_file *file; + struct stat st; *out = NULL; - if (source == GIT_ATTR_CACHE__FROM_INDEX) { + switch (source) { + case GIT_ATTR_FILE__IN_MEMORY: + /* in-memory attribute file doesn't need data */ + break; + case GIT_ATTR_FILE__FROM_INDEX: { git_oid id; - if ((error = attr_file_oid_from_index(&id, repo, ce->path)) < 0 || + if ((error = attr_file_oid_from_index(&id, repo, entry->path)) < 0 || (error = git_blob_lookup(&blob, repo, &id)) < 0) return error; data = git_blob_rawcontent(blob); - } else { - if ((error = git_futils_readbuffer(&content, ce->fullpath)) < 0) - /* always return ENOTFOUND so item will just be skipped */ - /* TODO: issue a warning once warnings API is available */ + break; + } + case GIT_ATTR_FILE__FROM_FILE: { + int fd; + + if (p_stat(entry->fullpath, &st) < 0) + return git_path_set_error(errno, entry->fullpath, "stat"); + if (S_ISDIR(st.st_mode)) return GIT_ENOTFOUND; + + /* For open or read errors, return ENOTFOUND to skip item */ + /* TODO: issue warning when warning API is available */ + + if ((fd = git_futils_open_ro(entry->fullpath)) < 0) + return GIT_ENOTFOUND; + + error = git_futils_readbuffer_fd(&content, fd, (size_t)st.st_size); + p_close(fd); + + if (error < 0) + return GIT_ENOTFOUND; + data = content.ptr; + break; + } + default: + giterr_set(GITERR_INVALID, "Unknown file source %d", source); + return -1; } - if ((error = git_attr_file__new(&file, ce, source)) < 0) + if ((error = git_attr_file__new(&file, entry, source)) < 0) goto cleanup; - if (parser && (error = parser(repo, file, data, payload)) < 0) + if (parser && (error = parser(repo, file, data)) < 0) { git_attr_file__free(file); - else - *out = file; + goto cleanup; + } + + /* write cache breaker */ + if (source == GIT_ATTR_FILE__FROM_INDEX) + git_oid_cpy(&file->cache_data.oid, git_blob_id(blob)); + else if (source == GIT_ATTR_FILE__FROM_FILE) + git_futils_filestamp_set_from_stat(&file->cache_data.stamp, &st); + /* else always cacheable */ + + *out = file; cleanup: git_blob_free(blob); @@ -144,18 +180,29 @@ int git_attr_file__out_of_date(git_repository *repo, git_attr_file *file) if (!file) return 1; - if (file->source == GIT_ATTR_CACHE__FROM_INDEX) { + switch (file->source) { + case GIT_ATTR_FILE__IN_MEMORY: + return 0; + + case GIT_ATTR_FILE__FROM_FILE: + return git_futils_filestamp_check( + &file->cache_data.stamp, file->entry->fullpath); + + case GIT_ATTR_FILE__FROM_INDEX: { int error; git_oid id; - if ((error = attr_file_oid_from_index(&id, repo, file->ce->path)) < 0) + if ((error = attr_file_oid_from_index( + &id, repo, file->entry->path)) < 0) return error; return (git_oid__cmp(&file->cache_data.oid, &id) != 0); } - return git_futils_filestamp_check( - &file->cache_data.stamp, file->ce->fullpath); + default: + giterr_set(GITERR_INVALID, "Invalid file type %d", file->source); + return -1; + } } static int sort_by_hash_and_name(const void *a_raw, const void *b_raw); @@ -166,22 +213,17 @@ static bool parse_optimized_patterns( const char *pattern); int git_attr_file__parse_buffer( - git_repository *repo, - git_attr_file *attrs, - const char *data, - void *payload) + git_repository *repo, git_attr_file *attrs, const char *data) { int error = 0; const char *scan = data, *context = NULL; git_attr_rule *rule = NULL; - GIT_UNUSED(payload); - /* if subdir file path, convert context for file paths */ - if (attrs->ce && - git_path_root(attrs->ce->path) < 0 && - !git__suffixcmp(attrs->ce->path, "/" GIT_ATTR_FILE)) - context = attrs->ce->path; + if (attrs->entry && + git_path_root(attrs->entry->path) < 0 && + !git__suffixcmp(attrs->entry->path, "/" GIT_ATTR_FILE)) + context = attrs->entry->path; if (git_mutex_lock(&attrs->lock) < 0) { giterr_set(GITERR_OS, "Failed to lock attribute file"); @@ -268,19 +310,18 @@ int git_attr_file__lookup_one( return 0; } -int git_attr_file__load_standalone( - git_attr_file **out, - const char *path) +int git_attr_file__load_standalone(git_attr_file **out, const char *path) { int error; git_attr_file *file; git_buf content = GIT_BUF_INIT; - error = git_attr_file__new(&file, NULL, GIT_ATTR_CACHE__FROM_FILE); + error = git_attr_file__new(&file, NULL, GIT_ATTR_FILE__FROM_FILE); if (error < 0) return error; - error = git_attr_cache_entry__new(&file->ce, NULL, path, &file->pool); + error = git_attr_cache__alloc_file_entry( + &file->entry, NULL, path, &file->pool); if (error < 0) { git_attr_file__free(file); return error; @@ -290,7 +331,7 @@ int git_attr_file__load_standalone( */ if (!(error = git_futils_readbuffer(&content, path))) { - error = git_attr_file__parse_buffer(NULL, file, content.ptr, NULL); + error = git_attr_file__parse_buffer(NULL, file, content.ptr); git_buf_free(&content); } diff --git a/src/attr_file.h b/src/attr_file.h index 9b4b8724b..c906be44d 100644 --- a/src/attr_file.h +++ b/src/attr_file.h @@ -13,7 +13,6 @@ #include "pool.h" #include "buffer.h" #include "fileops.h" -#include "attrcache.h" #define GIT_ATTR_FILE ".gitattributes" #define GIT_ATTR_FILE_INREPO "info/attributes" @@ -36,6 +35,14 @@ (GIT_ATTR_FNMATCH_ALLOWSPACE | \ GIT_ATTR_FNMATCH_ALLOWNEG | GIT_ATTR_FNMATCH_ALLOWMACRO) +typedef enum { + GIT_ATTR_FILE__IN_MEMORY = 0, + GIT_ATTR_FILE__FROM_FILE = 1, + GIT_ATTR_FILE__FROM_INDEX = 2, + + GIT_ATTR_FILE_NUM_SOURCES = 3 +} git_attr_file_source; + extern const char *git_attr__true; extern const char *git_attr__false; extern const char *git_attr__unset; @@ -46,10 +53,10 @@ typedef struct { unsigned int flags; } git_attr_fnmatch; -struct git_attr_rule { +typedef struct { git_attr_fnmatch match; git_vector assigns; /* vector of */ -}; +} git_attr_rule; typedef struct { git_refcount unused; @@ -64,19 +71,32 @@ typedef struct { const char *value; } git_attr_assignment; -struct git_attr_file { +typedef struct git_attr_file_entry git_attr_file_entry; + +typedef struct { git_refcount rc; git_mutex lock; - git_attr_cache_entry *ce; - git_attr_cache_source source; + git_attr_file_entry *entry; + git_attr_file_source source; git_vector rules; /* vector of or */ git_pool pool; union { git_oid oid; git_futils_filestamp stamp; } cache_data; +} git_attr_file; + +struct git_attr_file_entry { + git_attr_file *file[GIT_ATTR_FILE_NUM_SOURCES]; + const char *path; /* points into fullpath */ + char fullpath[GIT_FLEX_ARRAY]; }; +typedef int (*git_attr_file_parser)( + git_repository *repo, + git_attr_file *file, + const char *data); + typedef struct { git_buf full; char *path; @@ -90,31 +110,26 @@ typedef struct { int git_attr_file__new( git_attr_file **out, - git_attr_cache_entry *ce, - git_attr_cache_source source); + git_attr_file_entry *entry, + git_attr_file_source source); void git_attr_file__free(git_attr_file *file); int git_attr_file__load( git_attr_file **out, git_repository *repo, - git_attr_cache_entry *ce, - git_attr_cache_source source, - git_attr_cache_parser parser, - void *payload); + git_attr_file_entry *ce, + git_attr_file_source source, + git_attr_file_parser parser); int git_attr_file__load_standalone( - git_attr_file **out, - const char *path); + git_attr_file **out, const char *path); int git_attr_file__out_of_date( git_repository *repo, git_attr_file *file); int git_attr_file__parse_buffer( - git_repository *repo, - git_attr_file *attrs, - const char *data, - void *payload); + git_repository *repo, git_attr_file *attrs, const char *data); int git_attr_file__clear_rules( git_attr_file *file, bool need_lock); diff --git a/src/attrcache.c b/src/attrcache.c index a7dc0c887..925abb57f 100644 --- a/src/attrcache.c +++ b/src/attrcache.c @@ -24,7 +24,7 @@ GIT_INLINE(void) attr_cache_unlock(git_attr_cache *cache) git_mutex_unlock(&cache->lock); } -GIT_INLINE(git_attr_cache_entry *) attr_cache_lookup_entry( +GIT_INLINE(git_attr_file_entry *) attr_cache_lookup_entry( git_attr_cache *cache, const char *path) { khiter_t pos = git_strmap_lookup_index(cache->files, path); @@ -35,15 +35,15 @@ GIT_INLINE(git_attr_cache_entry *) attr_cache_lookup_entry( return NULL; } -int git_attr_cache_entry__new( - git_attr_cache_entry **out, +int git_attr_cache__alloc_file_entry( + git_attr_file_entry **out, const char *base, const char *path, git_pool *pool) { size_t baselen = 0, pathlen = strlen(path); - size_t cachesize = sizeof(git_attr_cache_entry) + pathlen + 1; - git_attr_cache_entry *ce; + size_t cachesize = sizeof(git_attr_file_entry) + pathlen + 1; + git_attr_file_entry *ce; if (base != NULL && git_path_root(path) < 0) { baselen = strlen(base); @@ -72,41 +72,41 @@ int git_attr_cache_entry__new( /* call with attrcache locked */ static int attr_cache_make_entry( - git_attr_cache_entry **out, git_repository *repo, const char *path) + git_attr_file_entry **out, git_repository *repo, const char *path) { int error = 0; git_attr_cache *cache = git_repository_attr_cache(repo); - git_attr_cache_entry *ce = NULL; + git_attr_file_entry *entry = NULL; - error = git_attr_cache_entry__new( - &ce, git_repository_workdir(repo), path, &cache->pool); + error = git_attr_cache__alloc_file_entry( + &entry, git_repository_workdir(repo), path, &cache->pool); if (!error) { - git_strmap_insert(cache->files, ce->path, ce, error); + git_strmap_insert(cache->files, entry->path, entry, error); if (error > 0) error = 0; } - *out = ce; + *out = entry; return error; } /* insert entry or replace existing if we raced with another thread */ static int attr_cache_upsert(git_attr_cache *cache, git_attr_file *file) { - git_attr_cache_entry *ce; + git_attr_file_entry *entry; git_attr_file *old; if (attr_cache_lock(cache) < 0) return -1; - ce = attr_cache_lookup_entry(cache, file->ce->path); + entry = attr_cache_lookup_entry(cache, file->entry->path); - old = ce->file[file->source]; - - GIT_REFCOUNT_OWN(file, ce); + GIT_REFCOUNT_OWN(file, entry); GIT_REFCOUNT_INC(file); - ce->file[file->source] = file; + + old = git__compare_and_swap( + &entry->file[file->source], entry->file[file->source], file); if (old) { GIT_REFCOUNT_OWN(old, NULL); @@ -120,7 +120,7 @@ static int attr_cache_upsert(git_attr_cache *cache, git_attr_file *file) static int attr_cache_remove(git_attr_cache *cache, git_attr_file *file) { int error = 0; - git_attr_cache_entry *ce; + git_attr_file_entry *entry; bool found = false; if (!file) @@ -128,27 +128,29 @@ static int attr_cache_remove(git_attr_cache *cache, git_attr_file *file) if ((error = attr_cache_lock(cache)) < 0) return error; - if ((ce = attr_cache_lookup_entry(cache, file->ce->path)) != NULL && - ce->file[file->source] == file) - { - ce->file[file->source] = NULL; - GIT_REFCOUNT_OWN(file, NULL); - found = true; - } + if ((entry = attr_cache_lookup_entry(cache, file->entry->path)) != NULL) + file = git__compare_and_swap(&entry->file[file->source], file, NULL); attr_cache_unlock(cache); - if (found) + if (found) { + GIT_REFCOUNT_OWN(file, NULL); git_attr_file__free(file); + } return error; } +/* Look up cache entry and file. + * - If entry is not present, create it while the cache is locked. + * - If file is present, increment refcount before returning it, so the + * cache can be unlocked and it won't go away. + */ static int attr_cache_lookup( git_attr_file **out_file, - git_attr_cache_entry **out_ce, + git_attr_file_entry **out_entry, git_repository *repo, - git_attr_cache_source source, + git_attr_file_source source, const char *base, const char *filename) { @@ -156,7 +158,7 @@ static int attr_cache_lookup( git_buf path = GIT_BUF_INIT; const char *wd = git_repository_workdir(repo), *relfile; git_attr_cache *cache = git_repository_attr_cache(repo); - git_attr_cache_entry *ce = NULL; + git_attr_file_entry *entry = NULL; git_attr_file *file = NULL; /* join base and path as needed */ @@ -174,20 +176,20 @@ static int attr_cache_lookup( if ((error = attr_cache_lock(cache)) < 0) goto cleanup; - ce = attr_cache_lookup_entry(cache, relfile); - if (!ce) { - if ((error = attr_cache_make_entry(&ce, repo, relfile)) < 0) + entry = attr_cache_lookup_entry(cache, relfile); + if (!entry) { + if ((error = attr_cache_make_entry(&entry, repo, relfile)) < 0) goto cleanup; - } else if (ce->file[source] != NULL) { - file = ce->file[source]; + } else if (entry->file[source] != NULL) { + file = entry->file[source]; GIT_REFCOUNT_INC(file); } attr_cache_unlock(cache); cleanup: - *out_file = file; - *out_ce = ce; + *out_file = file; + *out_entry = entry; git_buf_free(&path); return error; @@ -196,29 +198,26 @@ cleanup: int git_attr_cache__get( git_attr_file **out, git_repository *repo, - git_attr_cache_source source, + git_attr_file_source source, const char *base, const char *filename, - git_attr_cache_parser parser, - void *payload) + git_attr_file_parser parser) { int error = 0; git_attr_cache *cache = git_repository_attr_cache(repo); - git_attr_cache_entry *ce = NULL; + git_attr_file_entry *entry = NULL; git_attr_file *file = NULL; - if ((error = attr_cache_lookup(&file, &ce, repo, source, base, filename)) < 0) + if ((error = attr_cache_lookup( + &file, &entry, repo, source, base, filename)) < 0) goto cleanup; - /* if this is not a file backed entry, just create a new empty one */ - if (!parser) { - if (!file && !(error = git_attr_file__new(&file, ce, source))) - error = attr_cache_upsert(cache, file); - } - /* otherwise load and/or reload as needed */ - else if (!file || (error = git_attr_file__out_of_date(repo, file)) > 0) { - if (!(error = git_attr_file__load( - &file, repo, ce, source, parser, payload))) + /* if file not found or out of date, load up-to-date data and replace */ + if (!file || (error = git_attr_file__out_of_date(repo, file)) > 0) { + /* decrement refcount (if file was found) b/c we will not return it */ + git_attr_file__free(file); + + if (!(error = git_attr_file__load(&file, repo, entry, source, parser))) error = attr_cache_upsert(cache, file); } @@ -245,13 +244,13 @@ cleanup: bool git_attr_cache__is_cached( git_repository *repo, - git_attr_cache_source source, + git_attr_file_source source, const char *filename) { git_attr_cache *cache = git_repository_attr_cache(repo); git_strmap *files; khiter_t pos; - git_attr_cache_entry *ce; + git_attr_file_entry *entry; if (!(cache = git_repository_attr_cache(repo)) || !(files = cache->files)) @@ -261,9 +260,9 @@ bool git_attr_cache__is_cached( if (!git_strmap_valid_index(files, pos)) return false; - ce = git_strmap_value_at(files, pos); + entry = git_strmap_value_at(files, pos); - return ce && (ce->file[source] != NULL); + return entry && (entry->file[source] != NULL); } @@ -307,14 +306,15 @@ static void attr_cache__free(git_attr_cache *cache) unlock = (git_mutex_lock(&cache->lock) == 0); if (cache->files != NULL) { - git_attr_cache_entry *ce; + git_attr_file_entry *entry; + git_attr_file *file; int i; - git_strmap_foreach_value(cache->files, ce, { - for (i = 0; i < GIT_ATTR_CACHE_NUM_SOURCES; ++i) { - if (ce->file[i]) { - GIT_REFCOUNT_OWN(ce->file[i], NULL); - git_attr_file__free(ce->file[i]); + git_strmap_foreach_value(cache->files, entry, { + for (i = 0; i < GIT_ATTR_FILE_NUM_SOURCES; ++i) { + if ((file = git__swap(entry->file[i], NULL)) != NULL) { + GIT_REFCOUNT_OWN(file, NULL); + git_attr_file__free(file); } } }); @@ -345,7 +345,7 @@ static void attr_cache__free(git_attr_cache *cache) git__free(cache); } -int git_attr_cache__init(git_repository *repo) +int git_attr_cache__do_init(git_repository *repo) { int ret = 0; git_attr_cache *cache = git_repository_attr_cache(repo); diff --git a/src/attrcache.h b/src/attrcache.h index 8e7f022b0..be0a22f5c 100644 --- a/src/attrcache.h +++ b/src/attrcache.h @@ -7,9 +7,8 @@ #ifndef INCLUDE_attrcache_h__ #define INCLUDE_attrcache_h__ -#include "pool.h" +#include "attr_file.h" #include "strmap.h" -#include "buffer.h" #define GIT_ATTR_CONFIG "core.attributesfile" #define GIT_IGNORE_CONFIG "core.excludesfile" @@ -23,55 +22,35 @@ typedef struct { git_pool pool; } git_attr_cache; -extern int git_attr_cache__init(git_repository *repo); +extern int git_attr_cache__do_init(git_repository *repo); -typedef enum { - GIT_ATTR_CACHE__FROM_FILE = 0, - GIT_ATTR_CACHE__FROM_INDEX = 1, - - GIT_ATTR_CACHE_NUM_SOURCES = 2 -} git_attr_cache_source; - -typedef struct git_attr_file git_attr_file; -typedef struct git_attr_rule git_attr_rule; - -typedef struct { - git_attr_file *file[GIT_ATTR_CACHE_NUM_SOURCES]; - const char *path; /* points into fullpath */ - char fullpath[GIT_FLEX_ARRAY]; -} git_attr_cache_entry; - -typedef int (*git_attr_cache_parser)( - git_repository *repo, - git_attr_file *file, - const char *data, - void *payload); +#define git_attr_cache__init(REPO) \ + (git_repository_attr_cache(REPO) ? 0 : git_attr_cache__do_init(REPO)) /* get file - loading and reload as needed */ extern int git_attr_cache__get( git_attr_file **file, git_repository *repo, - git_attr_cache_source source, + git_attr_file_source source, const char *base, const char *filename, - git_attr_cache_parser parser, - void *payload); + git_attr_file_parser parser); extern bool git_attr_cache__is_cached( git_repository *repo, - git_attr_cache_source source, + git_attr_file_source source, const char *path); +extern int git_attr_cache__alloc_file_entry( + git_attr_file_entry **out, + const char *base, + const char *path, + git_pool *pool); + extern int git_attr_cache__insert_macro( git_repository *repo, git_attr_rule *macro); extern git_attr_rule *git_attr_cache__lookup_macro( git_repository *repo, const char *name); -extern int git_attr_cache_entry__new( - git_attr_cache_entry **out, - const char *base, - const char *path, - git_pool *pool); - #endif diff --git a/src/fileops.c b/src/fileops.c index d8d819151..13b8f6a39 100644 --- a/src/fileops.c +++ b/src/fileops.c @@ -132,6 +132,7 @@ int git_futils_readbuffer_fd(git_buf *buf, git_file fd, size_t len) if (read_size != (ssize_t)len) { giterr_set(GITERR_OS, "Failed to read descriptor"); + git_buf_free(buf); return -1; } @@ -829,3 +830,16 @@ void git_futils_filestamp_set( else memset(target, 0, sizeof(*target)); } + + +void git_futils_filestamp_set_from_stat( + git_futils_filestamp *stamp, struct stat *st) +{ + if (st) { + stamp->mtime = (git_time_t)st->st_mtime; + stamp->size = (git_off_t)st->st_size; + stamp->ino = (unsigned int)st->st_ino; + } else { + memset(stamp, 0, sizeof(*stamp)); + } +} diff --git a/src/fileops.h b/src/fileops.h index cfc2ce701..62227abae 100644 --- a/src/fileops.h +++ b/src/fileops.h @@ -317,4 +317,10 @@ extern int git_futils_filestamp_check( extern void git_futils_filestamp_set( git_futils_filestamp *tgt, const git_futils_filestamp *src); +/** + * Set file stamp data from stat structure + */ +extern void git_futils_filestamp_set_from_stat( + git_futils_filestamp *stamp, struct stat *st); + #endif /* INCLUDE_fileops_h__ */ diff --git a/src/ignore.c b/src/ignore.c index fbebd9ad3..deae204f8 100644 --- a/src/ignore.c +++ b/src/ignore.c @@ -1,7 +1,7 @@ #include "git2/ignore.h" #include "common.h" #include "ignore.h" -#include "attr_file.h" +#include "attrcache.h" #include "path.h" #include "config.h" @@ -10,30 +10,24 @@ #define GIT_IGNORE_DEFAULT_RULES ".\n..\n.git\n" static int parse_ignore_file( - git_repository *repo, - git_attr_file *attrs, - const char *data, - void *payload) + git_repository *repo, git_attr_file *attrs, const char *data) { int error = 0; int ignore_case = false; const char *scan = data, *context = NULL; git_attr_fnmatch *match = NULL; - /* either read ignore_case from ignores structure or use repo config */ - if (payload != NULL) - ignore_case = ((git_ignores *)payload)->ignore_case; - else if (git_repository__cvar(&ignore_case, repo, GIT_CVAR_IGNORECASE) < 0) + if (git_repository__cvar(&ignore_case, repo, GIT_CVAR_IGNORECASE) < 0) giterr_clear(); /* if subdir file path, convert context for file paths */ - if (attrs->ce && - git_path_root(attrs->ce->path) < 0 && - !git__suffixcmp(attrs->ce->path, "/" GIT_IGNORE_FILE)) - context = attrs->ce->path; + if (attrs->entry && + git_path_root(attrs->entry->path) < 0 && + !git__suffixcmp(attrs->entry->path, "/" GIT_IGNORE_FILE)) + context = attrs->entry->path; if (git_mutex_lock(&attrs->lock) < 0) { - giterr_set(GITERR_OS, "Failed to lock attribute file"); + giterr_set(GITERR_OS, "Failed to lock ignore file"); return -1; } @@ -84,8 +78,8 @@ static int push_ignore_file( git_attr_file *file = NULL; error = git_attr_cache__get( - &file, ignores->repo, GIT_ATTR_CACHE__FROM_FILE, - base, filename, parse_ignore_file, ignores); + &file, ignores->repo, GIT_ATTR_FILE__FROM_FILE, + base, filename, parse_ignore_file); if (error < 0) return error; @@ -111,14 +105,12 @@ static int get_internal_ignores(git_attr_file **out, git_repository *repo) if ((error = git_attr_cache__init(repo)) < 0) return error; - /* get with NULL parser, gives existing or empty git_attr_file */ error = git_attr_cache__get( - out, repo, GIT_ATTR_CACHE__FROM_FILE, - NULL, GIT_IGNORE_INTERNAL, NULL, NULL); + out, repo, GIT_ATTR_FILE__IN_MEMORY, NULL, GIT_IGNORE_INTERNAL, NULL); /* if internal rules list is empty, insert default rules */ if (!error && !(*out)->rules.length) - error = parse_ignore_file(repo, *out, GIT_IGNORE_DEFAULT_RULES, NULL); + error = parse_ignore_file(repo, *out, GIT_IGNORE_DEFAULT_RULES); return error; } @@ -199,7 +191,7 @@ int git_ignore__pop_dir(git_ignores *ign) { if (ign->ign_path.length > 0) { git_attr_file *file = git_vector_last(&ign->ign_path); - const char *start = file->ce->path, *end; + const char *start = file->entry->path, *end; /* - ign->dir looks something like "/home/user/a/b/" (or "a/b/c/d/") * - file->path looks something like "a/b/.gitignore @@ -302,35 +294,33 @@ cleanup: return 0; } -int git_ignore_add_rule( - git_repository *repo, - const char *rules) +int git_ignore_add_rule(git_repository *repo, const char *rules) { int error; git_attr_file *ign_internal = NULL; - if (!(error = get_internal_ignores(&ign_internal, repo))) { - error = parse_ignore_file(repo, ign_internal, rules, NULL); - git_attr_file__free(ign_internal); - } + if ((error = get_internal_ignores(&ign_internal, repo)) < 0) + return error; + + error = parse_ignore_file(repo, ign_internal, rules); + git_attr_file__free(ign_internal); return error; } -int git_ignore_clear_internal_rules( - git_repository *repo) +int git_ignore_clear_internal_rules(git_repository *repo) { int error; git_attr_file *ign_internal; - if (!(error = get_internal_ignores(&ign_internal, repo))) { - if (!(error = git_attr_file__clear_rules(ign_internal, true))) - error = parse_ignore_file( - repo, ign_internal, GIT_IGNORE_DEFAULT_RULES, NULL); + if ((error = get_internal_ignores(&ign_internal, repo)) < 0) + return error; - git_attr_file__free(ign_internal); - } + if (!(error = git_attr_file__clear_rules(ign_internal, true))) + error = parse_ignore_file( + repo, ign_internal, GIT_IGNORE_DEFAULT_RULES); + git_attr_file__free(ign_internal); return error; } diff --git a/tests/attr/file.c b/tests/attr/file.c index e35957b51..1f4108c3c 100644 --- a/tests/attr/file.c +++ b/tests/attr/file.c @@ -13,7 +13,7 @@ void test_attr_file__simple_read(void) cl_git_pass(git_attr_file__load_standalone(&file, cl_fixture("attr/attr0"))); - cl_assert_equal_s(cl_fixture("attr/attr0"), file->ce->path); + cl_assert_equal_s(cl_fixture("attr/attr0"), file->entry->path); cl_assert(file->rules.length == 1); rule = get_rule(0); @@ -39,7 +39,7 @@ void test_attr_file__match_variants(void) cl_git_pass(git_attr_file__load_standalone(&file, cl_fixture("attr/attr1"))); - cl_assert_equal_s(cl_fixture("attr/attr1"), file->ce->path); + cl_assert_equal_s(cl_fixture("attr/attr1"), file->entry->path); cl_assert(file->rules.length == 10); /* let's do a thorough check of this rule, then just verify @@ -123,7 +123,7 @@ void test_attr_file__assign_variants(void) cl_git_pass(git_attr_file__load_standalone(&file, cl_fixture("attr/attr2"))); - cl_assert_equal_s(cl_fixture("attr/attr2"), file->ce->path); + cl_assert_equal_s(cl_fixture("attr/attr2"), file->entry->path); cl_assert(file->rules.length == 11); check_one_assign(file, 0, 0, "pat0", "simple", EXPECT_TRUE, NULL); @@ -188,7 +188,7 @@ void test_attr_file__check_attr_examples(void) git_attr_assignment *assign; cl_git_pass(git_attr_file__load_standalone(&file, cl_fixture("attr/attr3"))); - cl_assert_equal_s(cl_fixture("attr/attr3"), file->ce->path); + cl_assert_equal_s(cl_fixture("attr/attr3"), file->entry->path); cl_assert(file->rules.length == 3); rule = get_rule(0); diff --git a/tests/attr/lookup.c b/tests/attr/lookup.c index 099597efc..030ea075d 100644 --- a/tests/attr/lookup.c +++ b/tests/attr/lookup.c @@ -10,7 +10,7 @@ void test_attr_lookup__simple(void) const char *value = NULL; cl_git_pass(git_attr_file__load_standalone(&file, cl_fixture("attr/attr0"))); - cl_assert_equal_s(cl_fixture("attr/attr0"), file->ce->path); + cl_assert_equal_s(cl_fixture("attr/attr0"), file->entry->path); cl_assert(file->rules.length == 1); cl_git_pass(git_attr_path__init(&path, "test", NULL)); @@ -130,7 +130,7 @@ void test_attr_lookup__match_variants(void) }; cl_git_pass(git_attr_file__load_standalone(&file, cl_fixture("attr/attr1"))); - cl_assert_equal_s(cl_fixture("attr/attr1"), file->ce->path); + cl_assert_equal_s(cl_fixture("attr/attr1"), file->entry->path); cl_assert(file->rules.length == 10); cl_git_pass(git_attr_path__init(&path, "/testing/for/pat0", NULL)); @@ -252,7 +252,7 @@ void test_attr_lookup__from_buffer(void) cl_git_pass(git_attr_file__new(&file, NULL, 0)); - cl_git_pass(git_attr_file__parse_buffer(NULL, file, "a* foo\nabc bar\n* baz", NULL)); + cl_git_pass(git_attr_file__parse_buffer(NULL, file, "a* foo\nabc bar\n* baz")); cl_assert(file->rules.length == 3); diff --git a/tests/attr/repo.c b/tests/attr/repo.c index 49cccdc5a..71dc7a5b5 100644 --- a/tests/attr/repo.c +++ b/tests/attr/repo.c @@ -68,9 +68,12 @@ void test_attr_repo__get_one(void) attr_check_expected(scan->expected, scan->expected_str, scan->attr, value); } - cl_assert(git_attr_cache__is_cached(g_repo, 0, ".git/info/attributes")); - cl_assert(git_attr_cache__is_cached(g_repo, 0, ".gitattributes")); - cl_assert(git_attr_cache__is_cached(g_repo, 0, "sub/.gitattributes")); + cl_assert(git_attr_cache__is_cached( + g_repo, GIT_ATTR_FILE__FROM_FILE, ".git/info/attributes")); + cl_assert(git_attr_cache__is_cached( + g_repo, GIT_ATTR_FILE__FROM_FILE, ".gitattributes")); + cl_assert(git_attr_cache__is_cached( + g_repo, GIT_ATTR_FILE__FROM_FILE, "sub/.gitattributes")); } void test_attr_repo__get_many(void) diff --git a/tests/status/ignore.c b/tests/status/ignore.c index d6c26a847..052a8eae8 100644 --- a/tests/status/ignore.c +++ b/tests/status/ignore.c @@ -54,8 +54,10 @@ void test_status_ignore__0(void) } /* confirm that ignore files were cached */ - cl_assert(git_attr_cache__is_cached(g_repo, 0, ".git/info/exclude")); - cl_assert(git_attr_cache__is_cached(g_repo, 0, ".gitignore")); + cl_assert(git_attr_cache__is_cached( + g_repo, GIT_ATTR_FILE__FROM_FILE, ".git/info/exclude")); + cl_assert(git_attr_cache__is_cached( + g_repo, GIT_ATTR_FILE__FROM_FILE, ".gitignore")); } From 8303827226db114a1157e6173e731f316c217851 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Thu, 17 Apr 2014 14:35:29 -0700 Subject: [PATCH 20/20] Some memory leak fixes --- src/attrcache.c | 52 +++++++++++++++++++++------------------- tests/threads/diff.c | 7 ++++-- tests/threads/iterator.c | 4 ++-- tests/threads/refdb.c | 3 +++ 4 files changed, 37 insertions(+), 29 deletions(-) diff --git a/src/attrcache.c b/src/attrcache.c index 925abb57f..88b68ebb9 100644 --- a/src/attrcache.c +++ b/src/attrcache.c @@ -121,7 +121,6 @@ static int attr_cache_remove(git_attr_cache *cache, git_attr_file *file) { int error = 0; git_attr_file_entry *entry; - bool found = false; if (!file) return 0; @@ -133,7 +132,7 @@ static int attr_cache_remove(git_attr_cache *cache, git_attr_file *file) attr_cache_unlock(cache); - if (found) { + if (file) { GIT_REFCOUNT_OWN(file, NULL); git_attr_file__free(file); } @@ -206,39 +205,42 @@ int git_attr_cache__get( int error = 0; git_attr_cache *cache = git_repository_attr_cache(repo); git_attr_file_entry *entry = NULL; - git_attr_file *file = NULL; + git_attr_file *file = NULL, *updated = NULL; if ((error = attr_cache_lookup( &file, &entry, repo, source, base, filename)) < 0) - goto cleanup; + return error; - /* if file not found or out of date, load up-to-date data and replace */ - if (!file || (error = git_attr_file__out_of_date(repo, file)) > 0) { - /* decrement refcount (if file was found) b/c we will not return it */ - git_attr_file__free(file); + /* load file if we don't have one or if existing one is out of date */ + if (!file || (error = git_attr_file__out_of_date(repo, file)) > 0) + error = git_attr_file__load(&updated, repo, entry, source, parser); - if (!(error = git_attr_file__load(&file, repo, entry, source, parser))) - error = attr_cache_upsert(cache, file); + /* if we loaded the file, insert into and/or update cache */ + if (updated) { + if ((error = attr_cache_upsert(cache, updated)) < 0) + git_attr_file__free(updated); + else { + git_attr_file__free(file); /* offset incref from lookup */ + file = updated; + } } - /* GIT_ENOTFOUND is okay when probing for the file. If the file did - * exist and now does not, we have to remove it from cache, however. - */ - if (error == GIT_ENOTFOUND) { - giterr_clear(); - error = 0; - - if (file != NULL) - error = attr_cache_remove(cache, file); + /* if file could not be loaded */ + if (error < 0) { + /* remove existing entry */ + if (file) { + git_attr_file__free(file); /* offset incref from lookup */ + attr_cache_remove(cache, file); + file = NULL; + } + /* no error if file simply doesn't exist */ + if (error == GIT_ENOTFOUND) { + giterr_clear(); + error = 0; + } } -cleanup: - if (error < 0 && file != NULL) { - git_attr_file__free(file); - file = NULL; - } *out = file; - return error; } diff --git a/tests/threads/diff.c b/tests/threads/diff.c index d33e75f2c..79b85800b 100644 --- a/tests/threads/diff.c +++ b/tests/threads/diff.c @@ -103,6 +103,7 @@ static void *run_index_diffs(void *arg) } git_diff_free(diff); + giterr_clear(); return arg; } @@ -139,8 +140,7 @@ static void *run_index_diffs_with_modifier(void *arg) git_thread_yield(); } - git_index_free(idx); - return arg; + goto done; } /* only use explicit index in this test to prevent reloading */ @@ -164,7 +164,10 @@ static void *run_index_diffs_with_modifier(void *arg) /* results will be unpredictable with index modifier thread running */ git_diff_free(diff); + +done: git_index_free(idx); + giterr_clear(); return arg; } diff --git a/tests/threads/iterator.c b/tests/threads/iterator.c index 4dd251fa5..8aeae1a6c 100644 --- a/tests/threads/iterator.c +++ b/tests/threads/iterator.c @@ -11,7 +11,7 @@ void test_threads_iterator__cleanup(void) static void *run_workdir_iterator(void *arg) { - int error = 0, thread = *(int *)arg; + int error = 0; git_iterator *iter; const git_index_entry *entry = NULL; @@ -35,7 +35,7 @@ static void *run_workdir_iterator(void *arg) cl_assert_equal_i(GIT_ITEROVER, error); git_iterator_free(iter); - + giterr_clear(); return arg; } diff --git a/tests/threads/refdb.c b/tests/threads/refdb.c index fbf6ac09b..3b35b45e3 100644 --- a/tests/threads/refdb.c +++ b/tests/threads/refdb.c @@ -37,6 +37,7 @@ static void *iterate_refs(void *arg) git_reference_iterator_free(i); + giterr_clear(); return arg; } @@ -115,6 +116,7 @@ static void *create_refs(void *arg) for (i = 0; i < 10; ++i) git_reference_free(ref[i]); + giterr_clear(); return arg; } @@ -141,6 +143,7 @@ static void *delete_refs(void *arg) } } + giterr_clear(); return arg; }