diff --git a/src/refdb_fs.c b/src/refdb_fs.c index 658bebb61..876e84588 100644 --- a/src/refdb_fs.c +++ b/src/refdb_fs.c @@ -15,6 +15,7 @@ #include "refdb.h" #include "refdb_fs.h" #include "iterator.h" +#include "sortedcache.h" #include #include @@ -53,243 +54,149 @@ typedef struct refdb_fs_backend { git_repository *repo; char *path; - git_refcache refcache; + git_sortedcache *refcache; int peeling_mode; } refdb_fs_backend; -static int reference_read( - git_buf *file_content, - time_t *mtime, - const char *repo_path, - const char *ref_name, - int *updated) +static int packref_cmp(const void *a_, const void *b_) { - git_buf path = GIT_BUF_INIT; - int result; - - assert(file_content && repo_path && ref_name); - - /* Determine the full path of the file */ - if (git_buf_joinpath(&path, repo_path, ref_name) < 0) - return -1; - - result = git_futils_readbuffer_updated(file_content, path.ptr, mtime, NULL, updated); - git_buf_free(&path); - - return result; + const struct packref *a = a_, *b = b_; + return strcmp(a->name, b->name); } -static int packed_parse_oid( - struct packref **ref_out, - const char **buffer_out, - const char *buffer_end) +static int packed_reload(refdb_fs_backend *backend) { - struct packref *ref = NULL; + int error; + git_buf packedrefs = GIT_BUF_INIT; + char *scan, *eof, *eol; - const char *buffer = *buffer_out; - const char *refname_begin, *refname_end; - - size_t refname_len; - git_oid id; - - refname_begin = (buffer + GIT_OID_HEXSZ + 1); - if (refname_begin >= buffer_end || refname_begin[-1] != ' ') - goto corrupt; - - /* Is this a valid object id? */ - if (git_oid_fromstr(&id, buffer) < 0) - goto corrupt; - - refname_end = memchr(refname_begin, '\n', buffer_end - refname_begin); - if (refname_end == NULL) - refname_end = buffer_end; - - if (refname_end[-1] == '\r') - refname_end--; - - refname_len = refname_end - refname_begin; - - ref = git__calloc(1, sizeof(struct packref) + refname_len + 1); - GITERR_CHECK_ALLOC(ref); - - memcpy(ref->name, refname_begin, refname_len); - ref->name[refname_len] = 0; - - git_oid_cpy(&ref->oid, &id); - - *ref_out = ref; - *buffer_out = refname_end + 1; - return 0; - -corrupt: - git__free(ref); - giterr_set(GITERR_REFERENCE, "The packed references file is corrupted"); - return -1; -} - -static int packed_parse_peel( - struct packref *tag_ref, - const char **buffer_out, - const char *buffer_end) -{ - const char *buffer = *buffer_out + 1; - - assert(buffer[-1] == '^'); - - /* Ensure it's not the first entry of the file */ - if (tag_ref == NULL) - goto corrupt; - - if (buffer + GIT_OID_HEXSZ > buffer_end) - goto corrupt; - - /* Is this a valid object id? */ - if (git_oid_fromstr(&tag_ref->peel, buffer) < 0) - goto corrupt; - - buffer = buffer + GIT_OID_HEXSZ; - if (*buffer == '\r') - buffer++; - - if (buffer != buffer_end) { - if (*buffer == '\n') - buffer++; - else - goto corrupt; - } - - tag_ref->flags |= PACKREF_HAS_PEEL; - *buffer_out = buffer; - return 0; - -corrupt: - giterr_set(GITERR_REFERENCE, "The packed references file is corrupted"); - return -1; -} - -static int packed_load(refdb_fs_backend *backend) -{ - int result, updated; - git_buf packfile = GIT_BUF_INIT; - const char *buffer_start, *buffer_end; - git_refcache *ref_cache = &backend->refcache; - - /* First we make sure we have allocated the hash table */ - if (ref_cache->packfile == NULL) { - ref_cache->packfile = git_strmap_alloc(); - GITERR_CHECK_ALLOC(ref_cache->packfile); - } - - if (backend->path == NULL) + if (!backend->path) return 0; - result = reference_read(&packfile, &ref_cache->packfile_time, - backend->path, GIT_PACKEDREFS_FILE, &updated); + error = git_sortedcache_lockandload(backend->refcache, &packedrefs); /* - * If we couldn't find the file, we need to clear the table and - * return. On any other error, we return that error. If everything - * went fine and the file wasn't updated, then there's nothing new - * for us here, so just return. Anything else means we need to - * refresh the packed refs. + * If we can't find the packed-refs, clear table and return. + * Any other error just gets passed through. + * If no error, and file wasn't changed, just return. + * Anything else means we need to refresh the packed refs. */ - if (result == GIT_ENOTFOUND) { - git_strmap_clear(ref_cache->packfile); - return 0; + if (error <= 0) { + if (error == GIT_ENOTFOUND) { + git_sortedcache_clear(backend->refcache, true); + giterr_clear(); + error = 0; + } + return error; } - if (result < 0) - return -1; + /* At this point, refresh the packed refs from the loaded buffer. */ - if (!updated) - return 0; + git_sortedcache_clear(backend->refcache, false); - /* - * At this point, we want to refresh the packed refs. We already - * have the contents in our buffer. - */ - git_strmap_clear(ref_cache->packfile); - - buffer_start = (const char *)packfile.ptr; - buffer_end = (const char *)(buffer_start) + packfile.size; + scan = (char *)packedrefs.ptr; + eof = scan + packedrefs.size; backend->peeling_mode = PEELING_NONE; - if (buffer_start[0] == '#') { + if (*scan == '#') { static const char *traits_header = "# pack-refs with: "; - if (git__prefixcmp(buffer_start, traits_header) == 0) { - char *traits = (char *)buffer_start + strlen(traits_header); - char *traits_end = strchr(traits, '\n'); + if (git__prefixcmp(scan, traits_header) == 0) { + scan += strlen(traits_header); + eol = strchr(scan, '\n'); - if (traits_end == NULL) + if (!eol) goto parse_failed; + *eol = '\0'; - *traits_end = '\0'; - - if (strstr(traits, " fully-peeled ") != NULL) { + if (strstr(scan, " fully-peeled ") != NULL) { backend->peeling_mode = PEELING_FULL; - } else if (strstr(traits, " peeled ") != NULL) { + } else if (strstr(scan, " peeled ") != NULL) { backend->peeling_mode = PEELING_STANDARD; } - buffer_start = traits_end + 1; + scan = eol + 1; } } - while (buffer_start < buffer_end && buffer_start[0] == '#') { - buffer_start = strchr(buffer_start, '\n'); - if (buffer_start == NULL) + while (scan < eof && *scan == '#') { + if (!(eol = strchr(scan, '\n'))) goto parse_failed; - - buffer_start++; + scan = eol + 1; } - while (buffer_start < buffer_end) { - int err; - struct packref *ref = NULL; + while (scan < eof) { + struct packref *ref; + git_oid oid; - if (packed_parse_oid(&ref, &buffer_start, buffer_end) < 0) + /* parse " \n" */ + + if (git_oid_fromstr(&oid, scan) < 0) goto parse_failed; + scan += GIT_OID_HEXSZ; - if (buffer_start[0] == '^') { - if (packed_parse_peel(ref, &buffer_start, buffer_end) < 0) + if (*scan++ != ' ') + goto parse_failed; + if (!(eol = strchr(scan, '\n'))) + goto parse_failed; + *eol = '\0'; + if (eol[-1] == '\r') + eol[-1] = '\0'; + + if (git_sortedcache_upsert((void **)&ref, backend->refcache, scan) < 0) + goto parse_failed; + scan = eol + 1; + + git_oid_cpy(&ref->oid, &oid); + + /* look for optional "^\n" */ + + if (*scan == '^') { + if (git_oid_fromstr(&oid, scan + 1) < 0) goto parse_failed; - } else if (backend->peeling_mode == PEELING_FULL || - (backend->peeling_mode == PEELING_STANDARD && - git__prefixcmp(ref->name, GIT_REFS_TAGS_DIR) == 0)) { - ref->flags |= PACKREF_CANNOT_PEEL; - } + scan += GIT_OID_HEXSZ + 1; - git_strmap_insert(ref_cache->packfile, ref->name, ref, err); - if (err < 0) - goto parse_failed; + if (scan < eof) { + if (!(eol = strchr(scan, '\n'))) + goto parse_failed; + scan = eol + 1; + } + + git_oid_cpy(&ref->peel, &oid); + ref->flags |= PACKREF_HAS_PEEL; + } + else if (backend->peeling_mode == PEELING_FULL || + (backend->peeling_mode == PEELING_STANDARD && + git__prefixcmp(ref->name, GIT_REFS_TAGS_DIR) == 0)) + ref->flags |= PACKREF_CANNOT_PEEL; } - git_buf_free(&packfile); + git_sortedcache_unlock(backend->refcache); + git_buf_free(&packedrefs); + return 0; parse_failed: - git_strmap_free(ref_cache->packfile); - ref_cache->packfile = NULL; - git_buf_free(&packfile); + giterr_set(GITERR_REFERENCE, "Corrupted packed references file"); + + git_sortedcache_clear(backend->refcache, false); + git_sortedcache_unlock(backend->refcache); + git_buf_free(&packedrefs); + return -1; } -static int loose_parse_oid(git_oid *oid, const char *filename, git_buf *file_content) +static int loose_parse_oid( + git_oid *oid, const char *filename, git_buf *file_content) { - size_t len; - const char *str; + const char *str = git_buf_cstr(file_content); - len = git_buf_len(file_content); - if (len < GIT_OID_HEXSZ) + if (git_buf_len(file_content) < GIT_OID_HEXSZ) goto corrupted; - /* str is guranteed to be zero-terminated */ - str = git_buf_cstr(file_content); - /* we need to get 40 OID characters from the file */ - if (git_oid_fromstr(oid, git_buf_cstr(file_content)) < 0) + if (git_oid_fromstr(oid, str) < 0) goto corrupted; /* If the file is longer than 40 chars, the 41st must be a space */ @@ -302,77 +209,71 @@ corrupted: return -1; } -static int loose_lookup_to_packfile( - struct packref **ref_out, - refdb_fs_backend *backend, - const char *name) +static int loose_readbuffer(git_buf *buf, const char *base, const char *path) { - git_buf ref_file = GIT_BUF_INIT; - struct packref *ref = NULL; - size_t name_len; + int error; - *ref_out = NULL; + /* build full path to file */ + if ((error = git_buf_joinpath(buf, base, path)) < 0 || + (error = git_futils_readbuffer(buf, buf->ptr)) < 0) + git_buf_free(buf); - if (reference_read(&ref_file, NULL, backend->path, name, NULL) < 0) - return -1; - - /* skip symbolic refs */ - if (!git__prefixcmp(git_buf_cstr(&ref_file), GIT_SYMREF)) { - git_buf_free(&ref_file); - return 0; - } - - git_buf_rtrim(&ref_file); - - name_len = strlen(name); - ref = git__calloc(1, sizeof(struct packref) + name_len + 1); - GITERR_CHECK_ALLOC(ref); - - memcpy(ref->name, name, name_len); - ref->name[name_len] = 0; - - if (loose_parse_oid(&ref->oid, name, &ref_file) < 0) { - git_buf_free(&ref_file); - git__free(ref); - return -1; - } - - ref->flags = PACKREF_WAS_LOOSE; - - *ref_out = ref; - git_buf_free(&ref_file); - return 0; + return error; } +static int loose_lookup_to_packfile(refdb_fs_backend *backend, const char *name) +{ + int error = 0; + git_buf ref_file = GIT_BUF_INIT; + struct packref *ref = NULL; + git_oid oid; + + /* if we fail to load the loose reference, assume someone changed + * the filesystem under us and skip it... + */ + if (loose_readbuffer(&ref_file, backend->path, name) < 0) { + giterr_clear(); + goto done; + } + + /* skip symbolic refs */ + if (!git__prefixcmp(git_buf_cstr(&ref_file), GIT_SYMREF)) + goto done; + + /* parse OID from file */ + if ((error = loose_parse_oid(&oid, name, &ref_file)) < 0) + goto done; + + git_sortedcache_lock(backend->refcache); + + if (!(error = git_sortedcache_upsert( + (void **)&ref, backend->refcache, name))) { + + git_oid_cpy(&ref->oid, &oid); + ref->flags = PACKREF_WAS_LOOSE; + } + + git_sortedcache_unlock(backend->refcache); + +done: + git_buf_free(&ref_file); + return error; +} static int _dirent_loose_load(void *data, git_buf *full_path) { refdb_fs_backend *backend = (refdb_fs_backend *)data; - void *old_ref = NULL; - struct packref *ref; const char *file_path; - int err; - if (git_path_isdir(full_path->ptr) == true) + if (git__suffixcmp(full_path->ptr, ".lock") == 0) + return 0; + + if (git_path_isdir(full_path->ptr)) return git_path_direach(full_path, _dirent_loose_load, backend); file_path = full_path->ptr + strlen(backend->path); - if (loose_lookup_to_packfile(&ref, backend, file_path) < 0) - return -1; - - if (!ref) - return 0; - - git_strmap_insert2( - backend->refcache.packfile, ref->name, ref, old_ref, err); - if (err < 0) { - git__free(ref); - return -1; - } - - git__free(old_ref); - return 0; + return loose_lookup_to_packfile(backend, file_path); } /* @@ -383,11 +284,8 @@ static int _dirent_loose_load(void *data, git_buf *full_path) */ static int packed_loadloose(refdb_fs_backend *backend) { + int error; git_buf refs_path = GIT_BUF_INIT; - int result; - - /* the packfile must have been previously loaded! */ - assert(backend->refcache.packfile); if (git_buf_joinpath(&refs_path, backend->path, GIT_REFS_DIR) < 0) return -1; @@ -397,10 +295,11 @@ static int packed_loadloose(refdb_fs_backend *backend) * This will overwrite any old packed entries with their * updated loose versions */ - result = git_path_direach(&refs_path, _dirent_loose_load, backend); + error = git_path_direach(&refs_path, _dirent_loose_load, backend); + git_buf_free(&refs_path); - return result; + return error; } static int refdb_fs_backend__exists( @@ -408,23 +307,17 @@ static int refdb_fs_backend__exists( git_refdb_backend *_backend, const char *ref_name) { - refdb_fs_backend *backend; + refdb_fs_backend *backend = (refdb_fs_backend *)_backend; git_buf ref_path = GIT_BUF_INIT; - assert(_backend); - backend = (refdb_fs_backend *)_backend; + assert(backend); - if (packed_load(backend) < 0) + if (packed_reload(backend) < 0 || + git_buf_joinpath(&ref_path, backend->path, ref_name) < 0) return -1; - if (git_buf_joinpath(&ref_path, backend->path, ref_name) < 0) - return -1; - - if (git_path_isfile(ref_path.ptr) == true || - git_strmap_exists(backend->refcache.packfile, ref_path.ptr)) - *exists = 1; - else - *exists = 0; + *exists = git_path_isfile(ref_path.ptr) || + git_sortedcache_lookup(backend->refcache, ref_name); git_buf_free(&ref_path); return 0; @@ -456,69 +349,39 @@ static int loose_lookup( refdb_fs_backend *backend, const char *ref_name) { - const char *target; - git_oid oid; git_buf ref_file = GIT_BUF_INIT; int error = 0; if (out) *out = NULL; - error = reference_read(&ref_file, NULL, backend->path, ref_name, NULL); + if ((error = loose_readbuffer(&ref_file, backend->path, ref_name)) < 0) + /* cannot read loose ref file - gah */; + else if (git__prefixcmp(git_buf_cstr(&ref_file), GIT_SYMREF) == 0) { + const char *target; - if (error < 0) - goto done; - - if (git__prefixcmp(git_buf_cstr(&ref_file), GIT_SYMREF) == 0) { git_buf_rtrim(&ref_file); - if ((target = loose_parse_symbolic(&ref_file)) == NULL) { + if (!(target = loose_parse_symbolic(&ref_file))) error = -1; - goto done; - } - - if (out) + else if (out != NULL) *out = git_reference__alloc_symbolic(ref_name, target); } else { - if ((error = loose_parse_oid(&oid, ref_name, &ref_file)) < 0) - goto done; + git_oid oid; - if (out) + if (!(error = loose_parse_oid(&oid, ref_name, &ref_file)) && + out != NULL) *out = git_reference__alloc(ref_name, &oid, NULL); } - if (out && *out == NULL) - error = -1; - -done: git_buf_free(&ref_file); return error; } -static int packed_map_entry( - struct packref **entry, - khiter_t *pos, - refdb_fs_backend *backend, - const char *ref_name) +static int ref_error_notfound(const char *name) { - git_strmap *packfile_refs; - - if (packed_load(backend) < 0) - return -1; - - /* Look up on the packfile */ - packfile_refs = backend->refcache.packfile; - - *pos = git_strmap_lookup_index(packfile_refs, ref_name); - - if (!git_strmap_valid_index(packfile_refs, *pos)) { - giterr_set(GITERR_REFERENCE, "Reference '%s' not found", ref_name); - return GIT_ENOTFOUND; - } - - *entry = git_strmap_value_at(packfile_refs, *pos); - - return 0; + giterr_set(GITERR_REFERENCE, "Reference '%s' not found", name); + return GIT_ENOTFOUND; } static int packed_lookup( @@ -526,18 +389,25 @@ static int packed_lookup( refdb_fs_backend *backend, const char *ref_name) { - struct packref *entry; - khiter_t pos; int error = 0; + struct packref *entry; - if ((error = packed_map_entry(&entry, &pos, backend, ref_name)) < 0) - return error; - - if ((*out = git_reference__alloc(ref_name, - &entry->oid, &entry->peel)) == NULL) + if (packed_reload(backend) < 0) return -1; - return 0; + git_sortedcache_lock(backend->refcache); + + entry = git_sortedcache_lookup(backend->refcache, ref_name); + if (!entry) { + error = ref_error_notfound(ref_name); + } else { + *out = git_reference__alloc(ref_name, &entry->oid, &entry->peel); + if (!*out) + error = -1; + } + + git_sortedcache_unlock(backend->refcache); + return error; } static int refdb_fs_backend__lookup( @@ -545,24 +415,22 @@ static int refdb_fs_backend__lookup( git_refdb_backend *_backend, const char *ref_name) { - refdb_fs_backend *backend; - int result; + refdb_fs_backend *backend = (refdb_fs_backend *)_backend; + int error; - assert(_backend); + assert(backend); - backend = (refdb_fs_backend *)_backend; - - if ((result = loose_lookup(out, backend, ref_name)) == 0) + if (!(error = loose_lookup(out, backend, ref_name))) return 0; /* only try to lookup this reference on the packfile if it * wasn't found on the loose refs; not if there was a critical error */ - if (result == GIT_ENOTFOUND) { + if (error == GIT_ENOTFOUND) { giterr_clear(); - result = packed_lookup(out, backend, ref_name); + error = packed_lookup(out, backend, ref_name); } - return result; + return error; } typedef struct { @@ -573,8 +441,8 @@ typedef struct { git_pool pool; git_vector loose; - unsigned int loose_pos; - khiter_t packed_pos; + size_t loose_pos; + size_t packed_pos; } refdb_fs_iter; static void refdb_fs_backend__iterator_free(git_reference_iterator *_iter) @@ -589,7 +457,6 @@ static void refdb_fs_backend__iterator_free(git_reference_iterator *_iter) static int iter_load_loose_paths(refdb_fs_backend *backend, refdb_fs_iter *iter) { int error = 0; - git_strmap *packfile = backend->refcache.packfile; git_buf path = GIT_BUF_INIT; git_iterator *fsit = NULL; const git_index_entry *entry = NULL; @@ -597,18 +464,18 @@ static int iter_load_loose_paths(refdb_fs_backend *backend, refdb_fs_iter *iter) if (!backend->path) /* do nothing if no path for loose refs */ return 0; - if (git_buf_printf(&path, "%s/refs", backend->path) < 0) - return -1; + if ((error = git_buf_printf(&path, "%s/refs", backend->path)) < 0 || + (error = git_iterator_for_filesystem( + &fsit, git_buf_cstr(&path), 0, NULL, NULL)) < 0) { + git_buf_free(&path); + return error; + } - if ((error = git_iterator_for_filesystem( - &fsit, git_buf_cstr(&path), 0, NULL, NULL)) < 0 || - (error = git_vector_init(&iter->loose, 8, NULL)) < 0 || - (error = git_buf_sets(&path, GIT_REFS_DIR)) < 0) - goto cleanup; + error = git_buf_sets(&path, GIT_REFS_DIR); - while (!git_iterator_advance(&entry, fsit)) { + while (!error && !git_iterator_advance(&entry, fsit)) { const char *ref_name; - khiter_t pos; + struct packref *ref; char *ref_dup; git_buf_truncate(&path, strlen(GIT_REFS_DIR)); @@ -619,26 +486,23 @@ static int iter_load_loose_paths(refdb_fs_backend *backend, refdb_fs_iter *iter) (iter->glob && p_fnmatch(iter->glob, ref_name, 0) != 0)) continue; - pos = git_strmap_lookup_index(packfile, ref_name); - if (git_strmap_valid_index(packfile, pos)) { - struct packref *ref = git_strmap_value_at(packfile, pos); + git_sortedcache_lock(backend->refcache); + ref = git_sortedcache_lookup(backend->refcache, ref_name); + if (ref) ref->flags |= PACKREF_SHADOWED; - } + git_sortedcache_unlock(backend->refcache); - if (!(ref_dup = git_pool_strdup(&iter->pool, ref_name))) { + ref_dup = git_pool_strdup(&iter->pool, ref_name); + if (!ref_dup) error = -1; - goto cleanup; - } - - if ((error = git_vector_insert(&iter->loose, ref_dup)) < 0) - goto cleanup; + else + error = git_vector_insert(&iter->loose, ref_dup); } -cleanup: git_iterator_free(fsit); git_buf_free(&path); - return 0; + return error; } static int refdb_fs_backend__iterator_next( @@ -646,7 +510,7 @@ static int refdb_fs_backend__iterator_next( { refdb_fs_iter *iter = (refdb_fs_iter *)_iter; refdb_fs_backend *backend = (refdb_fs_backend *)iter->parent.db->backend; - git_strmap *packfile = backend->refcache.packfile; + struct packref *ref; while (iter->loose_pos < iter->loose.length) { const char *path = git_vector_get(&iter->loose, iter->loose_pos++); @@ -657,17 +521,12 @@ static int refdb_fs_backend__iterator_next( giterr_clear(); } - while (iter->packed_pos < kh_end(packfile)) { - struct packref *ref = NULL; + git_sortedcache_lock(backend->refcache); - while (!kh_exist(packfile, iter->packed_pos)) { - iter->packed_pos++; - if (iter->packed_pos == kh_end(packfile)) - return GIT_ITEROVER; - } - - ref = kh_val(packfile, iter->packed_pos); - iter->packed_pos++; + while (iter->packed_pos < git_sortedcache_entrycount(backend->refcache)) { + ref = git_sortedcache_entry(backend->refcache, iter->packed_pos++); + if (!ref) /* stop now if another thread deleted refs and we past end */ + break; if (ref->flags & PACKREF_SHADOWED) continue; @@ -676,12 +535,11 @@ static int refdb_fs_backend__iterator_next( continue; *out = git_reference__alloc(ref->name, &ref->oid, &ref->peel); - if (*out == NULL) - return -1; - - return 0; + git_sortedcache_unlock(backend->refcache); + return (*out != NULL) ? 0 : -1; } + git_sortedcache_unlock(backend->refcache); return GIT_ITEROVER; } @@ -690,14 +548,10 @@ static int refdb_fs_backend__iterator_next_name( { refdb_fs_iter *iter = (refdb_fs_iter *)_iter; refdb_fs_backend *backend = (refdb_fs_backend *)iter->parent.db->backend; - git_strmap *packfile = backend->refcache.packfile; while (iter->loose_pos < iter->loose.length) { const char *path = git_vector_get(&iter->loose, iter->loose_pos++); - if (git_strmap_exists(packfile, path)) - continue; - if (loose_lookup(NULL, backend, path) != 0) { giterr_clear(); continue; @@ -707,22 +561,25 @@ static int refdb_fs_backend__iterator_next_name( return 0; } - while (iter->packed_pos < kh_end(packfile)) { - while (!kh_exist(packfile, iter->packed_pos)) { - iter->packed_pos++; - if (iter->packed_pos == kh_end(packfile)) - return GIT_ITEROVER; - } + git_sortedcache_lock(backend->refcache); - *out = kh_key(packfile, iter->packed_pos); - iter->packed_pos++; + while (iter->packed_pos < git_sortedcache_entrycount(backend->refcache)) { + struct packref *ref = + git_sortedcache_entry(backend->refcache, iter->packed_pos++); + + if (ref->flags & PACKREF_SHADOWED) + continue; + + *out = ref->name; if (iter->glob && p_fnmatch(iter->glob, *out, 0) != 0) continue; + git_sortedcache_unlock(backend->refcache); return 0; } + git_sortedcache_unlock(backend->refcache); return GIT_ITEROVER; } @@ -730,18 +587,18 @@ static int refdb_fs_backend__iterator( git_reference_iterator **out, git_refdb_backend *_backend, const char *glob) { refdb_fs_iter *iter; - refdb_fs_backend *backend; + refdb_fs_backend *backend = (refdb_fs_backend *)_backend; - assert(_backend); - backend = (refdb_fs_backend *)_backend; + assert(backend); - if (packed_load(backend) < 0) + if (packed_reload(backend) < 0) return -1; iter = git__calloc(1, sizeof(refdb_fs_iter)); GITERR_CHECK_ALLOC(iter); - if (git_pool_init(&iter->pool, 1, 0) < 0) + if (git_pool_init(&iter->pool, 1, 0) < 0 || + git_vector_init(&iter->loose, 8, NULL) < 0) goto fail; if (glob != NULL && @@ -786,15 +643,16 @@ static int reference_path_available( const char* old_ref, int force) { - struct packref *this_ref; + size_t i; - if (packed_load(backend) < 0) + if (packed_reload(backend) < 0) return -1; if (!force) { int exists; - if (refdb_fs_backend__exists(&exists, (git_refdb_backend *)backend, new_ref) < 0) + if (refdb_fs_backend__exists( + &exists, (git_refdb_backend *)backend, new_ref) < 0) return -1; if (exists) { @@ -805,14 +663,21 @@ static int reference_path_available( } } - git_strmap_foreach_value(backend->refcache.packfile, this_ref, { + git_sortedcache_lock(backend->refcache); + + for (i = 0; i < git_sortedcache_entrycount(backend->refcache); ++i) { + struct packref *this_ref = + git_sortedcache_entry(backend->refcache, i); + if (!ref_is_available(old_ref, new_ref, this_ref->name)) { + git_sortedcache_unlock(backend->refcache); giterr_set(GITERR_REFERENCE, "The path to reference '%s' collides with an existing one", new_ref); return -1; } - }); + } + git_sortedcache_unlock(backend->refcache); return 0; } @@ -839,12 +704,9 @@ static int loose_write(refdb_fs_backend *backend, const git_reference *ref) if (ref->type == GIT_REF_OID) { char oid[GIT_OID_HEXSZ + 1]; - - git_oid_fmt(oid, &ref->target.oid); - oid[GIT_OID_HEXSZ] = '\0'; + git_oid_nfmt(oid, sizeof(oid), &ref->target.oid); git_filebuf_printf(&file, "%s\n", oid); - } else if (ref->type == GIT_REF_SYMBOLIC) { git_filebuf_printf(&file, GIT_SYMREF "%s\n", ref->target.symbolic); } else { @@ -854,14 +716,6 @@ static int loose_write(refdb_fs_backend *backend, const git_reference *ref) return git_filebuf_commit(&file, GIT_REFS_FILE_MODE); } -static int packed_sort(const void *a, const void *b) -{ - const struct packref *ref_a = (const struct packref *)a; - const struct packref *ref_b = (const struct packref *)b; - - return strcmp(ref_a->name, ref_b->name); -} - /* * Find out what object this reference resolves to. * @@ -914,9 +768,7 @@ static int packed_find_peel(refdb_fs_backend *backend, struct packref *ref) static int packed_write_ref(struct packref *ref, git_filebuf *file) { char oid[GIT_OID_HEXSZ + 1]; - - git_oid_fmt(oid, &ref->oid); - oid[GIT_OID_HEXSZ] = 0; + git_oid_nfmt(oid, sizeof(oid), &ref->oid); /* * For references that peel to an object in the repo, we must @@ -930,8 +782,7 @@ static int packed_write_ref(struct packref *ref, git_filebuf *file) */ if (ref->flags & PACKREF_HAS_PEEL) { char peel[GIT_OID_HEXSZ + 1]; - git_oid_fmt(peel, &ref->peel); - peel[GIT_OID_HEXSZ] = 0; + git_oid_nfmt(peel, sizeof(peel), &ref->peel); if (git_filebuf_printf(file, "%s %s\n^%s\n", oid, ref->name, peel) < 0) return -1; @@ -954,16 +805,16 @@ static int packed_write_ref(struct packref *ref, git_filebuf *file) * is well-written, because we are destructing references * here otherwise. */ -static int packed_remove_loose( - refdb_fs_backend *backend, - git_vector *packing_list) +static int packed_remove_loose(refdb_fs_backend *backend) { size_t i; git_buf full_path = GIT_BUF_INIT; int failed = 0; - for (i = 0; i < packing_list->length; ++i) { - struct packref *ref = git_vector_get(packing_list, i); + /* backend->refcache is already locked when this is called */ + + for (i = 0; i < git_sortedcache_entrycount(backend->refcache); ++i) { + struct packref *ref = git_sortedcache_entry(backend->refcache, i); if ((ref->flags & PACKREF_WAS_LOOSE) == 0) continue; @@ -971,14 +822,13 @@ static int packed_remove_loose( if (git_buf_joinpath(&full_path, backend->path, ref->name) < 0) return -1; /* critical; do not try to recover on oom */ - if (git_path_exists(full_path.ptr) == true && p_unlink(full_path.ptr) < 0) { + if (git_path_exists(full_path.ptr) && p_unlink(full_path.ptr) < 0) { if (failed) continue; giterr_set(GITERR_REFERENCE, "Failed to remove loose reference '%s' after packing: %s", full_path.ptr, strerror(errno)); - failed = 1; } @@ -1002,33 +852,14 @@ static int packed_write(refdb_fs_backend *backend) git_filebuf pack_file = GIT_FILEBUF_INIT; size_t i; git_buf pack_file_path = GIT_BUF_INIT; - git_vector packing_list; - unsigned int total_refs; - assert(backend && backend->refcache.packfile); - - total_refs = - (unsigned int)git_strmap_num_entries(backend->refcache.packfile); - - if (git_vector_init(&packing_list, total_refs, packed_sort) < 0) + /* lock the cache to updates while we do this */ + if (git_sortedcache_lock(backend->refcache) < 0) return -1; - /* Load all the packfile into a vector */ - { - struct packref *reference; - - /* cannot fail: vector already has the right size */ - git_strmap_foreach_value(backend->refcache.packfile, reference, { - git_vector_insert(&packing_list, reference); - }); - } - - /* sort the vector so the entries appear sorted on the packfile */ - git_vector_sort(&packing_list); - - /* Now we can open the file! */ - if (git_buf_joinpath(&pack_file_path, - backend->path, GIT_PACKEDREFS_FILE) < 0) + /* Open the file! */ + if (git_buf_joinpath( + &pack_file_path, backend->path, GIT_PACKEDREFS_FILE) < 0) goto cleanup_memory; if (git_filebuf_open(&pack_file, pack_file_path.ptr, 0) < 0) @@ -1040,8 +871,9 @@ static int packed_write(refdb_fs_backend *backend) if (git_filebuf_printf(&pack_file, "%s\n", GIT_PACKEDREFS_HEADER) < 0) goto cleanup_packfile; - for (i = 0; i < packing_list.length; ++i) { - struct packref *ref = (struct packref *)git_vector_get(&packing_list, i); + for (i = 0; i < git_sortedcache_entrycount(backend->refcache); ++i) { + struct packref *ref = + git_sortedcache_entry(backend->refcache, i); if (packed_find_peel(backend, ref) < 0) goto cleanup_packfile; @@ -1057,16 +889,16 @@ static int packed_write(refdb_fs_backend *backend) /* when and only when the packfile has been properly written, * we can go ahead and remove the loose refs */ - if (packed_remove_loose(backend, &packing_list) < 0) - goto cleanup_memory; + if (packed_remove_loose(backend) < 0) + goto cleanup_memory; - { - struct stat st; - if (p_stat(pack_file_path.ptr, &st) == 0) - backend->refcache.packfile_time = st.st_mtime; - } + git_sortedcache_unlock(backend->refcache); + + /* update filestamp to latest value */ + if (git_futils_filestamp_check( + &backend->refcache->stamp, pack_file_path.ptr) < 0) + giterr_clear(); - git_vector_free(&packing_list); git_buf_free(&pack_file_path); /* we're good now */ @@ -1076,7 +908,7 @@ cleanup_packfile: git_filebuf_cleanup(&pack_file); cleanup_memory: - git_vector_free(&packing_list); + git_sortedcache_unlock(backend->refcache); git_buf_free(&pack_file_path); return -1; @@ -1087,11 +919,10 @@ static int refdb_fs_backend__write( const git_reference *ref, int force) { - refdb_fs_backend *backend; + refdb_fs_backend *backend = (refdb_fs_backend *)_backend; int error; - assert(_backend); - backend = (refdb_fs_backend *)_backend; + assert(backend); error = reference_path_available(backend, ref->name, NULL, force); if (error < 0) @@ -1104,17 +935,13 @@ static int refdb_fs_backend__delete( git_refdb_backend *_backend, const char *ref_name) { - refdb_fs_backend *backend; + refdb_fs_backend *backend = (refdb_fs_backend *)_backend; git_buf loose_path = GIT_BUF_INIT; - struct packref *pack_ref; - khiter_t pack_ref_pos; + size_t pack_pos; int error = 0; bool loose_deleted = 0; - assert(_backend); - assert(ref_name); - - backend = (refdb_fs_backend *)_backend; + assert(backend && ref_name); /* If a loose reference exists, remove it from the filesystem */ if (git_buf_joinpath(&loose_path, backend->path, ref_name) < 0) @@ -1130,19 +957,23 @@ static int refdb_fs_backend__delete( if (error != 0) return error; + if (packed_reload(backend) < 0) + return -1; + /* If a packed reference exists, remove it from the packfile and repack */ - error = packed_map_entry(&pack_ref, &pack_ref_pos, backend, ref_name); + if (git_sortedcache_lock(backend->refcache) < 0) + return -1; + + if (!(error = git_sortedcache_lookup_index( + &pack_pos, backend->refcache, ref_name))) + error = git_sortedcache_remove(backend->refcache, pack_pos, false); + + git_sortedcache_unlock(backend->refcache); if (error == GIT_ENOTFOUND) - return loose_deleted ? 0 : GIT_ENOTFOUND; + return loose_deleted ? 0 : ref_error_notfound(ref_name); - if (error == 0) { - git_strmap_delete_at(backend->refcache.packfile, pack_ref_pos); - git__free(pack_ref); - error = packed_write(backend); - } - - return error; + return packed_write(backend); } static int refdb_fs_backend__rename( @@ -1152,53 +983,44 @@ static int refdb_fs_backend__rename( const char *new_name, int force) { - refdb_fs_backend *backend; + refdb_fs_backend *backend = (refdb_fs_backend *)_backend; git_reference *old, *new; int error; - assert(_backend); - backend = (refdb_fs_backend *)_backend; + assert(backend); - error = reference_path_available(backend, new_name, old_name, force); - if (error < 0) + if ((error = reference_path_available( + backend, new_name, old_name, force)) < 0 || + (error = refdb_fs_backend__lookup(&old, _backend, old_name)) < 0) return error; - error = refdb_fs_backend__lookup(&old, _backend, old_name); - if (error < 0) - return error; - - error = refdb_fs_backend__delete(_backend, old_name); - if (error < 0) { + if ((error = refdb_fs_backend__delete(_backend, old_name)) < 0) { git_reference_free(old); return error; } - new = realloc(old, sizeof(git_reference) + strlen(new_name) + 1); - memcpy(new->name, new_name, strlen(new_name) + 1); + new = git_reference__set_name(old, new_name); + if (!new) { + git_reference_free(old); + return -1; + } - error = loose_write(backend, new); - if (error < 0) { + if ((error = loose_write(backend, new)) < 0 || out == NULL) { git_reference_free(new); return error; } - if (out) { - *out = new; - } else { - git_reference_free(new); - } - + *out = new; return 0; } static int refdb_fs_backend__compress(git_refdb_backend *_backend) { - refdb_fs_backend *backend; + refdb_fs_backend *backend = (refdb_fs_backend *)_backend; - assert(_backend); - backend = (refdb_fs_backend *)_backend; + assert(backend); - if (packed_load(backend) < 0 || /* load the existing packfile */ + if (packed_reload(backend) < 0 || /* load the existing packfile */ packed_loadloose(backend) < 0 || /* add all the loose refs */ packed_write(backend) < 0) /* write back to disk */ return -1; @@ -1206,29 +1028,13 @@ static int refdb_fs_backend__compress(git_refdb_backend *_backend) return 0; } -static void refcache_free(git_refcache *refs) -{ - assert(refs); - - if (refs->packfile) { - struct packref *reference; - - git_strmap_foreach_value(refs->packfile, reference, { - git__free(reference); - }); - - git_strmap_free(refs->packfile); - } -} - static void refdb_fs_backend__free(git_refdb_backend *_backend) { - refdb_fs_backend *backend; + refdb_fs_backend *backend = (refdb_fs_backend *)_backend; - assert(_backend); - backend = (refdb_fs_backend *)_backend; + assert(backend); - refcache_free(&backend->refcache); + git_sortedcache_free(backend->refcache); git__free(backend->path); git__free(backend); } @@ -1252,7 +1058,7 @@ static int setup_namespace(git_buf *path, git_repository *repo) if (parts == NULL) return -1; - /** + /* * From `man gitnamespaces`: * namespaces which include a / will expand to a hierarchy * of namespaces; for example, GIT_NAMESPACE=foo/bar will store @@ -1269,7 +1075,7 @@ static int setup_namespace(git_buf *path, git_repository *repo) if (git_futils_mkdir_r(git_buf_cstr(path), repo->path_repository, 0777) < 0) return -1; - /* Return the root of the namespaced path, i.e. without the trailing '/refs' */ + /* Return root of the namespaced path, i.e. without the trailing '/refs' */ git_buf_rtruncate_at_char(path, '/'); return 0; } @@ -1286,13 +1092,19 @@ int git_refdb_backend_fs( backend->repo = repository; - if (setup_namespace(&path, repository) < 0) { - git__free(backend); - return -1; - } + if (setup_namespace(&path, repository) < 0) + goto fail; backend->path = git_buf_detach(&path); + if (git_buf_joinpath(&path, backend->path, GIT_PACKEDREFS_FILE) < 0 || + git_sortedcache_new( + &backend->refcache, offsetof(struct packref, name), + NULL, NULL, packref_cmp, git_buf_cstr(&path)) < 0) + goto fail; + + git_buf_free(&path); + backend->parent.exists = &refdb_fs_backend__exists; backend->parent.lookup = &refdb_fs_backend__lookup; backend->parent.iterator = &refdb_fs_backend__iterator; @@ -1304,4 +1116,10 @@ int git_refdb_backend_fs( *backend_out = (git_refdb_backend *)backend; return 0; + +fail: + git_buf_free(&path); + git__free(backend->path); + git__free(backend); + return -1; } diff --git a/tests-clar/threads/refdb.c b/tests-clar/threads/refdb.c new file mode 100644 index 000000000..11e90a458 --- /dev/null +++ b/tests-clar/threads/refdb.c @@ -0,0 +1,200 @@ +#include "clar_libgit2.h" +#include "git2/refdb.h" +#include "refdb.h" + +static git_repository *g_repo; +static int g_expected = 0; + +void test_threads_refdb__initialize(void) +{ + g_repo = NULL; +} + +void test_threads_refdb__cleanup(void) +{ + cl_git_sandbox_cleanup(); + g_repo = NULL; +} + +#define REPEAT 20 +#define THREADS 20 + +static void *iterate_refs(void *arg) +{ + git_reference_iterator *i; + git_reference *ref; + int count = 0, *id = arg; + + usleep(THREADS - *id); + + cl_git_pass(git_reference_iterator_new(&i, g_repo)); + + for (count = 0; !git_reference_next(&ref, i); ++count) { + cl_assert(ref != NULL); + git_reference_free(ref); + } + + if (g_expected > 0) + cl_assert_equal_i(g_expected, count); + + git_reference_iterator_free(i); + + return arg; +} + +void test_threads_refdb__iterator(void) +{ + int r, t; + git_thread th[THREADS]; + int id[THREADS]; + git_oid head; + git_reference *ref; + char name[128]; + git_refdb *refdb; + + g_repo = cl_git_sandbox_init("testrepo2"); + + cl_git_pass(git_reference_name_to_id(&head, g_repo, "HEAD")); + + /* make a bunch of references */ + + for (r = 0; r < 200; ++r) { + snprintf(name, sizeof(name), "refs/heads/direct-%03d", r); + cl_git_pass(git_reference_create(&ref, g_repo, name, &head, 0)); + git_reference_free(ref); + } + + cl_git_pass(git_repository_refdb(&refdb, g_repo)); + cl_git_pass(git_refdb_compress(refdb)); + git_refdb_free(refdb); + + g_expected = 206; + + for (r = 0; r < REPEAT; ++r) { + g_repo = cl_git_sandbox_reopen(); /* reopen to flush caches */ + + for (t = 0; t < THREADS; ++t) { + id[t] = t; + cl_git_pass(git_thread_create(&th[t], NULL, iterate_refs, &id[t])); + } + + for (t = 0; t < THREADS; ++t) { + cl_git_pass(git_thread_join(th[t], NULL)); + } + + memset(th, 0, sizeof(th)); + } +} + +static void *create_refs(void *arg) +{ + int *id = arg, i; + git_oid head; + char name[128]; + git_reference *ref[10]; + + cl_git_pass(git_reference_name_to_id(&head, g_repo, "HEAD")); + + for (i = 0; i < 10; ++i) { + snprintf(name, sizeof(name), "refs/heads/thread-%03d-%02d", *id, i); + cl_git_pass(git_reference_create(&ref[i], g_repo, name, &head, 0)); + + if (i == 5) { + git_refdb *refdb; + cl_git_pass(git_repository_refdb(&refdb, g_repo)); + cl_git_pass(git_refdb_compress(refdb)); + git_refdb_free(refdb); + } + } + + for (i = 0; i < 10; ++i) + git_reference_free(ref[i]); + + return arg; +} + +static void *delete_refs(void *arg) +{ + int *id = arg, i; + git_reference *ref; + char name[128]; + + for (i = 0; i < 10; ++i) { + snprintf( + name, sizeof(name), "refs/heads/thread-%03d-%02d", (*id) & ~0x3, i); + + if (!git_reference_lookup(&ref, g_repo, name)) { + fprintf(stderr, "deleting %s\n", name); + cl_git_pass(git_reference_delete(ref)); + git_reference_delete(ref); + } + + if (i == 5) { + git_refdb *refdb; + cl_git_pass(git_repository_refdb(&refdb, g_repo)); + cl_git_pass(git_refdb_compress(refdb)); + git_refdb_free(refdb); + } + } + + return arg; +} + +void test_threads_refdb__edit_while_iterate(void) +{ + int r, t; + git_thread th[THREADS]; + int id[THREADS]; + git_oid head; + git_reference *ref; + char name[128]; + git_refdb *refdb; + + g_repo = cl_git_sandbox_init("testrepo2"); + + cl_git_pass(git_reference_name_to_id(&head, g_repo, "HEAD")); + + /* make a bunch of references */ + + for (r = 0; r < 50; ++r) { + snprintf(name, sizeof(name), "refs/heads/starter-%03d", r); + cl_git_pass(git_reference_create(&ref, g_repo, name, &head, 0)); + git_reference_free(ref); + } + + cl_git_pass(git_repository_refdb(&refdb, g_repo)); + cl_git_pass(git_refdb_compress(refdb)); + git_refdb_free(refdb); + + g_expected = -1; + + g_repo = cl_git_sandbox_reopen(); /* reopen to flush caches */ + + for (t = 0; t < THREADS; ++t) { + void *(*fn)(void *arg); + + switch (t & 0x3) { + case 0: fn = create_refs; break; + case 1: fn = delete_refs; break; + default: fn = iterate_refs; break; + } + + id[t] = t; + cl_git_pass(git_thread_create(&th[t], NULL, fn, &id[t])); + } + + for (t = 0; t < THREADS; ++t) { + cl_git_pass(git_thread_join(th[t], NULL)); + } + + memset(th, 0, sizeof(th)); + + for (t = 0; t < THREADS; ++t) { + id[t] = t; + cl_git_pass(git_thread_create(&th[t], NULL, iterate_refs, &id[t])); + } + + for (t = 0; t < THREADS; ++t) { + cl_git_pass(git_thread_join(th[t], NULL)); + } +}