From 4a863c06662053a8530a0dcb24e0a2daa33e05cf Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Thu, 3 Jan 2013 20:36:26 +0100 Subject: [PATCH 1/3] Sane refresh logic All the ODB backends have a specific refresh interface. When reading an object, first we attempt every single backend: if the read fails, then we refresh all the backends and retry the read one more time to see if the object has appeared. --- include/git2/odb.h | 20 ++++++++++ include/git2/odb_backend.h | 2 + src/odb.c | 55 +++++++++++++++++++++++++--- src/odb_pack.c | 75 ++++++++++++++++---------------------- 4 files changed, 103 insertions(+), 49 deletions(-) diff --git a/include/git2/odb.h b/include/git2/odb.h index f39e7b541..8fd1a95be 100644 --- a/include/git2/odb.h +++ b/include/git2/odb.h @@ -191,6 +191,26 @@ GIT_EXTERN(int) git_odb_read_header(size_t *len_out, git_otype *type_out, git_od */ GIT_EXTERN(int) git_odb_exists(git_odb *db, const git_oid *id); +/** + * Refresh the object database to load newly added files. + * + * If the object databases have changed on disk while the library + * is running, this function will force a reload of the underlying + * indexes. + * + * Use this function when you're confident that an external + * application has tampered with the ODB. + * + * NOTE that it is not necessary to call this function at all. The + * library will automatically attempt to refresh the ODB + * when a lookup fails, to see if the looked up object exists + * on disk but hasn't been loaded yet. + * + * @param db database to refresh + * @return 0 on success, error code otherwise + */ +GIT_EXTERN(int) git_odb_refresh(struct git_odb *db); + /** * List all objects available in the database * diff --git a/include/git2/odb_backend.h b/include/git2/odb_backend.h index 029c61b9f..dbc3981f6 100644 --- a/include/git2/odb_backend.h +++ b/include/git2/odb_backend.h @@ -89,6 +89,8 @@ struct git_odb_backend { struct git_odb_backend *, const git_oid *); + int (* refresh)(struct git_odb_backend *); + int (* foreach)( struct git_odb_backend *, git_odb_foreach_cb cb, diff --git a/src/odb.c b/src/odb.c index 216715afa..d0b23230c 100644 --- a/src/odb.c +++ b/src/odb.c @@ -609,14 +609,22 @@ int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id) { unsigned int i; int error = GIT_ENOTFOUND; + bool refreshed = false; git_rawobj raw; assert(out && db && id); + if (db->backends.length == 0) { + giterr_set(GITERR_ODB, "Failed to lookup object: no backends loaded"); + return GIT_ENOTFOUND; + } + *out = git_cache_get(&db->cache, id); if (*out != NULL) return 0; +attempt_lookup: + for (i = 0; i < db->backends.length && error < 0; ++i) { backend_internal *internal = git_vector_get(&db->backends, i); git_odb_backend *b = internal->backend; @@ -625,9 +633,13 @@ int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id) error = b->read(&raw.data, &raw.len, &raw.type, b, id); } - /* TODO: If no backends are configured, this returns GIT_ENOTFOUND but - * will never have called giterr_set(). - */ + if (error == GIT_ENOTFOUND && !refreshed) { + if ((error = git_odb_refresh(db)) < 0) + return error; + + refreshed = true; + goto attempt_lookup; + } if (error && error != GIT_PASSTHROUGH) return error; @@ -644,7 +656,7 @@ int git_odb_read_prefix( git_oid found_full_oid = {{0}}; git_rawobj raw; void *data = NULL; - bool found = false; + bool found = false, refreshed = false; assert(out && db); @@ -660,11 +672,13 @@ int git_odb_read_prefix( return 0; } +attempt_lookup: + for (i = 0; i < db->backends.length; ++i) { backend_internal *internal = git_vector_get(&db->backends, i); git_odb_backend *b = internal->backend; - if (b->read != NULL) { + if (b->read_prefix != NULL) { git_oid full_oid; error = b->read_prefix(&full_oid, &raw.data, &raw.len, &raw.type, b, short_id, len); if (error == GIT_ENOTFOUND || error == GIT_PASSTHROUGH) @@ -675,13 +689,23 @@ int git_odb_read_prefix( git__free(data); data = raw.data; + if (found && git_oid_cmp(&full_oid, &found_full_oid)) return git_odb__error_ambiguous("multiple matches for prefix"); + found_full_oid = full_oid; found = true; } } + if (!found && !refreshed) { + if ((error = git_odb_refresh(db)) < 0) + return error; + + refreshed = true; + goto attempt_lookup; + } + if (!found) return git_odb__error_notfound("no match for prefix", short_id); @@ -820,12 +844,31 @@ int git_odb_write_pack(struct git_odb_writepack **out, git_odb *db, git_transfer return error; } -void * git_odb_backend_malloc(git_odb_backend *backend, size_t len) +void *git_odb_backend_malloc(git_odb_backend *backend, size_t len) { GIT_UNUSED(backend); return git__malloc(len); } +int git_odb_refresh(struct git_odb *db) +{ + unsigned int i; + assert(db); + + for (i = 0; i < db->backends.length; ++i) { + backend_internal *internal = git_vector_get(&db->backends, i); + git_odb_backend *b = internal->backend; + + if (b->refresh != NULL) { + int error = b->refresh(b); + if (error < 0) + return error; + } + } + + return 0; +} + int git_odb__error_notfound(const char *message, const git_oid *oid) { if (oid != NULL) { diff --git a/src/odb_pack.c b/src/odb_pack.c index 0cdf552a0..e1b44d9ca 100644 --- a/src/odb_pack.c +++ b/src/odb_pack.c @@ -138,7 +138,6 @@ static int pack_window_contains(git_mwindow *win, off_t offset); static int packfile_sort__cb(const void *a_, const void *b_); static int packfile_load__cb(void *_data, git_buf *path); -static int packfile_refresh_all(struct pack_backend *backend); static int pack_entry_find(struct git_pack_entry *e, struct pack_backend *backend, const git_oid *oid); @@ -237,33 +236,6 @@ static int packfile_load__cb(void *_data, git_buf *path) return git_vector_insert(&backend->packs, pack); } -static int packfile_refresh_all(struct pack_backend *backend) -{ - int error; - struct stat st; - git_buf path = GIT_BUF_INIT; - - if (backend->pack_folder == NULL) - return 0; - - if (p_stat(backend->pack_folder, &st) < 0 || !S_ISDIR(st.st_mode)) - return git_odb__error_notfound("failed to refresh packfiles", NULL); - - git_buf_sets(&path, backend->pack_folder); - - /* reload all packs */ - error = git_path_direach(&path, packfile_load__cb, (void *)backend); - - git_buf_free(&path); - - if (error < 0) - return error; - - git_vector_sort(&backend->packs); - - return 0; -} - static int pack_entry_find_inner( struct git_pack_entry *e, struct pack_backend *backend, @@ -294,17 +266,12 @@ static int pack_entry_find_inner( static int pack_entry_find(struct git_pack_entry *e, struct pack_backend *backend, const git_oid *oid) { - int error; struct git_pack_file *last_found = backend->last_found; if (backend->last_found && git_pack_entry_find(e, backend->last_found, oid, GIT_OID_HEXSZ) == 0) return 0; - if (!pack_entry_find_inner(e, backend, oid, last_found)) - return 0; - if ((error = packfile_refresh_all(backend)) < 0) - return error; if (!pack_entry_find_inner(e, backend, oid, last_found)) return 0; @@ -356,17 +323,9 @@ static int pack_entry_find_prefix( const git_oid *short_oid, size_t len) { - unsigned found = 0; - int error; struct git_pack_file *last_found = backend->last_found; + unsigned int found = pack_entry_find_prefix_inner(e, backend, short_oid, len, last_found); - if ((found = pack_entry_find_prefix_inner(e, backend, short_oid, len, last_found)) > 0) - goto cleanup; - if ((error = packfile_refresh_all(backend)) < 0) - return error; - found = pack_entry_find_prefix_inner(e, backend, short_oid, len, last_found); - -cleanup: if (!found) return git_odb__error_notfound("no matching pack entry for prefix", short_oid); else if (found > 1) @@ -383,6 +342,34 @@ cleanup: * Implement the git_odb_backend API calls * ***********************************************************/ +static int pack_backend__refresh(git_odb_backend *_backend) +{ + struct pack_backend *backend = (struct pack_backend *)_backend; + + int error; + struct stat st; + git_buf path = GIT_BUF_INIT; + + if (backend->pack_folder == NULL) + return 0; + + if (p_stat(backend->pack_folder, &st) < 0 || !S_ISDIR(st.st_mode)) + return git_odb__error_notfound("failed to refresh packfiles", NULL); + + git_buf_sets(&path, backend->pack_folder); + + /* reload all packs */ + error = git_path_direach(&path, packfile_load__cb, (void *)backend); + + git_buf_free(&path); + + if (error < 0) + return error; + + git_vector_sort(&backend->packs); + return 0; +} + static int pack_backend__read_header(size_t *len_p, git_otype *type_p, struct git_odb_backend *backend, const git_oid *oid) { @@ -468,7 +455,7 @@ static int pack_backend__foreach(git_odb_backend *_backend, git_odb_foreach_cb c backend = (struct pack_backend *)_backend; /* Make sure we know about the packfiles */ - if ((error = packfile_refresh_all(backend)) < 0) + if ((error = pack_backend__refresh(_backend)) < 0) return error; git_vector_foreach(&backend->packs, i, p) { @@ -581,6 +568,7 @@ int git_odb_backend_one_pack(git_odb_backend **backend_out, const char *idx) backend->parent.read_prefix = &pack_backend__read_prefix; backend->parent.read_header = &pack_backend__read_header; backend->parent.exists = &pack_backend__exists; + backend->parent.refresh = &pack_backend__refresh; backend->parent.foreach = &pack_backend__foreach; backend->parent.free = &pack_backend__free; @@ -619,6 +607,7 @@ int git_odb_backend_pack(git_odb_backend **backend_out, const char *objects_dir) backend->parent.read_prefix = &pack_backend__read_prefix; backend->parent.read_header = &pack_backend__read_header; backend->parent.exists = &pack_backend__exists; + backend->parent.refresh = &pack_backend__refresh; backend->parent.foreach = &pack_backend__foreach; backend->parent.writepack = &pack_backend__writepack; backend->parent.free = &pack_backend__free; From 891a4681ebba330f9ef33c609c5435e580ca1551 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Fri, 4 Jan 2013 17:42:41 +0100 Subject: [PATCH 2/3] dat errorcode --- src/odb.c | 3 ++- src/odb_pack.c | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/odb.c b/src/odb.c index d0b23230c..a0d30630a 100644 --- a/src/odb.c +++ b/src/odb.c @@ -608,7 +608,7 @@ int git_odb__read_header_or_object( int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id) { unsigned int i; - int error = GIT_ENOTFOUND; + int error; bool refreshed = false; git_rawobj raw; @@ -624,6 +624,7 @@ int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id) return 0; attempt_lookup: + error = GIT_ENOTFOUND; for (i = 0; i < db->backends.length && error < 0; ++i) { backend_internal *internal = git_vector_get(&db->backends, i); diff --git a/src/odb_pack.c b/src/odb_pack.c index e1b44d9ca..9d0c4c0e7 100644 --- a/src/odb_pack.c +++ b/src/odb_pack.c @@ -600,7 +600,12 @@ int git_odb_backend_pack(git_odb_backend **backend_out, const char *objects_dir) } if (git_path_isdir(git_buf_cstr(&path)) == true) { + int error; + backend->pack_folder = git_buf_detach(&path); + error = pack_backend__refresh((git_odb_backend *)backend); + if (error < 0) + return error; } backend->parent.read = &pack_backend__read; From 8fe6bc5c47640537a20c351adc5ccaa6a212312b Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Thu, 10 Jan 2013 15:43:08 +0100 Subject: [PATCH 3/3] odb: Refresh on `exists` query too --- src/odb.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/odb.c b/src/odb.c index a0d30630a..24381e70e 100644 --- a/src/odb.c +++ b/src/odb.c @@ -529,6 +529,7 @@ int git_odb_exists(git_odb *db, const git_oid *id) git_odb_object *object; unsigned int i; bool found = false; + bool refreshed = false; assert(db && id); @@ -537,6 +538,7 @@ int git_odb_exists(git_odb *db, const git_oid *id) return (int)true; } +attempt_lookup: for (i = 0; i < db->backends.length && !found; ++i) { backend_internal *internal = git_vector_get(&db->backends, i); git_odb_backend *b = internal->backend; @@ -545,6 +547,16 @@ int git_odb_exists(git_odb *db, const git_oid *id) found = b->exists(b, id); } + if (!found && !refreshed) { + if (git_odb_refresh(db) < 0) { + giterr_clear(); + return (int)false; + } + + refreshed = true; + goto attempt_lookup; + } + return (int)found; } @@ -674,7 +686,6 @@ int git_odb_read_prefix( } attempt_lookup: - for (i = 0; i < db->backends.length; ++i) { backend_internal *internal = git_vector_get(&db->backends, i); git_odb_backend *b = internal->backend;