Merge pull request #3656 from ethomson/exists_prefixes

Introduce `git_odb_expand_ids`

include/git2/odb.h

@@ -10,6 +10,7 @@
 #include "common.h"
 #include "types.h"
 #include "oid.h"
+#include "oidarray.h"
 
 /**
  * @file git2/odb.h
@@ -159,7 +160,8 @@ GIT_EXTERN(int) git_odb_read_header(size_t *len_out, git_otype *type_out, git_od
 GIT_EXTERN(int) git_odb_exists(git_odb *db, const git_oid *id);
 
 /**
- * Determine if objects can be found in the object database from a short OID.
+ * Determine if an object can be found in the object database by an
+ * abbreviated object ID.
  *
  * @param out The full OID of the found object if just one is found.
  * @param db The database to be searched for the given object.
@@ -171,6 +173,50 @@ GIT_EXTERN(int) git_odb_exists(git_odb *db, const git_oid *id);
 GIT_EXTERN(int) git_odb_exists_prefix(
 	git_oid *out, git_odb *db, const git_oid *short_id, size_t len);
 
+/**
+ * The information about object IDs to query in `git_odb_expand_ids`,
+ * which will be populated upon return.
+ */
+typedef struct git_odb_expand_id {
+	/** The object ID to expand */
+	git_oid id;
+
+	/**
+	 * The length of the object ID (in nibbles, or packets of 4 bits; the
+	 * number of hex characters)
+	 * */
+	unsigned short length;
+
+	/**
+	 * The (optional) type of the object to search for; leave as `0` or set
+	 * to `GIT_OBJ_ANY` to query for any object matching the ID.
+	 */
+	git_otype type;
+} git_odb_expand_id;
+
+/**
+ * Determine if one or more objects can be found in the object database
+ * by their abbreviated object ID and type.  The given array will be
+ * updated in place:  for each abbreviated ID that is unique in the
+ * database, and of the given type (if specified), the full object ID,
+ * object ID length (`GIT_OID_HEXSZ`) and type will be written back to
+ * the array.  For IDs that are not found (or are ambiguous), the
+ * array entry will be zeroed.
+ *
+ * Note that since this function operates on multiple objects, the
+ * underlying database will not be asked to be reloaded if an object is
+ * not found (which is unlike other object database operations.)
+ *
+ * @param db The database to be searched for the given objects.
+ * @param ids An array of short object IDs to search for
+ * @param count The length of the `ids` array
+ * @return 0 on success or an error code on failure
+ */
+GIT_EXTERN(int) git_odb_expand_ids(
+	git_odb *db,
+	git_odb_expand_id *ids,
+	size_t count);
+
 /**
  * Refresh the object database to load newly added files.
  *

src/odb.c

@@ -18,6 +18,7 @@
 
 #include "git2/odb_backend.h"
 #include "git2/oid.h"
+#include "git2/oidarray.h"
 
 #define GIT_ALTERNATES_FILE "info/alternates"
 
@@ -651,7 +652,7 @@ int git_odb_exists(git_odb *db, const git_oid *id)
 
 	if ((object = git_cache_get_raw(odb_cache(db), id)) != NULL) {
 		git_odb_object_free(object);
-		return (int)true;
+		return 1;
 	}
 
 	if (odb_exists_1(db, id, false))
@@ -716,23 +717,19 @@ int git_odb_exists_prefix(
 
 	if (len < GIT_OID_MINPREFIXLEN)
 		return git_odb__error_ambiguous("prefix length too short");
-	if (len > GIT_OID_HEXSZ)
-		len = GIT_OID_HEXSZ;
 
-	if (len == GIT_OID_HEXSZ) {
+	if (len >= GIT_OID_HEXSZ) {
 		if (git_odb_exists(db, short_id)) {
 			if (out)
 				git_oid_cpy(out, short_id);
 			return 0;
 		} else {
-			return git_odb__error_notfound("no match for id prefix", short_id);
+			return git_odb__error_notfound(
+				"no match for id prefix", short_id, len);
 		}
 	}
 
-	/* just copy valid part of short_id */
+	git_oid__cpy_prefix(&key, short_id, len);
-	memcpy(&key.id, short_id->id, (len + 1) / 2);
-	if (len & 1)
-		key.id[len / 2] &= 0xF0;
 
 	error = odb_exists_prefix_1(out, db, &key, len, false);
 
@@ -740,7 +737,63 @@ int git_odb_exists_prefix(
 		error = odb_exists_prefix_1(out, db, &key, len, true);
 
 	if (error == GIT_ENOTFOUND)
-		return git_odb__error_notfound("no match for id prefix", &key);
+		return git_odb__error_notfound("no match for id prefix", &key, len);
+
+	return error;
+}
+
+int git_odb_expand_ids(
+	git_odb *db,
+	git_odb_expand_id *ids,
+	size_t count)
+{
+	size_t len, i;
+	int error;
+
+	assert(db && ids);
+
+	for (i = 0; i < count; i++) {
+		git_odb_expand_id *query = &ids[i];
+		git_oid *actual_id = NULL, tmp;
+		git_otype query_type = (query->type == GIT_OBJ_ANY) ? 0 : query->type;
+		git_otype actual_type = 0;
+
+		/* if we were given a full object ID, simply look it up */
+		if (query->length >= GIT_OID_HEXSZ) {
+			error = git_odb_read_header(&len, &actual_type, db, &query->id);
+		}
+
+		/* otherwise, resolve the short id to full, then (optionally)
+		 * read the header.
+		 */
+		else if (query->length >= GIT_OID_MINPREFIXLEN) {
+	    	error = odb_exists_prefix_1(&tmp,
+				db, &query->id, query->length, false);
+
+			if (!error) {
+				actual_id = &tmp;
+				error = git_odb_read_header(&len, &actual_type, db, &tmp);
+			}
+		}
+
+		if (error < 0 && error != GIT_ENOTFOUND && error != GIT_EAMBIGUOUS)
+			break;
+
+		if (error == 0 && (query_type == actual_type || !query_type)) {
+			if (actual_id)
+				git_oid_cpy(&query->id, actual_id);
+
+			query->length = GIT_OID_HEXSZ;
+			query->type = actual_type;
+		} else {
+			memset(&query->id, 0, sizeof(git_oid));
+			query->length = 0;
+			query->type = 0;
+		}
+	}
+
+	if (!error)
+		giterr_clear();
 
 	return error;
 }
@@ -881,7 +934,7 @@ int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id)
 		error = odb_read_1(out, db, id, true);
 
 	if (error == GIT_ENOTFOUND)
-		return git_odb__error_notfound("no match for id", id);
+		return git_odb__error_notfound("no match for id", id, GIT_OID_HEXSZ);
 
 	return error;
 }
@@ -956,10 +1009,7 @@ int git_odb_read_prefix(
 			return 0;
 	}
 
-	/* just copy valid part of short_id */
+	git_oid__cpy_prefix(&key, short_id, len);
-	memcpy(&key.id, short_id->id, (len + 1) / 2);
-	if (len & 1)
-		key.id[len / 2] &= 0xF0;
 
 	error = read_prefix_1(out, db, &key, len, false);
 
@@ -967,7 +1017,7 @@ int git_odb_read_prefix(
 		error = read_prefix_1(out, db, &key, len, true);
 
 	if (error == GIT_ENOTFOUND)
-		return git_odb__error_notfound("no match for prefix", &key);
+		return git_odb__error_notfound("no match for prefix", &key, len);
 
 	return error;
 }
@@ -1223,12 +1273,14 @@ int git_odb_refresh(struct git_odb *db)
 	return 0;
 }
 
-int git_odb__error_notfound(const char *message, const git_oid *oid)
+int git_odb__error_notfound(
+	const char *message, const git_oid *oid, size_t oid_len)
 {
 	if (oid != NULL) {
 		char oid_str[GIT_OID_HEXSZ + 1];
-		git_oid_tostr(oid_str, sizeof(oid_str), oid);
+		git_oid_tostr(oid_str, oid_len, oid);
-		giterr_set(GITERR_ODB, "Object not found - %s (%s)", message, oid_str);
+		giterr_set(GITERR_ODB, "Object not found - %s (%.*s)",
+			message, oid_len, oid_str);
 	} else
 		giterr_set(GITERR_ODB, "Object not found - %s", message);
 

src/odb.h

@@ -82,7 +82,8 @@ int git_odb__hashlink(git_oid *out, const char *path);
 /*
  * Generate a GIT_ENOTFOUND error for the ODB.
  */
-int git_odb__error_notfound(const char *message, const git_oid *oid);
+int git_odb__error_notfound(
+	const char *message, const git_oid *oid, size_t oid_len);
 
 /*
  * Generate a GIT_EAMBIGUOUS error for the ODB.

src/odb_loose.c

@@ -547,7 +547,8 @@ static int locate_object_short_oid(
 
 	/* Check that directory exists */
 	if (git_path_isdir(object_location->ptr) == false)
-		return git_odb__error_notfound("no matching loose object for prefix", short_oid);
+		return git_odb__error_notfound("no matching loose object for prefix",
+			short_oid, len);
 
 	state.dir_len = git_buf_len(object_location);
 	state.short_oid_len = len;
@@ -560,7 +561,8 @@ static int locate_object_short_oid(
 		return error;
 
 	if (!state.found)
-		return git_odb__error_notfound("no matching loose object for prefix", short_oid);
+		return git_odb__error_notfound("no matching loose object for prefix",
+			short_oid, len);
 
 	if (state.found > 1)
 		return git_odb__error_ambiguous("multiple matches in loose objects");
@@ -613,9 +615,10 @@ static int loose_backend__read_header(size_t *len_p, git_otype *type_p, git_odb_
 	raw.len = 0;
 	raw.type = GIT_OBJ_BAD;
 
-	if (locate_object(&object_path, (loose_backend *)backend, oid) < 0)
+	if (locate_object(&object_path, (loose_backend *)backend, oid) < 0) {
-		error = git_odb__error_notfound("no matching loose object", oid);
+		error = git_odb__error_notfound("no matching loose object",
-	else if ((error = read_header_loose(&raw, &object_path)) == 0) {
+			oid, GIT_OID_HEXSZ);
+	} else if ((error = read_header_loose(&raw, &object_path)) == 0) {
 		*len_p = raw.len;
 		*type_p = raw.type;
 	}
@@ -633,9 +636,10 @@ static int loose_backend__read(void **buffer_p, size_t *len_p, git_otype *type_p
 
 	assert(backend && oid);
 
-	if (locate_object(&object_path, (loose_backend *)backend, oid) < 0)
+	if (locate_object(&object_path, (loose_backend *)backend, oid) < 0) {
-		error = git_odb__error_notfound("no matching loose object", oid);
+		error = git_odb__error_notfound("no matching loose object",
-	else if ((error = read_loose(&raw, &object_path)) == 0) {
+			oid, GIT_OID_HEXSZ);
+	} else if ((error = read_loose(&raw, &object_path)) == 0) {
 		*buffer_p = raw.data;
 		*len_p = raw.len;
 		*type_p = raw.type;

src/odb_pack.c

@@ -264,7 +264,8 @@ static int pack_entry_find(struct git_pack_entry *e, struct pack_backend *backen
 	if (!pack_entry_find_inner(e, backend, oid, last_found))
 		return 0;
 
-	return git_odb__error_notfound("failed to find pack entry", oid);
+	return git_odb__error_notfound(
+		"failed to find pack entry", oid, GIT_OID_HEXSZ);
 }
 
 static int pack_entry_find_prefix(
@@ -309,7 +310,8 @@ static int pack_entry_find_prefix(
 	}
 
 	if (!found)
-		return git_odb__error_notfound("no matching pack entry for prefix", short_oid);
+		return git_odb__error_notfound("no matching pack entry for prefix",
+			short_oid, len);
 	else
 		return 0;
 }
@@ -333,7 +335,7 @@ static int pack_backend__refresh(git_odb_backend *backend_)
 		return 0;
 
 	if (p_stat(backend->pack_folder, &st) < 0 || !S_ISDIR(st.st_mode))
-		return git_odb__error_notfound("failed to refresh packfiles", NULL);
+		return git_odb__error_notfound("failed to refresh packfiles", NULL, 0);
 
 	git_buf_sets(&path, backend->pack_folder);
 

src/oid.h

@@ -44,4 +44,13 @@ GIT_INLINE(int) git_oid__cmp(const git_oid *a, const git_oid *b)
 	return git_oid__hashcmp(a->id, b->id);
 }
 
+GIT_INLINE(void) git_oid__cpy_prefix(
+	git_oid *out, const git_oid *id, size_t len)
+{
+	memcpy(&out->id, id->id, (len + 1) / 2);
+
+	if (len & 1)
+		out->id[len / 2] &= 0xF0;
+}
+
 #endif

src/pack.c

@@ -1018,7 +1018,7 @@ static int packfile_open(struct git_pack_file *p)
 	unsigned char *idx_sha1;
 
 	if (p->index_version == -1 && pack_index_open(p) < 0)
-		return git_odb__error_notfound("failed to open packfile", NULL);
+		return git_odb__error_notfound("failed to open packfile", NULL, 0);
 
 	/* if mwf opened by another thread, return now */
 	if (git_mutex_lock(&p->lock) < 0)
@@ -1099,7 +1099,7 @@ int git_packfile__name(char **out, const char *path)
 	path_len = strlen(path);
 
 	if (path_len < strlen(".idx"))
-		return git_odb__error_notfound("invalid packfile path", NULL);
+		return git_odb__error_notfound("invalid packfile path", NULL, 0);
 
 	if (git_buf_printf(&buf, "%.*s.pack", (int)(path_len - strlen(".idx")), path) < 0)
 		return -1;
@@ -1117,7 +1117,7 @@ int git_packfile_alloc(struct git_pack_file **pack_out, const char *path)
 	*pack_out = NULL;
 
 	if (path_len < strlen(".idx"))
-		return git_odb__error_notfound("invalid packfile path", NULL);
+		return git_odb__error_notfound("invalid packfile path", NULL, 0);
 
 	GITERR_CHECK_ALLOC_ADD(&alloc_len, sizeof(*p), path_len);
 	GITERR_CHECK_ALLOC_ADD(&alloc_len, alloc_len, 2);
@@ -1143,7 +1143,7 @@ int git_packfile_alloc(struct git_pack_file **pack_out, const char *path)
 
 	if (p_stat(p->pack_name, &st) < 0 || !S_ISREG(st.st_mode)) {
 		git__free(p);
-		return git_odb__error_notfound("packfile not found", NULL);
+		return git_odb__error_notfound("packfile not found", NULL, 0);
 	}
 
 	/* ok, it looks sane as far as we can check without
@@ -1344,7 +1344,7 @@ static int pack_entry_find_offset(
 	}
 
 	if (!found)
-		return git_odb__error_notfound("failed to find offset for pack entry", short_oid);
+		return git_odb__error_notfound("failed to find offset for pack entry", short_oid, len);
 	if (found > 1)
 		return git_odb__error_ambiguous("found multiple offsets for pack entry");
 

tests/odb/mixed.c

@@ -108,3 +108,149 @@ void test_odb_mixed__dup_oid_prefix_0(void) {
 	cl_git_pass(git_odb_read_prefix(&obj, _odb, &oid, strlen(hex)));
 	git_odb_object_free(obj);
 }
+
+struct expand_id_test_data {
+	char *lookup_id;
+	char *expected_id;
+	git_otype expected_type;
+};
+
+struct expand_id_test_data expand_id_test_data[] = {
+	/* some prefixes and their expected values */
+	{ "dea509d0",  NULL, GIT_OBJ_ANY },
+	{ "00000000",  NULL, GIT_OBJ_ANY },
+	{ "dea509d0",  NULL, GIT_OBJ_ANY },
+	{ "dea509d09", "dea509d097ce692e167dfc6a48a7a280cc5e877e", GIT_OBJ_BLOB },
+	{ "dea509d0b", "dea509d0b3cb8ee0650f6ca210bc83f4678851ba", GIT_OBJ_BLOB },
+	{ "ce0136250", "ce013625030ba8dba906f756967f9e9ca394464a", GIT_OBJ_BLOB },
+	{ "0ddeaded",  NULL, GIT_OBJ_ANY },
+	{ "4d5979b",   "4d5979b468252190cb572ae758aca36928e8a91e", GIT_OBJ_TREE },
+	{ "0ddeaded",  NULL, GIT_OBJ_ANY },
+	{ "0ddeadede", "0ddeadede9e6d6ccddce0ee1e5749eed0485e5ea", GIT_OBJ_BLOB },
+	{ "0ddeaded9", "0ddeaded9502971eefe1e41e34d0e536853ae20f", GIT_OBJ_BLOB },
+	{ "f00b4e",    NULL, GIT_OBJ_ANY },
+
+	/* some full-length object ids */
+	{ "0000000000000000000000000000000000000000", NULL, GIT_OBJ_ANY },
+	{
+	  "dea509d097ce692e167dfc6a48a7a280cc5e877e",
+	  "dea509d097ce692e167dfc6a48a7a280cc5e877e",
+	  GIT_OBJ_BLOB
+	},
+	{ "f00f00f00f00f00f00f00f00f00f00f00f00f00f", NULL, GIT_OBJ_ANY },
+	{
+	  "4d5979b468252190cb572ae758aca36928e8a91e",
+	  "4d5979b468252190cb572ae758aca36928e8a91e",
+	  GIT_OBJ_TREE
+	},
+};
+
+static void setup_prefix_query(
+	git_odb_expand_id **out_ids,
+	size_t *out_num)
+{
+	git_odb_expand_id *ids;
+	size_t num, i;
+
+	num = ARRAY_SIZE(expand_id_test_data);
+
+	cl_assert((ids = git__calloc(num, sizeof(git_odb_expand_id))));
+
+	for (i = 0; i < num; i++) {
+		git_odb_expand_id *id = &ids[i];
+
+		size_t len = strlen(expand_id_test_data[i].lookup_id);
+
+		git_oid_fromstrn(&id->id, expand_id_test_data[i].lookup_id, len);
+		id->length = (unsigned short)len;
+		id->type = expand_id_test_data[i].expected_type;
+	}
+
+	*out_ids = ids;
+	*out_num = num;
+}
+
+static void assert_found_objects(git_odb_expand_id *ids)
+{
+	size_t num, i;
+
+	num = ARRAY_SIZE(expand_id_test_data);
+
+	for (i = 0; i < num; i++) {
+		git_oid expected_id = {{0}};
+		size_t expected_len = 0;
+		git_otype expected_type = 0;
+
+		if (expand_id_test_data[i].expected_id) {
+			git_oid_fromstr(&expected_id, expand_id_test_data[i].expected_id);
+			expected_len = GIT_OID_HEXSZ;
+			expected_type = expand_id_test_data[i].expected_type;
+		}
+
+		cl_assert_equal_oid(&expected_id, &ids[i].id);
+		cl_assert_equal_i(expected_len, ids[i].length);
+		cl_assert_equal_i(expected_type, ids[i].type);
+	}
+}
+
+static void assert_notfound_objects(git_odb_expand_id *ids)
+{
+	git_oid expected_id = {{0}};
+	size_t num, i;
+
+	num = ARRAY_SIZE(expand_id_test_data);
+
+	for (i = 0; i < num; i++) {
+		cl_assert_equal_oid(&expected_id, &ids[i].id);
+		cl_assert_equal_i(0, ids[i].length);
+		cl_assert_equal_i(0, ids[i].type);
+	}
+}
+
+void test_odb_mixed__expand_ids(void)
+{
+	git_odb_expand_id *ids;
+	size_t i, num;
+
+	/* test looking for the actual (correct) types */
+
+	setup_prefix_query(&ids, &num);
+	cl_git_pass(git_odb_expand_ids(_odb, ids, num));
+	assert_found_objects(ids);
+	git__free(ids);
+
+	/* test looking for an explicit `type == 0` */
+
+	setup_prefix_query(&ids, &num);
+
+	for (i = 0; i < num; i++)
+		ids[i].type = 0;
+
+	cl_git_pass(git_odb_expand_ids(_odb, ids, num));
+	assert_found_objects(ids);
+	git__free(ids);
+
+	/* test looking for an explicit GIT_OBJ_ANY */
+
+	setup_prefix_query(&ids, &num);
+
+	for (i = 0; i < num; i++)
+		ids[i].type = GIT_OBJ_ANY;
+
+	cl_git_pass(git_odb_expand_ids(_odb, ids, num));
+	assert_found_objects(ids);
+	git__free(ids);
+
+	/* test looking for the completely wrong type */
+
+	setup_prefix_query(&ids, &num);
+
+	for (i = 0; i < num; i++)
+		ids[i].type = (ids[i].type == GIT_OBJ_BLOB) ?
+			GIT_OBJ_TREE : GIT_OBJ_BLOB;
+
+	cl_git_pass(git_odb_expand_ids(_odb, ids, num));
+	assert_notfound_objects(ids);
+	git__free(ids);
+}
+