Merge pull request #2208 from libgit2/vmg/mempack

In-memory packing backend

include/git2/pack.h

@@ -114,6 +114,17 @@ GIT_EXTERN(int) git_packbuilder_insert_tree(git_packbuilder *pb, const git_oid *
  */
 GIT_EXTERN(int) git_packbuilder_insert_commit(git_packbuilder *pb, const git_oid *id);
 
+/**
+ * Write the contents of the packfile to an in-memory buffer
+ *
+ * The contents of the buffer will become a valid packfile, even though there
+ * will be no attached index
+ *
+ * @param buf Buffer where to write the packfile
+ * @param pb The packbuilder
+ */
+GIT_EXTERN(int) git_packbuilder_write_buf(git_buf *buf, git_packbuilder *pb);
+
 /**
  * Write the new pack and corresponding index file to path.
  *

include/git2/sys/mempack.h

@@ -0,0 +1,85 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_sys_git_odb_mempack_h__
+#define INCLUDE_sys_git_odb_mempack_h__
+
+#include "git2/common.h"
+#include "git2/types.h"
+#include "git2/oid.h"
+#include "git2/odb.h"
+
+/**
+ * @file git2/sys/mempack.h
+ * @brief Custom ODB backend that permits packing objects in-memory
+ * @defgroup git_backend Git custom backend APIs
+ * @ingroup Git
+ * @{
+ */
+GIT_BEGIN_DECL
+
+/**
+ *	Instantiate a new mempack backend.
+ *
+ *	The backend must be added to an existing ODB with the highest
+ *	priority.
+ *
+ *		git_mempack_new(&mempacker);
+ *		git_repository_odb(&odb, repository);
+ *		git_odb_add_backend(odb, mempacker, 999);
+ *
+ *	Once the backend has been loaded, all writes to the ODB will
+ *	instead be queued in memory, and can be finalized with
+ *	`git_mempack_dump`.
+ *
+ *	Subsequent reads will also be served from the in-memory store
+ *	to ensure consistency, until the memory store is dumped.
+ *
+ *	@param out Poiter where to store the ODB backend
+ *	@return 0 on success; error code otherwise
+ */
+int git_mempack_new(git_odb_backend **out);
+
+/**
+ *	Dump all the queued in-memory writes to a packfile.
+ *
+ *	The contents of the packfile will be stored in the given buffer.
+ *	It is the caller's responsability to ensure that the generated
+ *	packfile is available to the repository (e.g. by writing it
+ *	to disk, or doing something crazy like distributing it across
+ *	several copies of the repository over a network).
+ *
+ *	Once the generated packfile is available to the repository,
+ *	call `git_mempack_reset` to cleanup the memory store.
+ *
+ *	Calling `git_mempack_reset` before the packfile has been
+ *	written to disk will result in an inconsistent repository
+ *	(the objects in the memory store won't be accessible).
+ *
+ *	@param pack Buffer where to store the raw packfile
+ *	@param repo The active repository where the backend is loaded
+ *	@param backend The mempack backend
+ *	@return 0 on success; error code otherwise
+ */
+int git_mempack_dump(git_buf *pack, git_repository *repo, git_odb_backend *backend);
+
+/**
+ *	Reset the memory packer by clearing all the queued objects.
+ *
+ *	This assumes that `git_mempack_dump` has been called before to
+ *	store all the queued objects into a single packfile.
+ *
+ *	Alternatively, call `reset` without a previous dump to "undo"
+ *	all the recently written objects, giving transaction-like
+ *	semantics to the Git repository.
+ *
+ *	@param backend The mempack backend
+ */
+void git_mempack_reset(git_odb_backend *backend);
+
+GIT_END_DECL
+
+#endif

src/odb_mempack.c

@@ -0,0 +1,182 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "common.h"
+#include "git2/object.h"
+#include "git2/sys/odb_backend.h"
+#include "fileops.h"
+#include "hash.h"
+#include "odb.h"
+#include "array.h"
+#include "oidmap.h"
+
+#include "git2/odb_backend.h"
+#include "git2/types.h"
+#include "git2/pack.h"
+
+GIT__USE_OIDMAP;
+
+struct memobject {
+	git_oid oid;
+	size_t len;
+	git_otype type;
+	char data[];
+};
+
+struct memory_packer_db {
+	git_odb_backend parent;
+	git_oidmap *objects;
+	git_array_t(struct memobject *) commits;
+};
+
+static int impl__write(git_odb_backend *_backend, const git_oid *oid, const void *data, size_t len, git_otype type)
+{
+	struct memory_packer_db *db = (struct memory_packer_db *)_backend;
+	struct memobject *obj = NULL; 
+	khiter_t pos;
+	int rval;
+
+	pos = kh_put(oid, db->objects, oid, &rval);
+	if (rval < 0)
+		return -1;
+
+	if (rval == 0)
+		return 0;
+
+	obj = git__malloc(sizeof(struct memobject) + len);
+	GITERR_CHECK_ALLOC(obj);
+
+	memcpy(obj->data, data, len);
+	git_oid_cpy(&obj->oid, oid);
+	obj->len = len;
+	obj->type = type;
+
+	kh_key(db->objects, pos) = &obj->oid;
+	kh_val(db->objects, pos) = obj;
+
+	if (type == GIT_OBJ_COMMIT) {
+		struct memobject **store = git_array_alloc(db->commits);
+		GITERR_CHECK_ALLOC(store);
+		*store = obj;
+	}
+
+	return 0;
+}
+
+static int impl__exists(git_odb_backend *backend, const git_oid *oid)
+{
+	struct memory_packer_db *db = (struct memory_packer_db *)backend;
+	khiter_t pos;
+
+	pos = kh_get(oid, db->objects, oid);
+	if (pos != kh_end(db->objects))
+		return 1;
+
+	return 0;
+}
+
+static int impl__read(void **buffer_p, size_t *len_p, git_otype *type_p, git_odb_backend *backend, const git_oid *oid)
+{
+	struct memory_packer_db *db = (struct memory_packer_db *)backend;
+	struct memobject *obj = NULL;
+	khiter_t pos;
+
+	pos = kh_get(oid, db->objects, oid);
+	if (pos == kh_end(db->objects))
+		return GIT_ENOTFOUND;
+
+	obj = kh_val(db->objects, pos);
+
+	*len_p = obj->len;
+	*type_p = obj->type;
+	*buffer_p = git__malloc(obj->len);
+	GITERR_CHECK_ALLOC(*buffer_p);
+
+	memcpy(*buffer_p, obj->data, obj->len);
+	return 0;
+}
+
+static int impl__read_header(size_t *len_p, git_otype *type_p, git_odb_backend *backend, const git_oid *oid)
+{
+	struct memory_packer_db *db = (struct memory_packer_db *)backend;
+	struct memobject *obj = NULL;
+	khiter_t pos;
+
+	pos = kh_get(oid, db->objects, oid);
+	if (pos == kh_end(db->objects))
+		return GIT_ENOTFOUND;
+
+	obj = kh_val(db->objects, pos);
+
+	*len_p = obj->len;
+	*type_p = obj->type;
+	return 0;
+}
+
+int git_mempack_dump(git_buf *pack, git_repository *repo, git_odb_backend *_backend)
+{
+	struct memory_packer_db *db = (struct memory_packer_db *)_backend;
+	git_packbuilder *packbuilder;
+	uint32_t i;
+	int err = -1;
+
+	if (git_packbuilder_new(&packbuilder, repo) < 0)
+		return -1;
+
+	for (i = 0; i < db->commits.size; ++i) {
+		struct memobject *commit = db->commits.ptr[i];
+
+		err = git_packbuilder_insert_commit(packbuilder, &commit->oid);
+		if (err < 0)
+			goto cleanup;
+	}
+
+	err = git_packbuilder_write_buf(pack, packbuilder);
+
+cleanup:
+	git_packbuilder_free(packbuilder);
+	return err;
+}
+
+void git_mempack_reset(git_odb_backend *_backend)
+{
+	struct memory_packer_db *db = (struct memory_packer_db *)_backend;
+	struct memobject *object = NULL;
+
+	kh_foreach_value(db->objects, object, {
+		git__free(object);
+	});
+
+	git_array_clear(db->commits);
+}
+
+static void impl__free(git_odb_backend *_backend)
+{
+	git_mempack_reset(_backend);
+	git__free(_backend);
+}
+
+int git_mempack_new(git_odb_backend **out)
+{
+	struct memory_packer_db *db;
+
+	assert(out);
+
+	db = git__calloc(1, sizeof(struct memory_packer_db));
+	GITERR_CHECK_ALLOC(db);
+
+	db->objects = git_oidmap_alloc();
+
+	db->parent.read = &impl__read;
+	db->parent.write = &impl__write;
+	db->parent.read_header = &impl__read_header;
+	db->parent.exists = &impl__exists;
+	db->parent.free = &impl__free;
+
+	*out = (git_odb_backend *)db;
+	return 0;
+}