From 6105d597072cea0d1a89ee184826c2c98bf5d772 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Wed, 26 Mar 2014 18:17:08 +0100 Subject: [PATCH] In-memory packing backend --- include/git2/pack.h | 11 +++ include/git2/sys/mempack.h | 85 +++++++++++++++++ src/odb_mempack.c | 182 +++++++++++++++++++++++++++++++++++++ 3 files changed, 278 insertions(+) create mode 100644 include/git2/sys/mempack.h create mode 100644 src/odb_mempack.c diff --git a/include/git2/pack.h b/include/git2/pack.h index 11bb559d8..29c926c65 100644 --- a/include/git2/pack.h +++ b/include/git2/pack.h @@ -114,6 +114,17 @@ GIT_EXTERN(int) git_packbuilder_insert_tree(git_packbuilder *pb, const git_oid * */ GIT_EXTERN(int) git_packbuilder_insert_commit(git_packbuilder *pb, const git_oid *id); +/** + * Write the contents of the packfile to an in-memory buffer + * + * The contents of the buffer will become a valid packfile, even though there + * will be no attached index + * + * @param buf Buffer where to write the packfile + * @param pb The packbuilder + */ +GIT_EXTERN(int) git_packbuilder_write_buf(git_buf *buf, git_packbuilder *pb); + /** * Write the new pack and corresponding index file to path. * diff --git a/include/git2/sys/mempack.h b/include/git2/sys/mempack.h new file mode 100644 index 000000000..d3bc87b4d --- /dev/null +++ b/include/git2/sys/mempack.h @@ -0,0 +1,85 @@ +/* + * Copyright (C) the libgit2 contributors. All rights reserved. + * + * This file is part of libgit2, distributed under the GNU GPL v2 with + * a Linking Exception. For full terms see the included COPYING file. + */ +#ifndef INCLUDE_sys_git_odb_mempack_h__ +#define INCLUDE_sys_git_odb_mempack_h__ + +#include "git2/common.h" +#include "git2/types.h" +#include "git2/oid.h" +#include "git2/odb.h" + +/** + * @file git2/sys/mempack.h + * @brief Custom ODB backend that permits packing objects in-memory + * @defgroup git_backend Git custom backend APIs + * @ingroup Git + * @{ + */ +GIT_BEGIN_DECL + +/** + * Instantiate a new mempack backend. + * + * The backend must be added to an existing ODB with the highest + * priority. + * + * git_mempack_new(&mempacker); + * git_repository_odb(&odb, repository); + * git_odb_add_backend(odb, mempacker, 999); + * + * Once the backend has been loaded, all writes to the ODB will + * instead be queued in memory, and can be finalized with + * `git_mempack_dump`. + * + * Subsequent reads will also be served from the in-memory store + * to ensure consistency, until the memory store is dumped. + * + * @param out Poiter where to store the ODB backend + * @return 0 on success; error code otherwise + */ +int git_mempack_new(git_odb_backend **out); + +/** + * Dump all the queued in-memory writes to a packfile. + * + * The contents of the packfile will be stored in the given buffer. + * It is the caller's responsability to ensure that the generated + * packfile is available to the repository (e.g. by writing it + * to disk, or doing something crazy like distributing it across + * several copies of the repository over a network). + * + * Once the generated packfile is available to the repository, + * call `git_mempack_reset` to cleanup the memory store. + * + * Calling `git_mempack_reset` before the packfile has been + * written to disk will result in an inconsistent repository + * (the objects in the memory store won't be accessible). + * + * @param pack Buffer where to store the raw packfile + * @param repo The active repository where the backend is loaded + * @param backend The mempack backend + * @return 0 on success; error code otherwise + */ +int git_mempack_dump(git_buf *pack, git_repository *repo, git_odb_backend *backend); + +/** + * Reset the memory packer by clearing all the queued objects. + * + * This assumes that `git_mempack_dump` has been called before to + * store all the queued objects into a single packfile. + * + * Alternatively, call `reset` without a previous dump to "undo" + * all the recently written objects, giving transaction-like + * semantics to the Git repository. + * + * @param backend The mempack backend + */ +void git_mempack_reset(git_odb_backend *backend); + +GIT_END_DECL + +#endif diff --git a/src/odb_mempack.c b/src/odb_mempack.c new file mode 100644 index 000000000..d9b3a1824 --- /dev/null +++ b/src/odb_mempack.c @@ -0,0 +1,182 @@ +/* + * Copyright (C) the libgit2 contributors. All rights reserved. + * + * This file is part of libgit2, distributed under the GNU GPL v2 with + * a Linking Exception. For full terms see the included COPYING file. + */ + +#include "common.h" +#include "git2/object.h" +#include "git2/sys/odb_backend.h" +#include "fileops.h" +#include "hash.h" +#include "odb.h" +#include "array.h" +#include "oidmap.h" + +#include "git2/odb_backend.h" +#include "git2/types.h" +#include "git2/pack.h" + +GIT__USE_OIDMAP; + +struct memobject { + git_oid oid; + size_t len; + git_otype type; + char data[]; +}; + +struct memory_packer_db { + git_odb_backend parent; + git_oidmap *objects; + git_array_t(struct memobject *) commits; +}; + +static int impl__write(git_odb_backend *_backend, const git_oid *oid, const void *data, size_t len, git_otype type) +{ + struct memory_packer_db *db = (struct memory_packer_db *)_backend; + struct memobject *obj = NULL; + khiter_t pos; + int rval; + + pos = kh_put(oid, db->objects, oid, &rval); + if (rval < 0) + return -1; + + if (rval == 0) + return 0; + + obj = git__malloc(sizeof(struct memobject) + len); + GITERR_CHECK_ALLOC(obj); + + memcpy(obj->data, data, len); + git_oid_cpy(&obj->oid, oid); + obj->len = len; + obj->type = type; + + kh_key(db->objects, pos) = &obj->oid; + kh_val(db->objects, pos) = obj; + + if (type == GIT_OBJ_COMMIT) { + struct memobject **store = git_array_alloc(db->commits); + GITERR_CHECK_ALLOC(store); + *store = obj; + } + + return 0; +} + +static int impl__exists(git_odb_backend *backend, const git_oid *oid) +{ + struct memory_packer_db *db = (struct memory_packer_db *)backend; + khiter_t pos; + + pos = kh_get(oid, db->objects, oid); + if (pos != kh_end(db->objects)) + return 1; + + return 0; +} + +static int impl__read(void **buffer_p, size_t *len_p, git_otype *type_p, git_odb_backend *backend, const git_oid *oid) +{ + struct memory_packer_db *db = (struct memory_packer_db *)backend; + struct memobject *obj = NULL; + khiter_t pos; + + pos = kh_get(oid, db->objects, oid); + if (pos == kh_end(db->objects)) + return GIT_ENOTFOUND; + + obj = kh_val(db->objects, pos); + + *len_p = obj->len; + *type_p = obj->type; + *buffer_p = git__malloc(obj->len); + GITERR_CHECK_ALLOC(*buffer_p); + + memcpy(*buffer_p, obj->data, obj->len); + return 0; +} + +static int impl__read_header(size_t *len_p, git_otype *type_p, git_odb_backend *backend, const git_oid *oid) +{ + struct memory_packer_db *db = (struct memory_packer_db *)backend; + struct memobject *obj = NULL; + khiter_t pos; + + pos = kh_get(oid, db->objects, oid); + if (pos == kh_end(db->objects)) + return GIT_ENOTFOUND; + + obj = kh_val(db->objects, pos); + + *len_p = obj->len; + *type_p = obj->type; + return 0; +} + +int git_mempack_dump(git_buf *pack, git_repository *repo, git_odb_backend *_backend) +{ + struct memory_packer_db *db = (struct memory_packer_db *)_backend; + git_packbuilder *packbuilder; + uint32_t i; + int err = -1; + + if (git_packbuilder_new(&packbuilder, repo) < 0) + return -1; + + for (i = 0; i < db->commits.size; ++i) { + struct memobject *commit = db->commits.ptr[i]; + + err = git_packbuilder_insert_commit(packbuilder, &commit->oid); + if (err < 0) + goto cleanup; + } + + err = git_packbuilder_write_buf(pack, packbuilder); + +cleanup: + git_packbuilder_free(packbuilder); + return err; +} + +void git_mempack_reset(git_odb_backend *_backend) +{ + struct memory_packer_db *db = (struct memory_packer_db *)_backend; + struct memobject *object = NULL; + + kh_foreach_value(db->objects, object, { + git__free(object); + }); + + git_array_clear(db->commits); +} + +static void impl__free(git_odb_backend *_backend) +{ + git_mempack_reset(_backend); + git__free(_backend); +} + +int git_mempack_new(git_odb_backend **out) +{ + struct memory_packer_db *db; + + assert(out); + + db = git__calloc(1, sizeof(struct memory_packer_db)); + GITERR_CHECK_ALLOC(db); + + db->objects = git_oidmap_alloc(); + + db->parent.read = &impl__read; + db->parent.write = &impl__write; + db->parent.read_header = &impl__read_header; + db->parent.exists = &impl__exists; + db->parent.free = &impl__free; + + *out = (git_odb_backend *)db; + return 0; +}