From f6f72d7ef8091bf1fcf19f284e1db62a43f93381 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Wed, 23 Mar 2011 18:44:53 +0200 Subject: [PATCH] Improve the ODB writing backend Temporary files when doing streaming writes are now stored inside the Objects folder, to prevent issues when moving files between disks/partitions. Add support for block writes to the ODB again (for those backends that cannot implement streaming). --- include/git2/odb.h | 20 ++++++++++++ include/git2/odb_backend.h | 7 +++++ src/filebuf.c | 11 ++----- src/fileops.c | 64 ++------------------------------------ src/fileops.h | 2 +- src/odb.c | 35 +++++++++++++++++++++ src/odb_loose.c | 6 ++-- 7 files changed, 71 insertions(+), 74 deletions(-) diff --git a/include/git2/odb.h b/include/git2/odb.h index 1a0f0e64c..aa48b053a 100644 --- a/include/git2/odb.h +++ b/include/git2/odb.h @@ -156,6 +156,26 @@ GIT_EXTERN(int) git_odb_read_header(size_t *len_p, git_otype *type_p, git_odb *d */ GIT_EXTERN(int) git_odb_exists(git_odb *db, const git_oid *id); +/** + * Write an object directly into the ODB + * + * This method writes a full object straight into the ODB. + * For most cases, it is preferred to write objects through a write + * stream, which is both faster and less memory intensive, specially + * for big objects. + * + * This method is provided for compatibility with custom backends + * which are not able to support streaming writes + * + * @param oid pointer to store the OID result of the write + * @param odb object database where to store the object + * @param data buffer with the data to storr + * @param len size of the buffer + * @param type type of the data to store + * @return 0 on success; error code otherwise + */ +GIT_EXTERN(int) git_odb_write(git_oid *oid, git_odb *odb, const void *data, size_t len, git_otype type); + /** * Open a stream to write an object into the ODB * diff --git a/include/git2/odb_backend.h b/include/git2/odb_backend.h index 3875ec7f6..7eadef33e 100644 --- a/include/git2/odb_backend.h +++ b/include/git2/odb_backend.h @@ -55,6 +55,13 @@ struct git_odb_backend { struct git_odb_backend *, const git_oid *); + int (* write)( + git_oid *, + struct git_odb_backend *, + const void *, + size_t, + git_otype); + int (* writestream)( struct git_odb_stream **, struct git_odb_backend *, diff --git a/src/filebuf.c b/src/filebuf.c index 607ad618d..dff9373f6 100644 --- a/src/filebuf.c +++ b/src/filebuf.c @@ -151,8 +151,7 @@ int git_filebuf_open(git_filebuf *file, const char *path, int flags) int error; size_t path_len; - if (file == NULL) - return GIT_ERROR; + assert(file && path); memset(file, 0x0, sizeof(git_filebuf)); @@ -203,7 +202,7 @@ int git_filebuf_open(git_filebuf *file, const char *path, int flags) char tmp_path[GIT_PATH_MAX]; /* Open the file as temporary for locking */ - file->fd = gitfo_creat_tmp(tmp_path, "_filebuf_"); + file->fd = gitfo_mktemp(tmp_path, path); if (file->fd < 0) { error = GIT_EOSERR; goto cleanup; @@ -218,12 +217,6 @@ int git_filebuf_open(git_filebuf *file, const char *path, int flags) goto cleanup; } } else { - /* If the file is not temporary, make sure we have a path */ - if (path == NULL) { - error = GIT_ERROR; - goto cleanup; - } - path_len = strlen(path); /* Save the original path of the file */ diff --git a/src/fileops.c b/src/fileops.c index d754c49ee..5dd4a3806 100644 --- a/src/fileops.c +++ b/src/fileops.c @@ -25,11 +25,11 @@ int gitfo_mkdir_2file(const char *file_path) return GIT_SUCCESS; } -static int creat_tempfile(char *path_out, const char *tmp_dir, const char *filename) +int gitfo_mktemp(char *path_out, const char *filename) { int fd; - git__joinpath(path_out, tmp_dir, filename); + strcpy(path_out, filename); strcat(path_out, "_git2_XXXXXX"); #if defined(_MSC_VER) @@ -46,66 +46,6 @@ static int creat_tempfile(char *path_out, const char *tmp_dir, const char *filen return fd >= 0 ? fd : GIT_EOSERR; } -static const char *find_tmpdir(void) -{ - static int tmpdir_not_found = 0; - static char temp_dir[GIT_PATH_MAX]; - static const char *env_vars[] = { - "TEMP", "TMP", "TMPDIR" - }; - - unsigned int i, j; - char test_file[GIT_PATH_MAX]; - - if (tmpdir_not_found) - return NULL; - - if (temp_dir[0] != '\0') - return temp_dir; - - for (i = 0; i < ARRAY_SIZE(env_vars); ++i) { - char *env_path; - - env_path = getenv(env_vars[i]); - if (env_path == NULL) - continue; - - strcpy(temp_dir, env_path); - - /* Fix backslashes because Windows environment vars - * are probably fucked up */ - for (j = 0; j < strlen(temp_dir); ++j) - if (temp_dir[j] == '\\') - temp_dir[j] = '/'; - - if (creat_tempfile(test_file, temp_dir, "writetest") >= 0) { - gitfo_unlink(test_file); - return temp_dir; - } - } - - /* last resort: current folder. */ - strcpy(temp_dir, "./"); - if (creat_tempfile(test_file, temp_dir, "writetest") >= 0) { - gitfo_unlink(test_file); - return temp_dir; - } - - tmpdir_not_found = 1; - return NULL; -} - -int gitfo_creat_tmp(char *path_out, const char *filename) -{ - const char *tmp_dir; - - tmp_dir = find_tmpdir(); - if (tmp_dir == NULL) - return GIT_EOSERR; - - return creat_tempfile(path_out, tmp_dir, filename); -} - int gitfo_open(const char *path, int flags) { int fd = open(path, flags | O_BINARY); diff --git a/src/fileops.h b/src/fileops.h index ce236f608..6e0fd9d14 100644 --- a/src/fileops.h +++ b/src/fileops.h @@ -58,7 +58,7 @@ extern int gitfo_exists(const char *path); extern int gitfo_open(const char *path, int flags); extern int gitfo_creat(const char *path, int mode); extern int gitfo_creat_force(const char *path, int mode); -extern int gitfo_creat_tmp(char *path_out, const char *filename); +extern int gitfo_mktemp(char *path_out, const char *filename); extern int gitfo_isdir(const char *path); extern int gitfo_mkdir_recurs(const char *path, int mode); extern int gitfo_mkdir_2file(const char *path); diff --git a/src/odb.c b/src/odb.c index d825fd95c..33d5468d9 100644 --- a/src/odb.c +++ b/src/odb.c @@ -415,6 +415,41 @@ int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id) return error; } +int git_odb_write(git_oid *oid, git_odb *db, const void *data, size_t len, git_otype type) +{ + unsigned int i; + int error = GIT_ERROR; + + assert(oid && db); + + for (i = 0; i < db->backends.length && error < 0; ++i) { + backend_internal *internal = git_vector_get(&db->backends, i); + git_odb_backend *b = internal->backend; + + /* we don't write in alternates! */ + if (internal->is_alternate) + continue; + + if (b->write != NULL) + error = b->write(oid, b, data, len, type); + } + + /* if no backends were able to write the object directly, we try a streaming + * write to the backends; just write the whole object into the stream in one + * push */ + if (error < GIT_SUCCESS) { + git_odb_stream *stream; + + if ((error = git_odb_open_wstream(&stream, db, len, type)) == GIT_SUCCESS) { + stream->write(stream, data, len); + error = stream->finalize_write(oid, stream); + stream->free(stream); + } + } + + return error; +} + int git_odb_open_wstream(git_odb_stream **stream, git_odb *db, size_t size, git_otype type) { unsigned int i; diff --git a/src/odb_loose.c b/src/odb_loose.c index 4ab1128f3..8ee01cd2c 100644 --- a/src/odb_loose.c +++ b/src/odb_loose.c @@ -579,7 +579,7 @@ int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_backend loose_backend *backend; loose_writestream *stream; - char hdr[64]; + char hdr[64], tmp_path[GIT_PATH_MAX]; int hdrlen; int error; @@ -603,7 +603,9 @@ int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_backend stream->stream.free = &loose_backend__stream_free; stream->stream.mode = GIT_STREAM_WRONLY; - error = git_filebuf_open(&stream->fbuf, NULL, + git__joinpath(tmp_path, backend->objects_dir, "tmp_object"); + + error = git_filebuf_open(&stream->fbuf, tmp_path, GIT_FILEBUF_HASH_CONTENTS | GIT_FILEBUF_DEFLATE_CONTENTS | GIT_FILEBUF_TEMPORARY);