diff --git a/include/git2/blob.h b/include/git2/blob.h index 551770678..544dc7c41 100644 --- a/include/git2/blob.h +++ b/include/git2/blob.h @@ -115,6 +115,48 @@ GIT_EXTERN(int) git_blob_create_fromfile(git_oid *oid, git_repository *repo, con */ GIT_EXTERN(int) git_blob_create_fromdisk(git_oid *oid, git_repository *repo, const char *path); +/** + * Write a loose blob to the Object Database from a + * provider of chunks of data. + * + * Provided the `hintpath` parameter is filled, its value + * will help to determine what git filters should be applied + * to the object before it can be placed to the object database. + * + * + * The implementation of the callback has to respect the + * following rules: + * + * - `content` will have to be filled by the consumer. The maximum number + * of bytes that the buffer can accept per call is defined by the + * `max_length` parameter. Allocation and freeing of the buffer will be taken + * care of by the function. + * + * - The callback is expected to return the number of bytes + * that `content` have been filled with. + * + * - When there is no more data to stream, the callback should + * return 0. This will prevent it from being invoked anymore. + * + * - When an error occurs, the callback should return -1. + * + * + * @param oid Return the id of the written blob + * + * @param repo repository where the blob will be written. + * This repository can be bare or not. + * + * @param hintpath if not NULL, will help selecting the filters + * to apply onto the content of the blob to be created. + * + * @return GIT_SUCCESS or an error code + */ +GIT_EXTERN(int) git_blob_create_fromchunks( + git_oid *oid, + git_repository *repo, + const char *hintpath, + int (*source_cb)(char *content, size_t max_length, void *payload), + void *payload); /** * Write an in-memory buffer to the ODB as a blob diff --git a/src/blob.c b/src/blob.c index e25944b91..699adec6b 100644 --- a/src/blob.c +++ b/src/blob.c @@ -148,27 +148,31 @@ static int write_symlink( return error; } -static int blob_create_internal(git_oid *oid, git_repository *repo, const char *path) +static int blob_create_internal(git_oid *oid, git_repository *repo, const char *content_path, const char *hint_path, bool try_load_filters) { int error; struct stat st; git_odb *odb = NULL; git_off_t size; - if ((error = git_path_lstat(path, &st)) < 0 || (error = git_repository_odb__weakptr(&odb, repo)) < 0) + assert(hint_path || !try_load_filters); + + if ((error = git_path_lstat(content_path, &st)) < 0 || (error = git_repository_odb__weakptr(&odb, repo)) < 0) return error; size = st.st_size; if (S_ISLNK(st.st_mode)) { - error = write_symlink(oid, odb, path, (size_t)size); + error = write_symlink(oid, odb, content_path, (size_t)size); } else { git_vector write_filters = GIT_VECTOR_INIT; - int filter_count; + int filter_count = 0; - /* Load the filters for writing this file to the ODB */ - filter_count = git_filters_load( - &write_filters, repo, path, GIT_FILTER_TO_ODB); + if (try_load_filters) { + /* Load the filters for writing this file to the ODB */ + filter_count = git_filters_load( + &write_filters, repo, hint_path, GIT_FILTER_TO_ODB); + } if (filter_count < 0) { /* Negative value means there was a critical error */ @@ -176,10 +180,10 @@ static int blob_create_internal(git_oid *oid, git_repository *repo, const char * } else if (filter_count == 0) { /* No filters need to be applied to the document: we can stream * directly from disk */ - error = write_file_stream(oid, odb, path, size); + error = write_file_stream(oid, odb, content_path, size); } else { /* We need to apply one or more filters */ - error = write_file_filtered(oid, odb, path, &write_filters); + error = write_file_filtered(oid, odb, content_path, &write_filters); } git_filters_free(&write_filters); @@ -216,7 +220,7 @@ int git_blob_create_fromfile(git_oid *oid, git_repository *repo, const char *pat return -1; } - error = blob_create_internal(oid, repo, git_buf_cstr(&full_path)); + error = blob_create_internal(oid, repo, git_buf_cstr(&full_path), git_buf_cstr(&full_path), true); git_buf_free(&full_path); return error; @@ -232,8 +236,53 @@ int git_blob_create_fromdisk(git_oid *oid, git_repository *repo, const char *pat return error; } - error = blob_create_internal(oid, repo, git_buf_cstr(&full_path)); + error = blob_create_internal(oid, repo, git_buf_cstr(&full_path), git_buf_cstr(&full_path), true); git_buf_free(&full_path); return error; } + +#define BUFFER_SIZE 4096 + +int git_blob_create_fromchunks( + git_oid *oid, + git_repository *repo, + const char *hintpath, + int (*source_cb)(char *content, size_t max_length, void *payload), + void *payload) +{ + int error = -1, read_bytes; + char *content = NULL; + git_filebuf file = GIT_FILEBUF_INIT; + + content = git__malloc(BUFFER_SIZE); + GITERR_CHECK_ALLOC(content); + + if (git_filebuf_open(&file, hintpath == NULL ? "streamed" : hintpath, GIT_FILEBUF_TEMPORARY) < 0) + goto cleanup; + + while (1) { + read_bytes = source_cb(content, BUFFER_SIZE, payload); + + assert(read_bytes <= BUFFER_SIZE); + + if (read_bytes <= 0) + break; + + if (git_filebuf_write(&file, content, read_bytes) < 0) + goto cleanup; + } + + if (read_bytes < 0) + goto cleanup; + + if (git_filebuf_flush(&file) < 0) + goto cleanup; + + error = blob_create_internal(oid, repo, file.path_lock, hintpath, hintpath != NULL); + +cleanup: + git_filebuf_cleanup(&file); + git__free(content); + return error; +} diff --git a/tests-clar/object/blob/fromchunks.c b/tests-clar/object/blob/fromchunks.c new file mode 100644 index 000000000..228e969b6 --- /dev/null +++ b/tests-clar/object/blob/fromchunks.c @@ -0,0 +1,87 @@ +#include "clar_libgit2.h" +#include "buffer.h" +#include "posix.h" +#include "path.h" +#include "fileops.h" + +static git_repository *repo; +static char textual_content[] = "libgit2\n\r\n\0"; + +void test_object_blob_fromchunks__initialize(void) +{ + repo = cl_git_sandbox_init("testrepo.git"); +} + +void test_object_blob_fromchunks__cleanup(void) +{ + cl_git_sandbox_cleanup(); +} + +static int text_chunked_source_cb(char *content, size_t max_length, void *payload) +{ + int *count; + + GIT_UNUSED(max_length); + + count = (int *)payload; + (*count)--; + + if (*count == 0) + return 0; + + strcpy(content, textual_content); + return strlen(textual_content); +} + +void test_object_blob_fromchunks__can_create_a_blob_from_a_in_memory_chunk_provider(void) +{ + git_oid expected_oid, oid; + git_object *blob; + int howmany = 7; + + cl_git_pass(git_oid_fromstr(&expected_oid, "321cbdf08803c744082332332838df6bd160f8f9")); + + cl_git_fail(git_object_lookup(&blob, repo, &expected_oid, GIT_OBJ_ANY)); + + cl_git_pass(git_blob_create_fromchunks(&oid, repo, NULL, text_chunked_source_cb, &howmany)); + + cl_git_pass(git_object_lookup(&blob, repo, &expected_oid, GIT_OBJ_ANY)); + git_object_free(blob); +} + +#define GITATTR "* text=auto\n" \ + "*.txt text\n" \ + "*.data binary\n" + +static void write_attributes(git_repository *repo) +{ + git_buf buf = GIT_BUF_INIT; + + cl_git_pass(git_buf_joinpath(&buf, git_repository_path(repo), "info")); + cl_git_pass(git_buf_joinpath(&buf, git_buf_cstr(&buf), "attributes")); + + cl_git_pass(git_futils_mkpath2file(git_buf_cstr(&buf), 0777)); + cl_git_rewritefile(git_buf_cstr(&buf), GITATTR); + + git_buf_free(&buf); +} + +static void assert_named_chunked_blob(const char *expected_sha, const char *fake_name) +{ + git_oid expected_oid, oid; + int howmany = 7; + + cl_git_pass(git_oid_fromstr(&expected_oid, expected_sha)); + + cl_git_pass(git_blob_create_fromchunks(&oid, repo, fake_name, text_chunked_source_cb, &howmany)); + cl_assert(git_oid_cmp(&expected_oid, &oid) == 0); +} + +void test_object_blob_fromchunks__creating_a_blob_from_chunks_honors_the_attributes_directives(void) +{ + write_attributes(repo); + + assert_named_chunked_blob("321cbdf08803c744082332332838df6bd160f8f9", "dummy.data"); + assert_named_chunked_blob("e9671e138a780833cb689753570fd10a55be84fb", "dummy.txt"); + assert_named_chunked_blob("e9671e138a780833cb689753570fd10a55be84fb", "dummy.dunno"); +}