From fbdc9db3643e89e26d6e49f7b4e4ec1c668a4f0d Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 22 Jan 2015 16:10:06 -0600 Subject: [PATCH 01/13] filters: introduce streaming filters Add structures and preliminary functions to take a buffer, file or blob and write the contents in chunks through an arbitrary number of chained filters, finally writing into a user-provided function accept the contents. --- include/git2/filter.h | 16 +++ include/git2/sys/filter.h | 14 +++ include/git2/types.h | 3 + src/filter.c | 240 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 265 insertions(+), 8 deletions(-) diff --git a/include/git2/filter.h b/include/git2/filter.h index 5b3f40394..600356c71 100644 --- a/include/git2/filter.h +++ b/include/git2/filter.h @@ -137,6 +137,22 @@ GIT_EXTERN(int) git_filter_list_apply_to_blob( git_filter_list *filters, git_blob *blob); +GIT_EXTERN(int) git_filter_list_stream_data( + git_filter_list *filters, + git_buf *data, + git_filter_stream *target); + +GIT_EXTERN(int) git_filter_list_stream_file( + git_filter_list *filters, + git_repository *repo, + const char *path, + git_filter_stream *target); + +GIT_EXTERN(int) git_filter_list_stream_blob( + git_filter_list *filters, + git_blob *blob, + git_filter_stream *target); + /** * Free a git_filter_list * diff --git a/include/git2/sys/filter.h b/include/git2/sys/filter.h index 60248271a..cc06c54ad 100644 --- a/include/git2/sys/filter.h +++ b/include/git2/sys/filter.h @@ -208,6 +208,19 @@ typedef int (*git_filter_apply_fn)( const git_buf *from, const git_filter_source *src); +struct git_filter_stream { + int (*write)(git_filter_stream *stream, const char *buffer, size_t len); + int (*close)(git_filter_stream *stream); + void (*free)(git_filter_stream *stream); +}; + +typedef int (*git_filter_stream_fn)( + git_filter_stream **out, + git_filter *self, + void **payload, + const git_filter_source *src, + git_filter_stream *next); + /** * Callback to clean up after filtering has been applied * @@ -247,6 +260,7 @@ struct git_filter { git_filter_shutdown_fn shutdown; git_filter_check_fn check; git_filter_apply_fn apply; + git_filter_stream_fn stream; git_filter_cleanup_fn cleanup; }; diff --git a/include/git2/types.h b/include/git2/types.h index 35e1573c7..3ed97d189 100644 --- a/include/git2/types.h +++ b/include/git2/types.h @@ -410,6 +410,9 @@ typedef enum { GIT_SUBMODULE_RECURSE_ONDEMAND = 2, } git_submodule_recurse_t; +/** A stream to write filters. */ +typedef struct git_filter_stream git_filter_stream; + /** @} */ GIT_END_DECL diff --git a/src/filter.c b/src/filter.c index 7b54a76c0..1af17d80c 100644 --- a/src/filter.c +++ b/src/filter.c @@ -703,12 +703,8 @@ int git_filter_list_apply_to_file( return error; } -int git_filter_list_apply_to_blob( - git_buf *out, - git_filter_list *filters, - git_blob *blob) +static int buf_from_blob(git_buf *out, git_blob *blob) { - git_buf in = GIT_BUF_INIT; git_off_t rawsize = git_blob_rawsize(blob); if (!git__is_sizet(rawsize)) { @@ -716,12 +712,240 @@ int git_filter_list_apply_to_blob( return -1; } - in.ptr = (char *)git_blob_rawcontent(blob); - in.asize = 0; - in.size = (size_t)rawsize; + out->ptr = (char *)git_blob_rawcontent(blob); + out->asize = 0; + out->size = (size_t)rawsize; + + return 0; +} + +int git_filter_list_apply_to_blob( + git_buf *out, + git_filter_list *filters, + git_blob *blob) +{ + git_buf in = GIT_BUF_INIT; + + if (buf_from_blob(&in, blob) < 0) + return -1; if (filters) git_oid_cpy(&filters->source.oid, git_blob_id(blob)); return git_filter_list_apply_to_data(out, filters, &in); } + +struct proxy_stream { + git_filter_stream base; + git_filter *filter; + const git_filter_source *source; + void **payload; + git_buf input; + git_buf output; + git_filter_stream *target; +}; + +static int proxy_stream_write( + git_filter_stream *s, const char *buffer, size_t len) +{ + struct proxy_stream *proxy_stream = (struct proxy_stream *)s; + assert(proxy_stream); + + return git_buf_put(&proxy_stream->input, buffer, len); +} + +static int proxy_stream_close(git_filter_stream *s) +{ + struct proxy_stream *proxy_stream = (struct proxy_stream *)s; + git_buf *writebuf; + int error; + + assert(proxy_stream); + + error = proxy_stream->filter->apply( + proxy_stream->filter, + proxy_stream->payload, + &proxy_stream->output, + &proxy_stream->input, + proxy_stream->source); + + if (error == GIT_PASSTHROUGH) { + writebuf = &proxy_stream->input; + } else if (error == 0) { + git_buf_sanitize(&proxy_stream->output); + writebuf = &proxy_stream->output; + } else { + return error; + } + + if ((error = proxy_stream->target->write( + proxy_stream->target, writebuf->ptr, writebuf->size)) == 0) + error = proxy_stream->target->close(proxy_stream->target); + + return error; +} + +static void proxy_stream_free(git_filter_stream *s) +{ + struct proxy_stream *proxy_stream = (struct proxy_stream *)s; + assert(proxy_stream); + + git_buf_free(&proxy_stream->input); + git_buf_free(&proxy_stream->output); + git__free(proxy_stream); +} + +static int proxy_stream_init( + git_filter_stream **out, + git_filter *filter, + void **payload, + const git_filter_source *source, + git_filter_stream *target) +{ + struct proxy_stream *proxy_stream = git__calloc(1, sizeof(struct proxy_stream)); + GITERR_CHECK_ALLOC(proxy_stream); + + proxy_stream->base.write = proxy_stream_write; + proxy_stream->base.close = proxy_stream_close; + proxy_stream->base.free = proxy_stream_free; + proxy_stream->filter = filter; + proxy_stream->payload = payload; + proxy_stream->source = source; + proxy_stream->target = target; + + *out = (git_filter_stream *)proxy_stream; + return 0; +} + +static int stream_list_init( + git_filter_stream **out, + git_vector *streams, + git_filter_list *filters, + git_filter_stream *target) +{ + git_vector filter_streams = GIT_VECTOR_INIT; + git_filter_stream *last_stream = target; + size_t i; + int error = 0; + + *out = NULL; + + if (!filters) { + *out = target; + return 0; + } + + /* Create filters last to first to get the chaining direction */ + for (i = 0; i < git_array_size(filters->filters); ++i) { + size_t filter_idx = (filters->source.mode == GIT_FILTER_TO_WORKTREE) ? + git_array_size(filters->filters) - 1 - i : i; + git_filter_entry *fe = git_array_get(filters->filters, filter_idx); + git_filter_stream *filter_stream; + git_filter_stream_fn stream_init; + + assert(fe->filter->stream || fe->filter->apply); + + /* If necessary, create a stream that proxies the one-shot apply */ + stream_init = fe->filter->stream ? + fe->filter->stream : proxy_stream_init; + + if ((error = stream_init(&filter_stream, fe->filter, &fe->payload, &filters->source, last_stream)) < 0) + return error; + + git_vector_insert(&filter_streams, filter_stream); + last_stream = filter_stream; + } + + *out = last_stream; + return 0; +} + +void stream_list_free(git_vector *streams) +{ + git_filter_stream *stream; + size_t i; + + git_vector_foreach(streams, i, stream) + stream->free(stream); +} + +#define STREAM_BUFSIZE 10240 + +/* TODO: maybe not use filter_stream as a target but create one */ +int git_filter_list_stream_file( + git_filter_list *filters, + git_buf *data, + git_repository *repo, + const char *path, + git_filter_stream *target) +{ + char buf[STREAM_BUFSIZE]; + git_buf abspath = GIT_BUF_INIT, raw = GIT_BUF_INIT; + const char *base = repo ? git_repository_workdir(repo) : NULL; + git_vector filter_streams = GIT_VECTOR_INIT; + git_filter_stream *stream_start; + ssize_t readlen; + int fd, error; + + if ((error = stream_list_init( + &stream_start, &filter_streams, filters, target)) < 0 || + (error = git_path_join_unrooted(&abspath, path, base, NULL)) < 0) + goto done; + + if ((fd = git_futils_open_ro(path)) < 0) { + error = fd; + goto done; + } + + while ((readlen = p_read(fd, buf, STREAM_BUFSIZE)) > 0) { + if ((error = stream_start->write(stream_start, data->ptr, data->size)) < 0) + goto done; + } + + if (!readlen) + error = stream_start->close(stream_start); + else if (readlen < 0) + error = readlen; + + p_close(fd); + +done: + stream_list_free(&filter_streams); + git_buf_free(&abspath); + return error; +} + +int git_filter_list_stream_data( + git_filter_list *filters, + git_buf *data, + git_filter_stream *target) +{ + git_vector filter_streams = GIT_VECTOR_INIT; + git_filter_stream *stream_start; + int error = 0; + + if ((error = stream_list_init( + &stream_start, &filter_streams, filters, target)) == 0 && + (error = + stream_start->write(stream_start, data->ptr, data->size)) == 0) + error = stream_start->close(stream_start); + + stream_list_free(&filter_streams); + return error; +} + +int git_filter_list_stream_blob( + git_filter_list *filters, + git_blob *blob, + git_filter_stream *target) +{ + git_buf in = GIT_BUF_INIT; + + if (buf_from_blob(&in, blob) < 0) + return -1; + + if (filters) + git_oid_cpy(&filters->source.oid, git_blob_id(blob)); + + return git_filter_list_stream_data(filters, &in, target); +} From 5555696f8a48d2319a7ca0918726711e6a81a924 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 22 Jan 2015 16:11:22 -0600 Subject: [PATCH 02/13] filters: stream internally Migrate the `git_filter_list_apply_*` functions over to using the new filter streams. --- src/filter.c | 158 +++++++++++++++++++++++++++------------------------ 1 file changed, 83 insertions(+), 75 deletions(-) diff --git a/src/filter.c b/src/filter.c index 1af17d80c..700afbac3 100644 --- a/src/filter.c +++ b/src/filter.c @@ -617,67 +617,70 @@ static int filter_list_out_buffer_from_raw( return 0; } -int git_filter_list_apply_to_data( - git_buf *tgt, git_filter_list *fl, git_buf *src) +struct buf_stream { + git_filter_stream base; + git_buf *target; + bool complete; +}; + +static int buf_stream_write( + git_filter_stream *s, const char *buffer, size_t len) { - int error = 0; - uint32_t i; - git_buf *dbuffer[2], local = GIT_BUF_INIT; - unsigned int si = 0; + struct buf_stream *buf_stream = (struct buf_stream *)s; + assert(buf_stream); + + assert(buf_stream->complete == 0); + + return git_buf_put(buf_stream->target, buffer, len); +} + +static int buf_stream_close(git_filter_stream *s) +{ + struct buf_stream *buf_stream = (struct buf_stream *)s; + assert(buf_stream); + + assert(buf_stream->complete == 0); + buf_stream->complete = 1; + + return 0; +} + +static void buf_stream_free(git_filter_stream *s) +{ + GIT_UNUSED(s); +} + +static void buf_stream_init(struct buf_stream *writer, git_buf *target) +{ + memset(writer, 0, sizeof(struct buf_stream)); + + writer->base.write = buf_stream_write; + writer->base.close = buf_stream_close; + writer->base.free = buf_stream_free; + writer->target = target; + + git_buf_clear(target); +} + +int git_filter_list_apply_to_data( + git_buf *tgt, git_filter_list *filters, git_buf *src) +{ + struct buf_stream writer; + int error; git_buf_sanitize(tgt); git_buf_sanitize(src); - if (!fl) + if (!filters) return filter_list_out_buffer_from_raw(tgt, src->ptr, src->size); - dbuffer[0] = src; - dbuffer[1] = tgt; + buf_stream_init(&writer, tgt); - /* if `src` buffer is reallocable, then use it, otherwise copy it */ - if (!git_buf_is_allocated(src)) { - if (git_buf_set(&local, src->ptr, src->size) < 0) - return -1; - dbuffer[0] = &local; - } - - for (i = 0; i < git_array_size(fl->filters); ++i) { - unsigned int di = 1 - si; - uint32_t fidx = (fl->source.mode == GIT_FILTER_TO_WORKTREE) ? - i : git_array_size(fl->filters) - 1 - i; - git_filter_entry *fe = git_array_get(fl->filters, fidx); - - dbuffer[di]->size = 0; - - /* Apply the filter from dbuffer[src] to the other buffer; - * if the filtering is canceled by the user mid-filter, - * we skip to the next filter without changing the source - * of the double buffering (so that the text goes through - * cleanly). - */ - - error = fe->filter->apply( - fe->filter, &fe->payload, dbuffer[di], dbuffer[si], &fl->source); - - if (error == GIT_PASSTHROUGH) { - /* PASSTHROUGH means filter decided not to process the buffer */ - error = 0; - } else if (!error) { - git_buf_sanitize(dbuffer[di]); /* force NUL termination */ - si = di; /* swap buffers */ - } else { - tgt->size = 0; - goto cleanup; - } - } - - /* Ensure that the output ends up in dbuffer[1] (i.e. the dest) */ - if (si != 1) - git_buf_swap(dbuffer[0], dbuffer[1]); - -cleanup: - git_buf_free(&local); /* don't leak if we allocated locally */ + if ((error = git_filter_list_stream_data(filters, src, + (git_filter_stream *)&writer)) < 0) + return error; + assert(writer.complete); return error; } @@ -687,19 +690,16 @@ int git_filter_list_apply_to_file( git_repository *repo, const char *path) { + struct buf_stream writer; int error; - const char *base = repo ? git_repository_workdir(repo) : NULL; - git_buf abspath = GIT_BUF_INIT, raw = GIT_BUF_INIT; - if (!(error = git_path_join_unrooted(&abspath, path, base, NULL)) && - !(error = git_futils_readbuffer(&raw, abspath.ptr))) - { - error = git_filter_list_apply_to_data(out, filters, &raw); + buf_stream_init(&writer, out); - git_buf_free(&raw); - } + if ((error = git_filter_list_stream_file( + filters, repo, path, (git_filter_stream *)&writer)) < 0) + return error; - git_buf_free(&abspath); + assert(writer.complete); return error; } @@ -724,15 +724,17 @@ int git_filter_list_apply_to_blob( git_filter_list *filters, git_blob *blob) { - git_buf in = GIT_BUF_INIT; + struct buf_stream writer; + int error; - if (buf_from_blob(&in, blob) < 0) - return -1; + buf_stream_init(&writer, out); - if (filters) - git_oid_cpy(&filters->source.oid, git_blob_id(blob)); + if ((error = git_filter_list_stream_blob( + filters, blob, (git_filter_stream *)&writer)) < 0) + return error; - return git_filter_list_apply_to_data(out, filters, &in); + assert(writer.complete); + return error; } struct proxy_stream { @@ -823,7 +825,6 @@ static int stream_list_init( git_filter_list *filters, git_filter_stream *target) { - git_vector filter_streams = GIT_VECTOR_INIT; git_filter_stream *last_stream = target; size_t i; int error = 0; @@ -845,14 +846,19 @@ static int stream_list_init( assert(fe->filter->stream || fe->filter->apply); - /* If necessary, create a stream that proxies the one-shot apply */ + /* If necessary, create a stream that proxies the traditional + * application. + */ stream_init = fe->filter->stream ? fe->filter->stream : proxy_stream_init; - if ((error = stream_init(&filter_stream, fe->filter, &fe->payload, &filters->source, last_stream)) < 0) - return error; + error = stream_init(&filter_stream, fe->filter, + &fe->payload, &filters->source, last_stream); - git_vector_insert(&filter_streams, filter_stream); + if (error < 0) + return error; + + git_vector_insert(streams, filter_stream); last_stream = filter_stream; } @@ -867,6 +873,7 @@ void stream_list_free(git_vector *streams) git_vector_foreach(streams, i, stream) stream->free(stream); + git_vector_free(streams); } #define STREAM_BUFSIZE 10240 @@ -874,13 +881,12 @@ void stream_list_free(git_vector *streams) /* TODO: maybe not use filter_stream as a target but create one */ int git_filter_list_stream_file( git_filter_list *filters, - git_buf *data, git_repository *repo, const char *path, git_filter_stream *target) { char buf[STREAM_BUFSIZE]; - git_buf abspath = GIT_BUF_INIT, raw = GIT_BUF_INIT; + git_buf abspath = GIT_BUF_INIT; const char *base = repo ? git_repository_workdir(repo) : NULL; git_vector filter_streams = GIT_VECTOR_INIT; git_filter_stream *stream_start; @@ -898,7 +904,7 @@ int git_filter_list_stream_file( } while ((readlen = p_read(fd, buf, STREAM_BUFSIZE)) > 0) { - if ((error = stream_start->write(stream_start, data->ptr, data->size)) < 0) + if ((error = stream_start->write(stream_start, buf, readlen)) < 0) goto done; } @@ -924,6 +930,8 @@ int git_filter_list_stream_data( git_filter_stream *stream_start; int error = 0; + git_buf_sanitize(data); + if ((error = stream_list_init( &stream_start, &filter_streams, filters, target)) == 0 && (error = From e78f5c9f4282319cf8e1afc8631e6259e91653a6 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 22 Jan 2015 16:11:36 -0600 Subject: [PATCH 03/13] checkout: stream the blob into the filters Use the new streaming filter API during checkout. --- src/checkout.c | 141 ++++++++++++++++++++++++++++++++----------------- src/filter.c | 2 + 2 files changed, 94 insertions(+), 49 deletions(-) diff --git a/src/checkout.c b/src/checkout.c index 880af3dff..623ac92a1 100644 --- a/src/checkout.c +++ b/src/checkout.c @@ -17,6 +17,7 @@ #include "git2/diff.h" #include "git2/submodule.h" #include "git2/sys/index.h" +#include "git2/sys/filter.h" #include "refs.h" #include "repository.h" @@ -1371,22 +1372,106 @@ static int mkpath2file( return error; } -static int buffer_to_file( +struct checkout_stream { + git_filter_stream base; + const char *path; + int fd; + int open; +}; + +static int checkout_stream_write( + git_filter_stream *s, const char *buffer, size_t len) +{ + struct checkout_stream *stream = (struct checkout_stream *)s; + int ret; + + if ((ret = p_write(stream->fd, buffer, len)) < 0) + giterr_set(GITERR_OS, "Could not write to '%s'", stream->path); + + return ret; +} + +static int checkout_stream_close(git_filter_stream *s) +{ + struct checkout_stream *stream = (struct checkout_stream *)s; + assert(stream && stream->open); + + stream->open = 0; + return 0; +} + +static void checkout_stream_free(git_filter_stream *s) +{ + GIT_UNUSED(s); +} + +static int blob_content_to_file( checkout_data *data, struct stat *st, - git_buf *buf, + git_blob *blob, const char *path, - mode_t file_mode) + const char *hint_path, + mode_t entry_filemode) { - int error; + int flags = data->opts.file_open_flags; + mode_t file_mode = data->opts.file_mode ? + data->opts.file_mode : entry_filemode; + struct checkout_stream writer; + mode_t mode; + git_filter_list *fl = NULL; + int fd; + int error = 0; + + if (hint_path == NULL) + hint_path = path; if ((error = mkpath2file(data, path, data->opts.dir_mode)) < 0) return error; - if ((error = git_futils_writebuffer( - buf, path, data->opts.file_open_flags, file_mode)) < 0) + if (flags <= 0) + flags = O_CREAT | O_TRUNC | O_WRONLY; + if (!(mode = file_mode)) + mode = GIT_FILEMODE_BLOB; + + if ((fd = p_open(path, flags, mode)) < 0) { + giterr_set(GITERR_OS, "Could not open '%s' for writing", path); + return fd; + } + + if (!data->opts.disable_filters && + (error = git_filter_list__load_with_attr_session( + &fl, data->repo, &data->attr_session, blob, hint_path, + GIT_FILTER_TO_WORKTREE, GIT_FILTER_OPT_DEFAULT))) return error; + /* setup the writer */ + memset(&writer, 0, sizeof(struct checkout_stream)); + writer.base.write = checkout_stream_write; + writer.base.close = checkout_stream_close; + writer.base.free = checkout_stream_free; + writer.path = path; + writer.fd = fd; + writer.open = 1; + + error = git_filter_list_stream_blob(fl, blob, (git_filter_stream *)&writer); + + assert(writer.open == 0); + + git_filter_list_free(fl); + p_close(fd); + + if (error < 0) + return error; + + if (GIT_PERMS_IS_EXEC(mode)) { + data->perfdata.chmod_calls++; + + if ((error = p_chmod(path, mode)) < 0) { + giterr_set(GITERR_OS, "Failed to set permissions on '%s'", path); + return error; + } + } + if (st) { data->perfdata.stat_calls++; @@ -1394,53 +1479,11 @@ static int buffer_to_file( giterr_set(GITERR_OS, "Error statting '%s'", path); return error; } - } - if (GIT_PERMS_IS_EXEC(file_mode)) { - data->perfdata.chmod_calls++; - - if ((error = p_chmod(path, file_mode)) < 0) - giterr_set(GITERR_OS, "Failed to set permissions on '%s'", path); - } - - return error; -} - -static int blob_content_to_file( - checkout_data *data, - struct stat *st, - git_blob *blob, - const char *path, - const char * hint_path, - mode_t entry_filemode) -{ - mode_t file_mode = data->opts.file_mode ? - data->opts.file_mode : entry_filemode; - git_buf out = GIT_BUF_INIT; - git_filter_list *fl = NULL; - int error = 0; - - if (hint_path == NULL) - hint_path = path; - - if (!data->opts.disable_filters) - error = git_filter_list__load_with_attr_session( - &fl, data->repo, &data->attr_session, blob, hint_path, - GIT_FILTER_TO_WORKTREE, GIT_FILTER_OPT_DEFAULT); - - if (!error) - error = git_filter_list_apply_to_blob(&out, fl, blob); - - git_filter_list_free(fl); - - if (!error) { - error = buffer_to_file(data, st, &out, path, file_mode); st->st_mode = entry_filemode; - - git_buf_free(&out); } - return error; + return 0; } static int blob_content_to_link( diff --git a/src/filter.c b/src/filter.c index 700afbac3..d930b2399 100644 --- a/src/filter.c +++ b/src/filter.c @@ -522,6 +522,7 @@ int git_filter_list__load_with_attr_session( fe = git_array_alloc(fl->filters); GITERR_CHECK_ALLOC(fe); fe->filter = fdef->filter; + fe->stream = NULL; fe->payload = payload; } } @@ -590,6 +591,7 @@ int git_filter_list_push( fe = git_array_alloc(fl->filters); GITERR_CHECK_ALLOC(fe); fe->filter = filter; + fe->stream = NULL; fe->payload = payload; return 0; From 646364e780c65b1af2e28a7bdeabd22b67816454 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 17 Feb 2015 20:25:31 +0000 Subject: [PATCH 04/13] checkout: maintain temporary buffer for filters Let the filters use the checkout data's temporary buffer, instead of having to allocate new buffers each time. --- src/checkout.c | 3 +++ src/filter.c | 33 +++++++++++++++++++++------------ src/filter.h | 3 +++ 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/checkout.c b/src/checkout.c index 623ac92a1..35129d771 100644 --- a/src/checkout.c +++ b/src/checkout.c @@ -1444,6 +1444,9 @@ static int blob_content_to_file( GIT_FILTER_TO_WORKTREE, GIT_FILTER_OPT_DEFAULT))) return error; + if (fl) + git_filter_list__set_temp_buf(fl, &data->tmp); + /* setup the writer */ memset(&writer, 0, sizeof(struct checkout_stream)); writer.base.write = checkout_stream_write; diff --git a/src/filter.c b/src/filter.c index d930b2399..2cae24e20 100644 --- a/src/filter.c +++ b/src/filter.c @@ -34,6 +34,7 @@ typedef struct { struct git_filter_list { git_array_t(git_filter_entry) filters; git_filter_source source; + git_buf *temp_buf; char path[GIT_FLEX_ARRAY]; }; @@ -522,7 +523,6 @@ int git_filter_list__load_with_attr_session( fe = git_array_alloc(fl->filters); GITERR_CHECK_ALLOC(fe); fe->filter = fdef->filter; - fe->stream = NULL; fe->payload = payload; } } @@ -549,6 +549,11 @@ int git_filter_list_load( filters, repo, NULL, blob, path, mode, options); } +void git_filter_list__set_temp_buf(git_filter_list *fl, git_buf *temp_buf) +{ + fl->temp_buf = temp_buf; +} + void git_filter_list_free(git_filter_list *fl) { uint32_t i; @@ -591,7 +596,6 @@ int git_filter_list_push( fe = git_array_alloc(fl->filters); GITERR_CHECK_ALLOC(fe); fe->filter = filter; - fe->stream = NULL; fe->payload = payload; return 0; @@ -745,7 +749,8 @@ struct proxy_stream { const git_filter_source *source; void **payload; git_buf input; - git_buf output; + git_buf temp_buf; + git_buf *output; git_filter_stream *target; }; @@ -769,15 +774,15 @@ static int proxy_stream_close(git_filter_stream *s) error = proxy_stream->filter->apply( proxy_stream->filter, proxy_stream->payload, - &proxy_stream->output, + proxy_stream->output, &proxy_stream->input, proxy_stream->source); if (error == GIT_PASSTHROUGH) { writebuf = &proxy_stream->input; } else if (error == 0) { - git_buf_sanitize(&proxy_stream->output); - writebuf = &proxy_stream->output; + git_buf_sanitize(proxy_stream->output); + writebuf = proxy_stream->output; } else { return error; } @@ -795,13 +800,14 @@ static void proxy_stream_free(git_filter_stream *s) assert(proxy_stream); git_buf_free(&proxy_stream->input); - git_buf_free(&proxy_stream->output); + git_buf_free(&proxy_stream->temp_buf); git__free(proxy_stream); } static int proxy_stream_init( git_filter_stream **out, git_filter *filter, + git_buf *temp_buf, void **payload, const git_filter_source *source, git_filter_stream *target) @@ -816,6 +822,7 @@ static int proxy_stream_init( proxy_stream->payload = payload; proxy_stream->source = source; proxy_stream->target = target; + proxy_stream->output = temp_buf ? temp_buf : &proxy_stream->temp_buf; *out = (git_filter_stream *)proxy_stream; return 0; @@ -844,18 +851,20 @@ static int stream_list_init( git_array_size(filters->filters) - 1 - i : i; git_filter_entry *fe = git_array_get(filters->filters, filter_idx); git_filter_stream *filter_stream; - git_filter_stream_fn stream_init; assert(fe->filter->stream || fe->filter->apply); /* If necessary, create a stream that proxies the traditional * application. */ - stream_init = fe->filter->stream ? - fe->filter->stream : proxy_stream_init; - - error = stream_init(&filter_stream, fe->filter, + if (fe->filter->stream) + error = fe->filter->stream(&filter_stream, fe->filter, &fe->payload, &filters->source, last_stream); + else + /* Create a stream that proxies the one-shot apply */ + error = proxy_stream_init(&filter_stream, fe->filter, + filters->temp_buf, &fe->payload, &filters->source, + last_stream); if (error < 0) return error; diff --git a/src/filter.h b/src/filter.h index 390ffebad..53f3afdbb 100644 --- a/src/filter.h +++ b/src/filter.h @@ -24,6 +24,9 @@ typedef enum { GIT_CRLF_AUTO, } git_crlf_t; +extern void git_filter_list__set_temp_buf( + git_filter_list *fl, git_buf *temp_buf); + extern void git_filter_free(git_filter *filter); extern int git_filter_list__load_with_attr_session( From 8c2dfb38dbc782a6b964bfcfd43e4f46bb71526e Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 17 Feb 2015 13:27:06 -0500 Subject: [PATCH 05/13] filter: test a large file through the stream Test pushing a file on-disk into a streaming filter that compresses it into the ODB, and inflates it back into the working directory. --- tests/filter/stream.c | 221 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 tests/filter/stream.c diff --git a/tests/filter/stream.c b/tests/filter/stream.c new file mode 100644 index 000000000..ff5361a37 --- /dev/null +++ b/tests/filter/stream.c @@ -0,0 +1,221 @@ +#include "clar_libgit2.h" +#include "posix.h" +#include "blob.h" +#include "filter.h" +#include "buf_text.h" +#include "git2/sys/filter.h" +#include "git2/sys/repository.h" + +static git_repository *g_repo = NULL; + +static git_filter *create_compress_filter(void); +static git_filter *compress_filter; + +void test_filter_stream__initialize(void) +{ + compress_filter = create_compress_filter(); + + cl_git_pass(git_filter_register("compress", compress_filter, 50)); + g_repo = cl_git_sandbox_init("empty_standard_repo"); +} + +void test_filter_stream__cleanup(void) +{ + cl_git_sandbox_cleanup(); + g_repo = NULL; + + git_filter_unregister("compress"); +} + +#define CHUNKSIZE 10240 + +struct compress_stream { + git_filter_stream base; + git_filter_stream *next; + git_filter_mode_t mode; + char current; + size_t current_chunk; +}; + +static int compress_stream_write__deflated(struct compress_stream *stream, const char *buffer, size_t len) +{ + size_t idx = 0; + + while (len > 0) { + size_t chunkremain, chunksize; + + if (stream->current_chunk == 0) + stream->current = buffer[idx]; + + chunkremain = CHUNKSIZE - stream->current_chunk; + chunksize = min(chunkremain, len); + + stream->current_chunk += chunksize; + len -= chunksize; + idx += chunksize; + + if (stream->current_chunk == CHUNKSIZE) { + cl_git_pass(stream->next->write(stream->next, &stream->current, 1)); + stream->current_chunk = 0; + } + } + + return 0; +} + +static int compress_stream_write__inflated(struct compress_stream *stream, const char *buffer, size_t len) +{ + char inflated[CHUNKSIZE]; + size_t i, j; + + for (i = 0; i < len; i++) { + for (j = 0; j < CHUNKSIZE; j++) + inflated[j] = buffer[i]; + + cl_git_pass(stream->next->write(stream->next, inflated, CHUNKSIZE)); + } + + return 0; +} + +static int compress_stream_write(git_filter_stream *s, const char *buffer, size_t len) +{ + struct compress_stream *stream = (struct compress_stream *)s; + + return (stream->mode == GIT_FILTER_TO_ODB) ? + compress_stream_write__deflated(stream, buffer, len) : + compress_stream_write__inflated(stream, buffer, len); +} + +static int compress_stream_close(git_filter_stream *s) +{ + struct compress_stream *stream = (struct compress_stream *)s; + cl_assert_equal_i(0, stream->current_chunk); + stream->next->close(stream->next); + return 0; +} + +static void compress_stream_free(git_filter_stream *stream) +{ + git__free(stream); +} + +static int compress_filter_stream_init( + git_filter_stream **out, + git_filter *self, + void **payload, + const git_filter_source *src, + git_filter_stream *next) +{ + struct compress_stream *stream = git__calloc(1, sizeof(struct compress_stream)); + cl_assert(stream); + + GIT_UNUSED(self); + GIT_UNUSED(payload); + + stream->base.write = compress_stream_write; + stream->base.close = compress_stream_close; + stream->base.free = compress_stream_free; + stream->next = next; + stream->mode = git_filter_source_mode(src); + + *out = (git_filter_stream *)stream; + return 0; +} + +static void compress_filter_free(git_filter *f) +{ + git__free(f); +} + +git_filter *create_compress_filter(void) +{ + git_filter *filter = git__calloc(1, sizeof(git_filter)); + cl_assert(filter); + + filter->version = GIT_FILTER_VERSION; + filter->attributes = "+compress"; + filter->stream = compress_filter_stream_init; + filter->shutdown = compress_filter_free; + + return filter; +} + +static void writefile(const char *filename, size_t numchunks) +{ + git_buf path = GIT_BUF_INIT; + char buf[CHUNKSIZE]; + size_t i = 0, j = 0; + int fd; + + cl_git_pass(git_buf_joinpath(&path, "empty_standard_repo", filename)); + + fd = p_open(path.ptr, O_RDWR|O_CREAT, 0666); + cl_assert(fd >= 0); + + for (i = 0; i < numchunks; i++) { + for (j = 0; j < CHUNKSIZE; j++) { + buf[j] = i % 256; + } + + cl_git_pass(p_write(fd, buf, CHUNKSIZE)); + } + p_close(fd); + + git_buf_free(&path); +} + +static void test_stream(size_t numchunks) +{ + git_index *index; + const git_index_entry *entry; + git_blob *blob; + struct stat st; + git_checkout_options checkout_opts = GIT_CHECKOUT_OPTIONS_INIT; + + checkout_opts.checkout_strategy = GIT_CHECKOUT_FORCE; + + cl_git_mkfile( + "empty_standard_repo/.gitattributes", + "* compress\n"); + + /* write a file to disk */ + writefile("streamed_file", numchunks); + + /* place it in the index */ + cl_git_pass(git_repository_index(&index, g_repo)); + cl_git_pass(git_index_add_bypath(index, "streamed_file")); + cl_git_pass(git_index_write(index)); + + /* ensure it was appropriately compressed */ + cl_assert(entry = git_index_get_bypath(index, "streamed_file", 0)); + + cl_git_pass(git_blob_lookup(&blob, g_repo, &entry->id)); + cl_assert_equal_i(numchunks, git_blob_rawsize(blob)); + + /* check the file back out */ + cl_must_pass(p_unlink("empty_standard_repo/streamed_file")); + cl_git_pass(git_checkout_index(g_repo, index, &checkout_opts)); + + /* ensure it was decompressed */ + cl_must_pass(p_stat("empty_standard_repo/streamed_file", &st)); + cl_assert_equal_sz((numchunks * CHUNKSIZE), st.st_size); + + git_index_free(index); + git_blob_free(blob); +} + +/* write a 50KB file through the "compression" stream */ +void test_filter_stream__smallfile(void) +{ + test_stream(5); +} + +/* optionally write a 500 MB file through the compression stream */ +void test_filter_stream__bigfile(void) +{ + if (!cl_getenv("GITTEST_INVASIVE_FILESYSTEM")) + cl_skip(); + + test_stream(51200); +} From b75f15aaf1587594cd398b192cdc40aa83ba8e23 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Wed, 18 Feb 2015 09:25:32 -0500 Subject: [PATCH 06/13] git_writestream: from git_filter_stream --- include/git2/filter.h | 6 ++--- include/git2/sys/filter.h | 10 ++------ include/git2/types.h | 10 ++++++-- src/checkout.c | 10 ++++---- src/filter.c | 50 +++++++++++++++++++-------------------- tests/filter/stream.c | 16 ++++++------- 6 files changed, 51 insertions(+), 51 deletions(-) diff --git a/include/git2/filter.h b/include/git2/filter.h index 600356c71..c0aaf1508 100644 --- a/include/git2/filter.h +++ b/include/git2/filter.h @@ -140,18 +140,18 @@ GIT_EXTERN(int) git_filter_list_apply_to_blob( GIT_EXTERN(int) git_filter_list_stream_data( git_filter_list *filters, git_buf *data, - git_filter_stream *target); + git_writestream *target); GIT_EXTERN(int) git_filter_list_stream_file( git_filter_list *filters, git_repository *repo, const char *path, - git_filter_stream *target); + git_writestream *target); GIT_EXTERN(int) git_filter_list_stream_blob( git_filter_list *filters, git_blob *blob, - git_filter_stream *target); + git_writestream *target); /** * Free a git_filter_list diff --git a/include/git2/sys/filter.h b/include/git2/sys/filter.h index cc06c54ad..9b560fa75 100644 --- a/include/git2/sys/filter.h +++ b/include/git2/sys/filter.h @@ -208,18 +208,12 @@ typedef int (*git_filter_apply_fn)( const git_buf *from, const git_filter_source *src); -struct git_filter_stream { - int (*write)(git_filter_stream *stream, const char *buffer, size_t len); - int (*close)(git_filter_stream *stream); - void (*free)(git_filter_stream *stream); -}; - typedef int (*git_filter_stream_fn)( - git_filter_stream **out, + git_writestream **out, git_filter *self, void **payload, const git_filter_source *src, - git_filter_stream *next); + git_writestream *next); /** * Callback to clean up after filtering has been applied diff --git a/include/git2/types.h b/include/git2/types.h index 3ed97d189..c90ac4776 100644 --- a/include/git2/types.h +++ b/include/git2/types.h @@ -410,8 +410,14 @@ typedef enum { GIT_SUBMODULE_RECURSE_ONDEMAND = 2, } git_submodule_recurse_t; -/** A stream to write filters. */ -typedef struct git_filter_stream git_filter_stream; +/** A type to write in a streaming fashion, for example, for filters. */ +typedef struct git_writestream git_writestream; + +struct git_writestream { + int (*write)(git_writestream *stream, const char *buffer, size_t len); + int (*close)(git_writestream *stream); + void (*free)(git_writestream *stream); +}; /** @} */ GIT_END_DECL diff --git a/src/checkout.c b/src/checkout.c index 35129d771..1585a4fb5 100644 --- a/src/checkout.c +++ b/src/checkout.c @@ -1373,14 +1373,14 @@ static int mkpath2file( } struct checkout_stream { - git_filter_stream base; + git_writestream base; const char *path; int fd; int open; }; static int checkout_stream_write( - git_filter_stream *s, const char *buffer, size_t len) + git_writestream *s, const char *buffer, size_t len) { struct checkout_stream *stream = (struct checkout_stream *)s; int ret; @@ -1391,7 +1391,7 @@ static int checkout_stream_write( return ret; } -static int checkout_stream_close(git_filter_stream *s) +static int checkout_stream_close(git_writestream *s) { struct checkout_stream *stream = (struct checkout_stream *)s; assert(stream && stream->open); @@ -1400,7 +1400,7 @@ static int checkout_stream_close(git_filter_stream *s) return 0; } -static void checkout_stream_free(git_filter_stream *s) +static void checkout_stream_free(git_writestream *s) { GIT_UNUSED(s); } @@ -1456,7 +1456,7 @@ static int blob_content_to_file( writer.fd = fd; writer.open = 1; - error = git_filter_list_stream_blob(fl, blob, (git_filter_stream *)&writer); + error = git_filter_list_stream_blob(fl, blob, (git_writestream *)&writer); assert(writer.open == 0); diff --git a/src/filter.c b/src/filter.c index 2cae24e20..af5902e06 100644 --- a/src/filter.c +++ b/src/filter.c @@ -624,13 +624,13 @@ static int filter_list_out_buffer_from_raw( } struct buf_stream { - git_filter_stream base; + git_writestream base; git_buf *target; bool complete; }; static int buf_stream_write( - git_filter_stream *s, const char *buffer, size_t len) + git_writestream *s, const char *buffer, size_t len) { struct buf_stream *buf_stream = (struct buf_stream *)s; assert(buf_stream); @@ -640,7 +640,7 @@ static int buf_stream_write( return git_buf_put(buf_stream->target, buffer, len); } -static int buf_stream_close(git_filter_stream *s) +static int buf_stream_close(git_writestream *s) { struct buf_stream *buf_stream = (struct buf_stream *)s; assert(buf_stream); @@ -651,7 +651,7 @@ static int buf_stream_close(git_filter_stream *s) return 0; } -static void buf_stream_free(git_filter_stream *s) +static void buf_stream_free(git_writestream *s) { GIT_UNUSED(s); } @@ -683,7 +683,7 @@ int git_filter_list_apply_to_data( buf_stream_init(&writer, tgt); if ((error = git_filter_list_stream_data(filters, src, - (git_filter_stream *)&writer)) < 0) + (git_writestream *)&writer)) < 0) return error; assert(writer.complete); @@ -702,7 +702,7 @@ int git_filter_list_apply_to_file( buf_stream_init(&writer, out); if ((error = git_filter_list_stream_file( - filters, repo, path, (git_filter_stream *)&writer)) < 0) + filters, repo, path, (git_writestream *)&writer)) < 0) return error; assert(writer.complete); @@ -736,7 +736,7 @@ int git_filter_list_apply_to_blob( buf_stream_init(&writer, out); if ((error = git_filter_list_stream_blob( - filters, blob, (git_filter_stream *)&writer)) < 0) + filters, blob, (git_writestream *)&writer)) < 0) return error; assert(writer.complete); @@ -744,18 +744,18 @@ int git_filter_list_apply_to_blob( } struct proxy_stream { - git_filter_stream base; + git_writestream base; git_filter *filter; const git_filter_source *source; void **payload; git_buf input; git_buf temp_buf; git_buf *output; - git_filter_stream *target; + git_writestream *target; }; static int proxy_stream_write( - git_filter_stream *s, const char *buffer, size_t len) + git_writestream *s, const char *buffer, size_t len) { struct proxy_stream *proxy_stream = (struct proxy_stream *)s; assert(proxy_stream); @@ -763,7 +763,7 @@ static int proxy_stream_write( return git_buf_put(&proxy_stream->input, buffer, len); } -static int proxy_stream_close(git_filter_stream *s) +static int proxy_stream_close(git_writestream *s) { struct proxy_stream *proxy_stream = (struct proxy_stream *)s; git_buf *writebuf; @@ -794,7 +794,7 @@ static int proxy_stream_close(git_filter_stream *s) return error; } -static void proxy_stream_free(git_filter_stream *s) +static void proxy_stream_free(git_writestream *s) { struct proxy_stream *proxy_stream = (struct proxy_stream *)s; assert(proxy_stream); @@ -805,12 +805,12 @@ static void proxy_stream_free(git_filter_stream *s) } static int proxy_stream_init( - git_filter_stream **out, + git_writestream **out, git_filter *filter, git_buf *temp_buf, void **payload, const git_filter_source *source, - git_filter_stream *target) + git_writestream *target) { struct proxy_stream *proxy_stream = git__calloc(1, sizeof(struct proxy_stream)); GITERR_CHECK_ALLOC(proxy_stream); @@ -824,17 +824,17 @@ static int proxy_stream_init( proxy_stream->target = target; proxy_stream->output = temp_buf ? temp_buf : &proxy_stream->temp_buf; - *out = (git_filter_stream *)proxy_stream; + *out = (git_writestream *)proxy_stream; return 0; } static int stream_list_init( - git_filter_stream **out, + git_writestream **out, git_vector *streams, git_filter_list *filters, - git_filter_stream *target) + git_writestream *target) { - git_filter_stream *last_stream = target; + git_writestream *last_stream = target; size_t i; int error = 0; @@ -850,7 +850,7 @@ static int stream_list_init( size_t filter_idx = (filters->source.mode == GIT_FILTER_TO_WORKTREE) ? git_array_size(filters->filters) - 1 - i : i; git_filter_entry *fe = git_array_get(filters->filters, filter_idx); - git_filter_stream *filter_stream; + git_writestream *filter_stream; assert(fe->filter->stream || fe->filter->apply); @@ -879,7 +879,7 @@ static int stream_list_init( void stream_list_free(git_vector *streams) { - git_filter_stream *stream; + git_writestream *stream; size_t i; git_vector_foreach(streams, i, stream) @@ -894,13 +894,13 @@ int git_filter_list_stream_file( git_filter_list *filters, git_repository *repo, const char *path, - git_filter_stream *target) + git_writestream *target) { char buf[STREAM_BUFSIZE]; git_buf abspath = GIT_BUF_INIT; const char *base = repo ? git_repository_workdir(repo) : NULL; git_vector filter_streams = GIT_VECTOR_INIT; - git_filter_stream *stream_start; + git_writestream *stream_start; ssize_t readlen; int fd, error; @@ -935,10 +935,10 @@ done: int git_filter_list_stream_data( git_filter_list *filters, git_buf *data, - git_filter_stream *target) + git_writestream *target) { git_vector filter_streams = GIT_VECTOR_INIT; - git_filter_stream *stream_start; + git_writestream *stream_start; int error = 0; git_buf_sanitize(data); @@ -956,7 +956,7 @@ int git_filter_list_stream_data( int git_filter_list_stream_blob( git_filter_list *filters, git_blob *blob, - git_filter_stream *target) + git_writestream *target) { git_buf in = GIT_BUF_INIT; diff --git a/tests/filter/stream.c b/tests/filter/stream.c index ff5361a37..f7456b8a4 100644 --- a/tests/filter/stream.c +++ b/tests/filter/stream.c @@ -30,8 +30,8 @@ void test_filter_stream__cleanup(void) #define CHUNKSIZE 10240 struct compress_stream { - git_filter_stream base; - git_filter_stream *next; + git_writestream base; + git_writestream *next; git_filter_mode_t mode; char current; size_t current_chunk; @@ -78,7 +78,7 @@ static int compress_stream_write__inflated(struct compress_stream *stream, const return 0; } -static int compress_stream_write(git_filter_stream *s, const char *buffer, size_t len) +static int compress_stream_write(git_writestream *s, const char *buffer, size_t len) { struct compress_stream *stream = (struct compress_stream *)s; @@ -87,7 +87,7 @@ static int compress_stream_write(git_filter_stream *s, const char *buffer, size_ compress_stream_write__inflated(stream, buffer, len); } -static int compress_stream_close(git_filter_stream *s) +static int compress_stream_close(git_writestream *s) { struct compress_stream *stream = (struct compress_stream *)s; cl_assert_equal_i(0, stream->current_chunk); @@ -95,17 +95,17 @@ static int compress_stream_close(git_filter_stream *s) return 0; } -static void compress_stream_free(git_filter_stream *stream) +static void compress_stream_free(git_writestream *stream) { git__free(stream); } static int compress_filter_stream_init( - git_filter_stream **out, + git_writestream **out, git_filter *self, void **payload, const git_filter_source *src, - git_filter_stream *next) + git_writestream *next) { struct compress_stream *stream = git__calloc(1, sizeof(struct compress_stream)); cl_assert(stream); @@ -119,7 +119,7 @@ static int compress_filter_stream_init( stream->next = next; stream->mode = git_filter_source_mode(src); - *out = (git_filter_stream *)stream; + *out = (git_writestream *)stream; return 0; } From f7c0125f47ce23d52685648e076e663bc3a8939e Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Wed, 18 Feb 2015 09:28:07 -0500 Subject: [PATCH 07/13] filter streams: base -> parent --- src/filter.c | 16 ++++++++-------- tests/filter/stream.c | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/filter.c b/src/filter.c index af5902e06..08aa14b1b 100644 --- a/src/filter.c +++ b/src/filter.c @@ -624,7 +624,7 @@ static int filter_list_out_buffer_from_raw( } struct buf_stream { - git_writestream base; + git_writestream parent; git_buf *target; bool complete; }; @@ -660,9 +660,9 @@ static void buf_stream_init(struct buf_stream *writer, git_buf *target) { memset(writer, 0, sizeof(struct buf_stream)); - writer->base.write = buf_stream_write; - writer->base.close = buf_stream_close; - writer->base.free = buf_stream_free; + writer->parent.write = buf_stream_write; + writer->parent.close = buf_stream_close; + writer->parent.free = buf_stream_free; writer->target = target; git_buf_clear(target); @@ -744,7 +744,7 @@ int git_filter_list_apply_to_blob( } struct proxy_stream { - git_writestream base; + git_writestream parent; git_filter *filter; const git_filter_source *source; void **payload; @@ -815,9 +815,9 @@ static int proxy_stream_init( struct proxy_stream *proxy_stream = git__calloc(1, sizeof(struct proxy_stream)); GITERR_CHECK_ALLOC(proxy_stream); - proxy_stream->base.write = proxy_stream_write; - proxy_stream->base.close = proxy_stream_close; - proxy_stream->base.free = proxy_stream_free; + proxy_stream->parent.write = proxy_stream_write; + proxy_stream->parent.close = proxy_stream_close; + proxy_stream->parent.free = proxy_stream_free; proxy_stream->filter = filter; proxy_stream->payload = payload; proxy_stream->source = source; diff --git a/tests/filter/stream.c b/tests/filter/stream.c index f7456b8a4..4dafce42e 100644 --- a/tests/filter/stream.c +++ b/tests/filter/stream.c @@ -30,7 +30,7 @@ void test_filter_stream__cleanup(void) #define CHUNKSIZE 10240 struct compress_stream { - git_writestream base; + git_writestream parent; git_writestream *next; git_filter_mode_t mode; char current; @@ -113,9 +113,9 @@ static int compress_filter_stream_init( GIT_UNUSED(self); GIT_UNUSED(payload); - stream->base.write = compress_stream_write; - stream->base.close = compress_stream_close; - stream->base.free = compress_stream_free; + stream->parent.write = compress_stream_write; + stream->parent.close = compress_stream_close; + stream->parent.free = compress_stream_free; stream->next = next; stream->mode = git_filter_source_mode(src); From b49edddcd0af70caee7eb4905dc26199adccabfd Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Wed, 18 Feb 2015 09:40:52 -0500 Subject: [PATCH 08/13] checkout: let the stream writer close the fd --- src/checkout.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/checkout.c b/src/checkout.c index 1585a4fb5..e48a8557c 100644 --- a/src/checkout.c +++ b/src/checkout.c @@ -1397,7 +1397,7 @@ static int checkout_stream_close(git_writestream *s) assert(stream && stream->open); stream->open = 0; - return 0; + return p_close(stream->fd); } static void checkout_stream_free(git_writestream *s) @@ -1461,7 +1461,6 @@ static int blob_content_to_file( assert(writer.open == 0); git_filter_list_free(fl); - p_close(fd); if (error < 0) return error; From d4cf167515d3ed7b27c1358fc2e19b9caf66e8ad Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 19 Feb 2015 10:05:33 -0500 Subject: [PATCH 09/13] buffer: introduce git_buf_attach_notowned Provide a convenience function that creates a buffer that can be provided to callers but will not be freed via `git_buf_free`, so the buffer creator maintains the allocation lifecycle of the buffer's contents. --- src/attr.c | 5 ++--- src/blob.c | 10 ++++------ src/buffer.c | 14 ++++++++++++++ src/buffer.h | 6 ++++++ src/diff_driver.c | 9 ++++----- src/filter.c | 28 +++++----------------------- 6 files changed, 35 insertions(+), 37 deletions(-) diff --git a/src/attr.c b/src/attr.c index 44593da81..38420807a 100644 --- a/src/attr.c +++ b/src/attr.c @@ -282,9 +282,8 @@ static int system_attr_file( * a consumer. This allows them to treat this as a regular `git_buf`, * but their call to `git_buf_free` will not attempt to free it. */ - out->ptr = attr_session->sysdir.ptr; - out->size = attr_session->sysdir.size; - out->asize = 0; + git_buf_attach_notowned( + out, attr_session->sysdir.ptr, attr_session->sysdir.size); return 0; } diff --git a/src/blob.c b/src/blob.c index 30d5b705b..ba8769cdc 100644 --- a/src/blob.c +++ b/src/blob.c @@ -329,15 +329,13 @@ cleanup: int git_blob_is_binary(const git_blob *blob) { - git_buf content; + git_buf content = GIT_BUF_INIT; assert(blob); - content.ptr = blob->odb_object->buffer; - content.size = - min(blob->odb_object->cached.size, GIT_FILTER_BYTES_TO_CHECK_NUL); - content.asize = 0; - + git_buf_attach_notowned(&content, blob->odb_object->buffer, + min(blob->odb_object->cached.size, + GIT_FILTER_BYTES_TO_CHECK_NUL)); return git_buf_text_is_binary(&content); } diff --git a/src/buffer.c b/src/buffer.c index 3deb0329c..f633c5e02 100644 --- a/src/buffer.c +++ b/src/buffer.c @@ -500,6 +500,20 @@ void git_buf_attach(git_buf *buf, char *ptr, size_t asize) } } +void git_buf_attach_notowned(git_buf *buf, const char *ptr, size_t size) +{ + if (git_buf_is_allocated(buf)) + git_buf_free(buf); + + if (!size) { + git_buf_init(buf, 0); + } else { + buf->ptr = (char *)ptr; + buf->asize = 0; + buf->size = size; + } +} + int git_buf_join_n(git_buf *buf, char separator, int nbuf, ...) { va_list ap; diff --git a/src/buffer.h b/src/buffer.h index 52342e309..093ed9b60 100644 --- a/src/buffer.h +++ b/src/buffer.h @@ -74,6 +74,12 @@ extern void git_buf_swap(git_buf *buf_a, git_buf *buf_b); extern char *git_buf_detach(git_buf *buf); extern void git_buf_attach(git_buf *buf, char *ptr, size_t asize); +/* Populates a `git_buf` where the contents are not "owned" by the + * buffer, and calls to `git_buf_free` will not free the given buf. + */ +extern void git_buf_attach_notowned( + git_buf *buf, const char *ptr, size_t size); + /** * Test if there have been any reallocation failures with this git_buf. * diff --git a/src/diff_driver.c b/src/diff_driver.c index 7313ab573..049e6ef2a 100644 --- a/src/diff_driver.c +++ b/src/diff_driver.c @@ -418,14 +418,13 @@ void git_diff_driver_update_options( int git_diff_driver_content_is_binary( git_diff_driver *driver, const char *content, size_t content_len) { - git_buf search; - - search.ptr = (char *)content; - search.size = min(content_len, GIT_FILTER_BYTES_TO_CHECK_NUL); - search.asize = 0; + git_buf search = GIT_BUF_INIT; GIT_UNUSED(driver); + git_buf_attach_notowned(&search, content, + min(content_len, GIT_FILTER_BYTES_TO_CHECK_NUL)); + /* TODO: provide encoding / binary detection callbacks that can * be UTF-8 aware, etc. For now, instead of trying to be smart, * let's just use the simple NUL-byte detection that core git uses. diff --git a/src/filter.c b/src/filter.c index 08aa14b1b..4009a61f5 100644 --- a/src/filter.c +++ b/src/filter.c @@ -606,23 +606,6 @@ size_t git_filter_list_length(const git_filter_list *fl) return fl ? git_array_size(fl->filters) : 0; } -static int filter_list_out_buffer_from_raw( - git_buf *out, const void *ptr, size_t size) -{ - if (git_buf_is_allocated(out)) - git_buf_free(out); - - if (!size) { - git_buf_init(out, 0); - } else { - out->ptr = (char *)ptr; - out->asize = 0; - out->size = size; - } - - return 0; -} - struct buf_stream { git_writestream parent; git_buf *target; @@ -677,8 +660,10 @@ int git_filter_list_apply_to_data( git_buf_sanitize(tgt); git_buf_sanitize(src); - if (!filters) - return filter_list_out_buffer_from_raw(tgt, src->ptr, src->size); + if (!filters) { + git_buf_attach_notowned(tgt, src->ptr, src->size); + return 0; + } buf_stream_init(&writer, tgt); @@ -718,10 +703,7 @@ static int buf_from_blob(git_buf *out, git_blob *blob) return -1; } - out->ptr = (char *)git_blob_rawcontent(blob); - out->asize = 0; - out->size = (size_t)rawsize; - + git_buf_attach_notowned(out, git_blob_rawcontent(blob), (size_t)rawsize); return 0; } From 795eaccd667ce24c290b6211ca0c6a84f84e7c2b Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 19 Feb 2015 11:09:54 -0500 Subject: [PATCH 10/13] git_filter_opt_t -> git_filter_flag_t For consistency with the rest of the library, where an opt is an options *structure*. --- include/git2/filter.h | 10 +++++----- include/git2/sys/filter.h | 4 ++-- src/blob.c | 4 ++-- src/checkout.c | 4 ++-- src/crlf.c | 2 +- src/diff.c | 2 +- src/diff_file.c | 2 +- src/filter.c | 20 ++++++++++---------- src/filter.h | 2 +- src/repository.c | 2 +- tests/filter/crlf.c | 2 +- 11 files changed, 27 insertions(+), 27 deletions(-) diff --git a/include/git2/filter.h b/include/git2/filter.h index c0aaf1508..dc59e6341 100644 --- a/include/git2/filter.h +++ b/include/git2/filter.h @@ -39,9 +39,9 @@ typedef enum { * Filter option flags. */ typedef enum { - GIT_FILTER_OPT_DEFAULT = 0u, - GIT_FILTER_OPT_ALLOW_UNSAFE = (1u << 0), -} git_filter_opt_t; + GIT_FILTER_DEFAULT = 0u, + GIT_FILTER_ALLOW_UNSAFE = (1u << 0), +} git_filter_flag_t; /** * A filter that can transform file data @@ -83,7 +83,7 @@ typedef struct git_filter_list git_filter_list; * @param blob The blob to which the filter will be applied (if known) * @param path Relative path of the file to be filtered * @param mode Filtering direction (WT->ODB or ODB->WT) - * @param options Combination of `git_filter_opt_t` flags + * @param flags Combination of `git_filter_flag_t` flags * @return 0 on success (which could still return NULL if no filters are * needed for the requested file), <0 on error */ @@ -93,7 +93,7 @@ GIT_EXTERN(int) git_filter_list_load( git_blob *blob, /* can be NULL */ const char *path, git_filter_mode_t mode, - uint32_t options); + uint32_t flags); /** * Apply filter list to a data buffer. diff --git a/include/git2/sys/filter.h b/include/git2/sys/filter.h index 9b560fa75..5fd8d5566 100644 --- a/include/git2/sys/filter.h +++ b/include/git2/sys/filter.h @@ -123,9 +123,9 @@ GIT_EXTERN(const git_oid *) git_filter_source_id(const git_filter_source *src); GIT_EXTERN(git_filter_mode_t) git_filter_source_mode(const git_filter_source *src); /** - * Get the combination git_filter_opt_t options to be applied + * Get the combination git_filter_flag_t options to be applied */ -GIT_EXTERN(uint32_t) git_filter_source_options(const git_filter_source *src); +GIT_EXTERN(uint32_t) git_filter_source_flags(const git_filter_source *src); /* * struct git_filter diff --git a/src/blob.c b/src/blob.c index ba8769cdc..cf0329064 100644 --- a/src/blob.c +++ b/src/blob.c @@ -199,7 +199,7 @@ int git_blob__create_from_paths( /* Load the filters for writing this file to the ODB */ error = git_filter_list_load( &fl, repo, NULL, hint_path, - GIT_FILTER_TO_ODB, GIT_FILTER_OPT_DEFAULT); + GIT_FILTER_TO_ODB, GIT_FILTER_DEFAULT); if (error < 0) /* well, that didn't work */; @@ -357,7 +357,7 @@ int git_blob_filtered_content( if (!(error = git_filter_list_load( &fl, git_blob_owner(blob), blob, path, - GIT_FILTER_TO_WORKTREE, GIT_FILTER_OPT_DEFAULT))) { + GIT_FILTER_TO_WORKTREE, GIT_FILTER_DEFAULT))) { error = git_filter_list_apply_to_blob(out, fl, blob); diff --git a/src/checkout.c b/src/checkout.c index e48a8557c..a4dd20999 100644 --- a/src/checkout.c +++ b/src/checkout.c @@ -1441,7 +1441,7 @@ static int blob_content_to_file( if (!data->opts.disable_filters && (error = git_filter_list__load_with_attr_session( &fl, data->repo, &data->attr_session, blob, hint_path, - GIT_FILTER_TO_WORKTREE, GIT_FILTER_OPT_DEFAULT))) + GIT_FILTER_TO_WORKTREE, GIT_FILTER_DEFAULT))) return error; if (fl) @@ -2054,7 +2054,7 @@ static int checkout_write_merge( if ((error = git_filter_list__load_with_attr_session( &fl, data->repo, &data->attr_session, NULL, git_buf_cstr(&path_workdir), - GIT_FILTER_TO_WORKTREE, GIT_FILTER_OPT_DEFAULT)) < 0 || + GIT_FILTER_TO_WORKTREE, GIT_FILTER_DEFAULT)) < 0 || (error = git_filter_list_apply_to_data(&out_data, fl, &in_data)) < 0) goto done; } else { diff --git a/src/crlf.c b/src/crlf.c index c0a73990f..b5d1dbf32 100644 --- a/src/crlf.c +++ b/src/crlf.c @@ -302,7 +302,7 @@ static int crlf_check( return error; /* downgrade FAIL to WARN if ALLOW_UNSAFE option is used */ - if ((git_filter_source_options(src) & GIT_FILTER_OPT_ALLOW_UNSAFE) && + if ((git_filter_source_flags(src) & GIT_FILTER_ALLOW_UNSAFE) && ca.safe_crlf == GIT_SAFE_CRLF_FAIL) ca.safe_crlf = GIT_SAFE_CRLF_WARN; } diff --git a/src/diff.c b/src/diff.c index 07eae03e7..815351b21 100644 --- a/src/diff.c +++ b/src/diff.c @@ -600,7 +600,7 @@ int git_diff__oid_for_entry( error = -1; } else if (!(error = git_filter_list_load( &fl, diff->repo, NULL, entry.path, - GIT_FILTER_TO_ODB, GIT_FILTER_OPT_ALLOW_UNSAFE))) + GIT_FILTER_TO_ODB, GIT_FILTER_ALLOW_UNSAFE))) { int fd = git_futils_open_ro(full_path.ptr); if (fd < 0) diff --git a/src/diff_file.c b/src/diff_file.c index 96be0942b..f7061ae83 100644 --- a/src/diff_file.c +++ b/src/diff_file.c @@ -302,7 +302,7 @@ static int diff_file_content_load_workdir_file( if ((error = git_filter_list_load( &fl, fc->repo, NULL, fc->file->path, - GIT_FILTER_TO_ODB, GIT_FILTER_OPT_ALLOW_UNSAFE)) < 0) + GIT_FILTER_TO_ODB, GIT_FILTER_ALLOW_UNSAFE)) < 0) goto cleanup; /* if there are no filters, try to mmap the file */ diff --git a/src/filter.c b/src/filter.c index 4009a61f5..88877fead 100644 --- a/src/filter.c +++ b/src/filter.c @@ -23,7 +23,7 @@ struct git_filter_source { git_oid oid; /* zero if unknown (which is likely) */ uint16_t filemode; /* zero if unknown */ git_filter_mode_t mode; - uint32_t options; + uint32_t flags; }; typedef struct { @@ -372,9 +372,9 @@ git_filter_mode_t git_filter_source_mode(const git_filter_source *src) return src->mode; } -uint32_t git_filter_source_options(const git_filter_source *src) +uint32_t git_filter_source_flags(const git_filter_source *src) { - return src->options; + return src->flags; } static int filter_list_new( @@ -394,7 +394,7 @@ static int filter_list_new( fl->source.repo = src->repo; fl->source.path = fl->path; fl->source.mode = src->mode; - fl->source.options = src->options; + fl->source.flags = src->flags; *out = fl; return 0; @@ -449,13 +449,13 @@ int git_filter_list_new( git_filter_list **out, git_repository *repo, git_filter_mode_t mode, - uint32_t options) + uint32_t flags) { git_filter_source src = { 0 }; src.repo = repo; src.path = NULL; src.mode = mode; - src.options = options; + src.flags = flags; return filter_list_new(out, &src); } @@ -466,7 +466,7 @@ int git_filter_list__load_with_attr_session( git_blob *blob, /* can be NULL */ const char *path, git_filter_mode_t mode, - uint32_t options) + uint32_t flags) { int error = 0; git_filter_list *fl = NULL; @@ -481,7 +481,7 @@ int git_filter_list__load_with_attr_session( src.repo = repo; src.path = path; src.mode = mode; - src.options = options; + src.flags = flags; if (blob) git_oid_cpy(&src.oid, git_blob_id(blob)); @@ -543,10 +543,10 @@ int git_filter_list_load( git_blob *blob, /* can be NULL */ const char *path, git_filter_mode_t mode, - uint32_t options) + uint32_t flags) { return git_filter_list__load_with_attr_session( - filters, repo, NULL, blob, path, mode, options); + filters, repo, NULL, blob, path, mode, flags); } void git_filter_list__set_temp_buf(git_filter_list *fl, git_buf *temp_buf) diff --git a/src/filter.h b/src/filter.h index 53f3afdbb..a38c6c8fa 100644 --- a/src/filter.h +++ b/src/filter.h @@ -36,7 +36,7 @@ extern int git_filter_list__load_with_attr_session( git_blob *blob, /* can be NULL */ const char *path, git_filter_mode_t mode, - uint32_t options); + uint32_t flags); /* * Available filters diff --git a/src/repository.c b/src/repository.c index c9275078f..23c99b0f0 100644 --- a/src/repository.c +++ b/src/repository.c @@ -1849,7 +1849,7 @@ int git_repository_hashfile( if (strlen(as_path) > 0) { error = git_filter_list_load( &fl, repo, NULL, as_path, - GIT_FILTER_TO_ODB, GIT_FILTER_OPT_DEFAULT); + GIT_FILTER_TO_ODB, GIT_FILTER_DEFAULT); if (error < 0) return error; } else { diff --git a/tests/filter/crlf.c b/tests/filter/crlf.c index a31dac965..406d3b6b0 100644 --- a/tests/filter/crlf.c +++ b/tests/filter/crlf.c @@ -123,7 +123,7 @@ void test_filter_crlf__with_safecrlf_and_unsafe_allowed(void) cl_repo_set_bool(g_repo, "core.safecrlf", true); cl_git_pass(git_filter_list_new( - &fl, g_repo, GIT_FILTER_TO_ODB, GIT_FILTER_OPT_ALLOW_UNSAFE)); + &fl, g_repo, GIT_FILTER_TO_ODB, GIT_FILTER_ALLOW_UNSAFE)); crlf = git_filter_lookup(GIT_FILTER_CRLF); cl_assert(crlf != NULL); From d05218b06ff201cd373fc764e0d87af67f7932c7 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 19 Feb 2015 11:25:26 -0500 Subject: [PATCH 11/13] filter: add `git_filter_list__load_ext` Refactor `git_filter_list__load_with_attr_reader` into `git_filter_list__load_ext`, which takes a `git_filter_options`. --- src/checkout.c | 18 ++++++++++++------ src/filter.c | 18 +++++++++++------- src/filter.h | 12 +++++++++--- 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/src/checkout.c b/src/checkout.c index a4dd20999..2c47147e8 100644 --- a/src/checkout.c +++ b/src/checkout.c @@ -1416,6 +1416,7 @@ static int blob_content_to_file( int flags = data->opts.file_open_flags; mode_t file_mode = data->opts.file_mode ? data->opts.file_mode : entry_filemode; + git_filter_options filter_opts = GIT_FILTER_OPTIONS_INIT; struct checkout_stream writer; mode_t mode; git_filter_list *fl = NULL; @@ -1438,10 +1439,12 @@ static int blob_content_to_file( return fd; } + filter_opts.attr_session = &data->attr_session; + if (!data->opts.disable_filters && - (error = git_filter_list__load_with_attr_session( - &fl, data->repo, &data->attr_session, blob, hint_path, - GIT_FILTER_TO_WORKTREE, GIT_FILTER_DEFAULT))) + (error = git_filter_list__load_ext( + &fl, data->repo, blob, hint_path, + GIT_FILTER_TO_WORKTREE, &filter_opts))) return error; if (fl) @@ -2003,6 +2006,7 @@ static int checkout_write_merge( git_merge_file_result result = {0}; git_filebuf output = GIT_FILEBUF_INIT; git_filter_list *fl = NULL; + git_filter_options filter_opts = GIT_FILTER_OPTIONS_INIT; int error = 0; if (data->opts.checkout_strategy & GIT_CHECKOUT_CONFLICT_STYLE_DIFF3) @@ -2052,9 +2056,11 @@ static int checkout_write_merge( in_data.ptr = (char *)result.ptr; in_data.size = result.len; - if ((error = git_filter_list__load_with_attr_session( - &fl, data->repo, &data->attr_session, NULL, git_buf_cstr(&path_workdir), - GIT_FILTER_TO_WORKTREE, GIT_FILTER_DEFAULT)) < 0 || + filter_opts.attr_session = &data->attr_session; + + if ((error = git_filter_list__load_ext( + &fl, data->repo, NULL, git_buf_cstr(&path_workdir), + GIT_FILTER_TO_WORKTREE, &filter_opts)) < 0 || (error = git_filter_list_apply_to_data(&out_data, fl, &in_data)) < 0) goto done; } else { diff --git a/src/filter.c b/src/filter.c index 88877fead..646f1bc11 100644 --- a/src/filter.c +++ b/src/filter.c @@ -459,14 +459,13 @@ int git_filter_list_new( return filter_list_new(out, &src); } -int git_filter_list__load_with_attr_session( +int git_filter_list__load_ext( git_filter_list **filters, git_repository *repo, - git_attr_session *attr_session, git_blob *blob, /* can be NULL */ const char *path, git_filter_mode_t mode, - uint32_t flags) + git_filter_options *filter_opts) { int error = 0; git_filter_list *fl = NULL; @@ -481,7 +480,8 @@ int git_filter_list__load_with_attr_session( src.repo = repo; src.path = path; src.mode = mode; - src.flags = flags; + src.flags = filter_opts->flags; + if (blob) git_oid_cpy(&src.oid, git_blob_id(blob)); @@ -494,7 +494,7 @@ int git_filter_list__load_with_attr_session( if (fdef->nattrs > 0) { error = filter_list_check_attributes( - &values, repo, attr_session, fdef, &src); + &values, repo, filter_opts->attr_session, fdef, &src); if (error == GIT_ENOTFOUND) { error = 0; @@ -545,8 +545,12 @@ int git_filter_list_load( git_filter_mode_t mode, uint32_t flags) { - return git_filter_list__load_with_attr_session( - filters, repo, NULL, blob, path, mode, flags); + git_filter_options filter_opts = GIT_FILTER_OPTIONS_INIT; + + filter_opts.flags = flags; + + return git_filter_list__load_ext( + filters, repo, blob, path, mode, &filter_opts); } void git_filter_list__set_temp_buf(git_filter_list *fl, git_buf *temp_buf) diff --git a/src/filter.h b/src/filter.h index a38c6c8fa..85ef4a6f0 100644 --- a/src/filter.h +++ b/src/filter.h @@ -24,19 +24,25 @@ typedef enum { GIT_CRLF_AUTO, } git_crlf_t; +typedef struct { + git_attr_session *attr_session; + uint32_t flags; +} git_filter_options; + +#define GIT_FILTER_OPTIONS_INIT {0} + extern void git_filter_list__set_temp_buf( git_filter_list *fl, git_buf *temp_buf); extern void git_filter_free(git_filter *filter); -extern int git_filter_list__load_with_attr_session( +extern int git_filter_list__load_ext( git_filter_list **filters, git_repository *repo, - git_attr_session *attr_session, git_blob *blob, /* can be NULL */ const char *path, git_filter_mode_t mode, - uint32_t flags); + git_filter_options *filter_opts); /* * Available filters From 9c9aa1bad3b364f7dfdccd5f6abe880f7e18c499 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 19 Feb 2015 11:32:55 -0500 Subject: [PATCH 12/13] filter: take `temp_buf` in `git_filter_options` --- src/checkout.c | 5 ++--- src/filter.c | 13 ++++++------- src/filter.h | 4 +--- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/checkout.c b/src/checkout.c index 2c47147e8..f71be26f9 100644 --- a/src/checkout.c +++ b/src/checkout.c @@ -1440,6 +1440,7 @@ static int blob_content_to_file( } filter_opts.attr_session = &data->attr_session; + filter_opts.temp_buf = &data->tmp; if (!data->opts.disable_filters && (error = git_filter_list__load_ext( @@ -1447,9 +1448,6 @@ static int blob_content_to_file( GIT_FILTER_TO_WORKTREE, &filter_opts))) return error; - if (fl) - git_filter_list__set_temp_buf(fl, &data->tmp); - /* setup the writer */ memset(&writer, 0, sizeof(struct checkout_stream)); writer.base.write = checkout_stream_write; @@ -2057,6 +2055,7 @@ static int checkout_write_merge( in_data.size = result.len; filter_opts.attr_session = &data->attr_session; + filter_opts.temp_buf = &data->tmp; if ((error = git_filter_list__load_ext( &fl, data->repo, NULL, git_buf_cstr(&path_workdir), diff --git a/src/filter.c b/src/filter.c index 646f1bc11..4fbf84f6a 100644 --- a/src/filter.c +++ b/src/filter.c @@ -517,8 +517,12 @@ int git_filter_list__load_ext( else if (error < 0) break; else { - if (!fl && (error = filter_list_new(&fl, &src)) < 0) - return error; + if (!fl) { + if ((error = filter_list_new(&fl, &src)) < 0) + return error; + + fl->temp_buf = filter_opts->temp_buf; + } fe = git_array_alloc(fl->filters); GITERR_CHECK_ALLOC(fe); @@ -553,11 +557,6 @@ int git_filter_list_load( filters, repo, blob, path, mode, &filter_opts); } -void git_filter_list__set_temp_buf(git_filter_list *fl, git_buf *temp_buf) -{ - fl->temp_buf = temp_buf; -} - void git_filter_list_free(git_filter_list *fl) { uint32_t i; diff --git a/src/filter.h b/src/filter.h index 85ef4a6f0..5062afba5 100644 --- a/src/filter.h +++ b/src/filter.h @@ -26,14 +26,12 @@ typedef enum { typedef struct { git_attr_session *attr_session; + git_buf *temp_buf; uint32_t flags; } git_filter_options; #define GIT_FILTER_OPTIONS_INIT {0} -extern void git_filter_list__set_temp_buf( - git_filter_list *fl, git_buf *temp_buf); - extern void git_filter_free(git_filter *filter); extern int git_filter_list__load_ext( From feb0e022867552c039c02fe39db7b7c3d63ae327 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 19 Feb 2015 12:14:06 -0500 Subject: [PATCH 13/13] tests: separate INVASIVE filesystem tests Introduce GITTEST_INVASIVE_FS_STRUCTURE for things that are invasive to your filesystem structure (like creating folders at your filesystem root) and GITTEST_INVASIVE_FS_SIZE for things that write lots of data. --- .travis.yml | 1 + appveyor.yml | 3 ++- tests/filter/stream.c | 2 +- tests/repo/init.c | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index fc513458b..68b29b1e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ compiler: env: global: - secure: "YnhS+8n6B+uoyaYfaJ3Lei7cSJqHDPiKJCKFIF2c87YDfmCvAJke8QtE7IzjYDs7UFkTCM4ox+ph2bERUrxZbSCyEkHdjIZpKuMJfYWja/jgMqTMxdyOH9y8JLFbZsSXDIXDwqBlC6vVyl1fP90M35wuWcNTs6tctfVWVofEFbs=" + - GITTEST_INVASIVE_FS_SIZE=1 matrix: - OPTIONS="-DTHREADSAFE=ON -DCMAKE_BUILD_TYPE=Release" - OPTIONS="-DTHREADSAFE=OFF -DBUILD_EXAMPLES=ON" diff --git a/appveyor.yml b/appveyor.yml index 8ac6728c3..d155485fd 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -3,7 +3,8 @@ branches: only: - master environment: - GITTEST_INVASIVE_FILESYSTEM: 1 + GITTEST_INVASIVE_FS_STRUCTURE: 1 + GITTEST_INVASIVE_FS_SIZE: 1 matrix: - GENERATOR: "Visual Studio 11" diff --git a/tests/filter/stream.c b/tests/filter/stream.c index 4dafce42e..603f19494 100644 --- a/tests/filter/stream.c +++ b/tests/filter/stream.c @@ -214,7 +214,7 @@ void test_filter_stream__smallfile(void) /* optionally write a 500 MB file through the compression stream */ void test_filter_stream__bigfile(void) { - if (!cl_getenv("GITTEST_INVASIVE_FILESYSTEM")) + if (!cl_getenv("GITTEST_INVASIVE_FS_SIZE")) cl_skip(); test_stream(51200); diff --git a/tests/repo/init.c b/tests/repo/init.c index 91747c9f5..076156817 100644 --- a/tests/repo/init.c +++ b/tests/repo/init.c @@ -722,7 +722,7 @@ void test_repo_init__at_filesystem_root(void) git_buf root = GIT_BUF_INIT; int root_len; - if (!cl_getenv("GITTEST_INVASIVE_FILESYSTEM")) + if (!cl_getenv("GITTEST_INVASIVE_FS_STRUCTURE")) cl_skip(); root_len = git_path_root(sandbox);