diff --git a/examples/network/Makefile b/examples/network/Makefile index ed0c2099f..c21869ac9 100644 --- a/examples/network/Makefile +++ b/examples/network/Makefile @@ -2,7 +2,7 @@ default: all CC = gcc CFLAGS += -g -CFLAGS += -I../../include -L../../ -lgit2 +CFLAGS += -I../../include -L../../ -lgit2 -lpthread OBJECTS = \ git2.o \ diff --git a/examples/network/fetch.c b/examples/network/fetch.c index cdd4a4662..d4a39746f 100644 --- a/examples/network/fetch.c +++ b/examples/network/fetch.c @@ -3,95 +3,108 @@ #include #include #include +#include -static int rename_packfile(char *packname, git_indexer *idx) +struct dl_data { + git_remote *remote; + git_off_t *bytes; + git_indexer_stats *stats; + int ret; + int finished; +}; + +static void *download(void *ptr) { - char path[GIT_PATH_MAX], oid[GIT_OID_HEXSZ + 1], *slash; - int ret; + struct dl_data *data = (struct dl_data *)ptr; - strcpy(path, packname); - slash = strrchr(path, '/'); + // Connect to the remote end specifying that we want to fetch + // information from it. + if (git_remote_connect(data->remote, GIT_DIR_FETCH) < 0) { + data->ret = -1; + goto exit; + } - if (!slash) - return GIT_EINVALIDARGS; + // Download the packfile and index it. This function updates the + // amount of received data and the indexer stats which lets you + // inform the user about progress. + if (git_remote_download(data->remote, data->bytes, data->stats) < 0) { + data->ret = -1; + goto exit; + } - memset(oid, 0x0, sizeof(oid)); - // The name of the packfile is given by it's hash which you can get - // with git_indexer_hash after the index has been written out to - // disk. Rename the packfile to its "real" name in the same - // directory as it was originally (libgit2 stores it in the folder - // where the packs go, so a rename in place is the right thing to do here - git_oid_fmt(oid, git_indexer_hash(idx)); - ret = sprintf(slash + 1, "pack-%s.pack", oid); - if(ret < 0) - return GIT_EOSERR; + data->ret = 0; - printf("Renaming pack to %s\n", path); - return rename(packname, path); +exit: + data->finished = 1; + pthread_exit(&data->ret); +} + +int update_cb(const char *refname, const git_oid *a, const git_oid *b) +{ + const char *action; + char a_str[GIT_OID_HEXSZ+1], b_str[GIT_OID_HEXSZ+1]; + + git_oid_fmt(b_str, b); + b_str[GIT_OID_HEXSZ] = '\0'; + + if (git_oid_iszero(a)) { + printf("[new] %.20s %s\n", b_str, refname); + } else { + git_oid_fmt(a_str, a); + a_str[GIT_OID_HEXSZ] = '\0'; + printf("[updated] %.10s..%.10s %s\n", a_str, b_str, refname); + } + + return 0; } int fetch(git_repository *repo, int argc, char **argv) { git_remote *remote = NULL; - git_indexer *idx = NULL; + git_off_t bytes = 0; git_indexer_stats stats; - int error; - char *packname = NULL; + pthread_t worker; + struct dl_data data; - // Get the remote and connect to it + // Figure out whether it's a named remote or a URL printf("Fetching %s\n", argv[1]); - error = git_remote_new(&remote, repo, argv[1], NULL); - if (error < GIT_SUCCESS) - return error; - - error = git_remote_connect(remote, GIT_DIR_FETCH); - if (error < GIT_SUCCESS) - return error; - - // Download the packfile from the server. As we don't know its hash - // yet, it will get a temporary filename - error = git_remote_download(&packname, remote); - if (error < GIT_SUCCESS) - return error; - - // No error and a NULL packname means no packfile was needed - if (packname != NULL) { - printf("The packname is %s\n", packname); - - // Create a new instance indexer - error = git_indexer_new(&idx, packname); - if (error < GIT_SUCCESS) - return error; - - // This should be run in paralel, but it'd be too complicated for the example - error = git_indexer_run(idx, &stats); - if (error < GIT_SUCCESS) - return error; - - printf("Received %d objects\n", stats.total); - - // Write the index file. The index will be stored with the - // correct filename - error = git_indexer_write(idx); - if (error < GIT_SUCCESS) - return error; - - error = rename_packfile(packname, idx); - if (error < GIT_SUCCESS) - return error; + if (git_remote_load(&remote, repo, argv[1]) < 0) { + if (git_remote_new(&remote, repo, argv[1], NULL) < 0) + return -1; } + // Set up the information for the background worker thread + data.remote = remote; + data.bytes = &bytes; + data.stats = &stats; + data.ret = 0; + data.finished = 0; + memset(&stats, 0, sizeof(stats)); + + pthread_create(&worker, NULL, download, &data); + + // Loop while the worker thread is still running. Here we show processed + // and total objects in the pack and the amount of received + // data. Most frontends will probably want to show a percentage and + // the download rate. + do { + usleep(10000); + printf("\rReceived %d/%d objects in %d bytes", stats.processed, stats.total, bytes); + } while (!data.finished); + printf("\rReceived %d/%d objects in %d bytes\n", stats.processed, stats.total, bytes); + // Update the references in the remote's namespace to point to the // right commits. This may be needed even if there was no packfile // to download, which can happen e.g. when the branches have been // changed but all the neede objects are available locally. - error = git_remote_update_tips(remote); - if (error < GIT_SUCCESS) - return error; + if (git_remote_update_tips(remote, update_cb) < 0) + return -1; - free(packname); - git_indexer_free(idx); git_remote_free(remote); - return GIT_SUCCESS; + return 0; + +on_error: + git_remote_free(remote); + return -1; } diff --git a/include/git2/indexer.h b/include/git2/indexer.h index a70fab214..14bd0e402 100644 --- a/include/git2/indexer.h +++ b/include/git2/indexer.h @@ -41,7 +41,7 @@ GIT_EXTERN(int) git_indexer_stream_new(git_indexer_stream **out, const char *git * @param size the size of the data * @param stats stat storage */ -GIT_EXTERN(int) git_indexer_stream_add(git_indexer_stream *idx, void *data, size_t size, git_indexer_stats *stats); +GIT_EXTERN(int) git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t size, git_indexer_stats *stats); /** * Finalize the pack and index diff --git a/include/git2/remote.h b/include/git2/remote.h index e6537ec52..09b927e28 100644 --- a/include/git2/remote.h +++ b/include/git2/remote.h @@ -11,6 +11,7 @@ #include "repository.h" #include "refspec.h" #include "net.h" +#include "indexer.h" /** * @file git2/remote.h @@ -150,7 +151,7 @@ GIT_EXTERN(int) git_remote_ls(git_remote *remote, git_headlist_cb list_cb, void * @param filename where to store the temproray filename * @return GIT_SUCCESS or an error code */ -GIT_EXTERN(int) git_remote_download(char **filename, git_remote *remote); +GIT_EXTERN(int) git_remote_download(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats); /** * Check whether the remote is connected @@ -182,12 +183,10 @@ GIT_EXTERN(void) git_remote_free(git_remote *remote); /** * Update the tips to the new state * - * Make sure that you only call this once you've successfully indexed - * or expanded the packfile. - * * @param remote the remote to update + * @param cb callback to run on each ref update. 'a' is the old value, 'b' is then new value */ -GIT_EXTERN(int) git_remote_update_tips(git_remote *remote); +GIT_EXTERN(int) git_remote_update_tips(git_remote *remote, int (*cb)(const char *refname, const git_oid *a, const git_oid *b)); /** * Return whether a string is a valid remote URL diff --git a/src/fetch.c b/src/fetch.c index 57a6d0265..6fe1b5676 100644 --- a/src/fetch.c +++ b/src/fetch.c @@ -9,6 +9,7 @@ #include "git2/oid.h" #include "git2/refs.h" #include "git2/revwalk.h" +#include "git2/indexer.h" #include "common.h" #include "transport.h" @@ -101,30 +102,27 @@ int git_fetch_negotiate(git_remote *remote) return t->negotiate_fetch(t, remote->repo, &remote->refs); } -int git_fetch_download_pack(char **out, git_remote *remote) +int git_fetch_download_pack(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats) { - if(!remote->need_pack) { - *out = NULL; + if(!remote->need_pack) return 0; - } - return remote->transport->download_pack(out, remote->transport, remote->repo); + return remote->transport->download_pack(remote->transport, remote->repo, bytes, stats); } /* Receiving data from a socket and storing it is pretty much the same for git and HTTP */ int git_fetch__download_pack( - char **out, const char *buffered, size_t buffered_size, GIT_SOCKET fd, - git_repository *repo) + git_repository *repo, + git_off_t *bytes, + git_indexer_stats *stats) { - git_filebuf file = GIT_FILEBUF_INIT; - int error; + int recvd; char buff[1024]; - git_buf path = GIT_BUF_INIT; - static const char suff[] = "/objects/pack/pack-received"; gitno_buffer buf; + git_indexer_stream *idx; gitno_buffer_setup(&buf, buff, sizeof(buff), fd); @@ -133,41 +131,74 @@ int git_fetch__download_pack( return -1; } - if (git_buf_joinpath(&path, repo->path_repository, suff) < 0) + if (git_indexer_stream_new(&idx, git_repository_path(repo)) < 0) + return -1; + + memset(stats, 0, sizeof(git_indexer_stats)); + if (git_indexer_stream_add(idx, buffered, buffered_size, stats) < 0) goto on_error; - if (git_filebuf_open(&file, path.ptr, GIT_FILEBUF_TEMPORARY) < 0) - goto on_error; + *bytes = buffered_size; - /* Part of the packfile has been received, don't loose it */ - if (git_filebuf_write(&file, buffered, buffered_size) < 0) - goto on_error; - - while (1) { - if (git_filebuf_write(&file, buf.data, buf.offset) < 0) + do { + if (git_indexer_stream_add(idx, buf.data, buf.offset, stats) < 0) goto on_error; gitno_consume_n(&buf, buf.offset); - error = gitno_recv(&buf); - if (error < GIT_SUCCESS) + if ((recvd = gitno_recv(&buf)) < 0) goto on_error; - if (error == 0) /* Orderly shutdown */ - break; - } - *out = git__strdup(file.path_lock); - if (*out == NULL) + *bytes += recvd; + } while(recvd > 0); + + if (git_indexer_stream_finalize(idx, stats)) goto on_error; - /* A bit dodgy, but we need to keep the pack at the temporary path */ - if (git_filebuf_commit_at(&file, file.path_lock, GIT_PACK_FILE_MODE) < 0) - goto on_error; - - git_buf_free(&path); - + git_indexer_stream_free(idx); return 0; + on_error: - git_buf_free(&path); - git_filebuf_cleanup(&file); + git_indexer_stream_free(idx); + return -1; +} + +int git_fetch_setup_walk(git_revwalk **out, git_repository *repo) +{ + git_revwalk *walk; + git_strarray refs; + unsigned int i; + git_reference *ref; + + if (git_reference_listall(&refs, repo, GIT_REF_LISTALL) < 0) + return -1; + + if (git_revwalk_new(&walk, repo) < 0) + return -1; + + git_revwalk_sorting(walk, GIT_SORT_TIME); + + for (i = 0; i < refs.count; ++i) { + /* No tags */ + if (!git__prefixcmp(refs.strings[i], GIT_REFS_TAGS_DIR)) + continue; + + if (git_reference_lookup(&ref, repo, refs.strings[i]) < 0) + goto on_error; + + if (git_reference_type(ref) == GIT_REF_SYMBOLIC) + continue; + if (git_revwalk_push(walk, git_reference_oid(ref)) < 0) + goto on_error; + + git_reference_free(ref); + } + + git_strarray_free(&refs); + *out = walk; + return 0; + +on_error: + git_reference_free(ref); + git_strarray_free(&refs); return -1; } diff --git a/src/fetch.h b/src/fetch.h index c1ab84034..b3192a563 100644 --- a/src/fetch.h +++ b/src/fetch.h @@ -10,9 +10,10 @@ #include "netops.h" int git_fetch_negotiate(git_remote *remote); -int git_fetch_download_pack(char **out, git_remote *remote); +int git_fetch_download_pack(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats); -int git_fetch__download_pack(char **out, const char *buffered, size_t buffered_size, - GIT_SOCKET fd, git_repository *repo); +int git_fetch__download_pack(const char *buffered, size_t buffered_size, GIT_SOCKET fd, + git_repository *repo, git_off_t *bytes, git_indexer_stats *stats); +int git_fetch_setup_walk(git_revwalk **out, git_repository *repo); #endif diff --git a/src/indexer.c b/src/indexer.c index 1834d9884..1f8b512c2 100644 --- a/src/indexer.c +++ b/src/indexer.c @@ -288,7 +288,7 @@ on_error: return -1; } -int git_indexer_stream_add(git_indexer_stream *idx, void *data, size_t size, git_indexer_stats *stats) +int git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t size, git_indexer_stats *stats) { int error; struct git_pack_header hdr; diff --git a/src/pkt.c b/src/pkt.c index ee113cd46..2c9fe27da 100644 --- a/src/pkt.c +++ b/src/pkt.c @@ -287,19 +287,6 @@ static int buffer_want_with_caps(git_remote_head *head, git_transport_caps *caps return git_buf_printf(buf, "%04xwant %s%c%s\n", len, oid, 0, capstr); } -static int send_want_with_caps(git_remote_head *head, git_transport_caps *caps, GIT_SOCKET fd) -{ - git_buf buf = GIT_BUF_INIT; - int ret; - - if (buffer_want_with_caps(head, caps, &buf) < 0) - return -1; - - ret = gitno_send(fd, buf.ptr, buf.size, 0); - git_buf_free(&buf); - return ret; -} - /* * All "want" packets have the same length and format, so what we do * is overwrite the OID each time. @@ -341,47 +328,6 @@ int git_pkt_buffer_wants(const git_vector *refs, git_transport_caps *caps, git_b return git_pkt_buffer_flush(buf); } -int git_pkt_send_wants(const git_vector *refs, git_transport_caps *caps, GIT_SOCKET fd) -{ - unsigned int i = 0; - char buf[sizeof(pkt_want_prefix) + GIT_OID_HEXSZ + 1]; - git_remote_head *head; - - memcpy(buf, pkt_want_prefix, strlen(pkt_want_prefix)); - buf[sizeof(buf) - 2] = '\n'; - buf[sizeof(buf) - 1] = '\0'; - - /* If there are common caps, find the first one */ - if (caps->common) { - for (; i < refs->length; ++i) { - head = refs->contents[i]; - if (head->local) - continue; - else - break; - } - - if (send_want_with_caps(refs->contents[i], caps, fd) < 0) - return -1; - - /* Increase it here so it's correct whether we run this or not */ - i++; - } - - /* Continue from where we left off */ - for (; i < refs->length; ++i) { - head = refs->contents[i]; - if (head->local) - continue; - - git_oid_fmt(buf + strlen(pkt_want_prefix), &head->oid); - if (gitno_send(fd, buf, strlen(buf), 0) < 0) - return -1; - } - - return git_pkt_send_flush(fd); -} - int git_pkt_buffer_have(git_oid *oid, git_buf *buf) { char oidhex[GIT_OID_HEXSZ + 1]; @@ -391,21 +337,7 @@ int git_pkt_buffer_have(git_oid *oid, git_buf *buf) return git_buf_printf(buf, "%s%s\n", pkt_have_prefix, oidhex); } -int git_pkt_send_have(git_oid *oid, GIT_SOCKET fd) -{ - char buf[] = "0032have 0000000000000000000000000000000000000000\n"; - - git_oid_fmt(buf + strlen(pkt_have_prefix), oid); - return gitno_send(fd, buf, strlen(buf), 0); -} - - int git_pkt_buffer_done(git_buf *buf) { return git_buf_puts(buf, pkt_done_str); } - -int git_pkt_send_done(GIT_SOCKET fd) -{ - return gitno_send(fd, pkt_done_str, strlen(pkt_done_str), 0); -} diff --git a/src/pkt.h b/src/pkt.h index 1f8d62e1a..7e696f70f 100644 --- a/src/pkt.h +++ b/src/pkt.h @@ -68,11 +68,8 @@ int git_pkt_parse_line(git_pkt **head, const char *line, const char **out, size_ int git_pkt_buffer_flush(git_buf *buf); int git_pkt_send_flush(GIT_SOCKET s); int git_pkt_buffer_done(git_buf *buf); -int git_pkt_send_done(GIT_SOCKET s); int git_pkt_buffer_wants(const git_vector *refs, git_transport_caps *caps, git_buf *buf); -int git_pkt_send_wants(const git_vector *refs, git_transport_caps *caps, GIT_SOCKET fd); int git_pkt_buffer_have(git_oid *oid, git_buf *buf); -int git_pkt_send_have(git_oid *oid, GIT_SOCKET fd); void git_pkt_free(git_pkt *pkt); #endif diff --git a/src/remote.c b/src/remote.c index b48a23339..e1937df85 100644 --- a/src/remote.c +++ b/src/remote.c @@ -297,23 +297,24 @@ int git_remote_ls(git_remote *remote, git_headlist_cb list_cb, void *payload) return remote->transport->ls(remote->transport, list_cb, payload); } -int git_remote_download(char **filename, git_remote *remote) +int git_remote_download(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats) { int error; - assert(filename && remote); + assert(remote && bytes && stats); if ((error = git_fetch_negotiate(remote)) < 0) return error; - return git_fetch_download_pack(filename, remote); + return git_fetch_download_pack(remote, bytes, stats); } -int git_remote_update_tips(git_remote *remote) +int git_remote_update_tips(git_remote *remote, int (*cb)(const char *refname, const git_oid *a, const git_oid *b)) { int error = 0; unsigned int i = 0; git_buf refname = GIT_BUF_INIT; + git_oid old; git_vector *refs = &remote->refs; git_remote_head *head; git_reference *ref; @@ -338,17 +339,36 @@ int git_remote_update_tips(git_remote *remote) head = refs->contents[i]; if (git_refspec_transform_r(&refname, spec, head->name) < 0) - break; + goto on_error; + + error = git_reference_name_to_oid(&old, remote->repo, refname.ptr); + if (error < 0 && error != GIT_ENOTFOUND) + goto on_error; + + if (error == GIT_ENOTFOUND) + memset(&old, 0, GIT_OID_RAWSZ); + + if (!git_oid_cmp(&old, &head->oid)) + continue; if (git_reference_create_oid(&ref, remote->repo, refname.ptr, &head->oid, 1) < 0) break; git_reference_free(ref); + + if (cb != NULL) { + if (cb(refname.ptr, &old, &head->oid) < 0) + goto on_error; + } } git_buf_free(&refname); + return 0; + +on_error: + git_buf_free(&refname); + return -1; - return error; } int git_remote_connected(git_remote *remote) diff --git a/src/revwalk.c b/src/revwalk.c index a62576038..041dc1a1c 100644 --- a/src/revwalk.c +++ b/src/revwalk.c @@ -689,6 +689,13 @@ static int prepare_walk(git_revwalk *walk) commit_object *next, *two; commit_list *bases = NULL; + /* + * If walk->one is NULL, there were no positive references, + * so we know that the walk is already over. + */ + if (walk->one == NULL) + return GIT_EREVWALKOVER; + /* first figure out what the merge bases are */ if (merge_bases_many(&bases, walk, walk->one, &walk->twos) < 0) return -1; diff --git a/src/transport.h b/src/transport.h index 4c123571d..9be96fed6 100644 --- a/src/transport.h +++ b/src/transport.h @@ -8,6 +8,7 @@ #define INCLUDE_transport_h__ #include "git2/net.h" +#include "git2/indexer.h" #include "vector.h" #define GIT_CAP_OFS_DELTA "ofs-delta" @@ -65,23 +66,15 @@ struct git_transport { * Push the changes over */ int (*push)(struct git_transport *transport); - /** - * Send a 'done' message - */ - int (*send_done)(struct git_transport *transport); /** * Negotiate the minimal amount of objects that need to be * retrieved */ int (*negotiate_fetch)(struct git_transport *transport, git_repository *repo, const git_vector *wants); - /** - * Send a flush - */ - int (*send_flush)(struct git_transport *transport); /** * Download the packfile */ - int (*download_pack)(char **out, struct git_transport *transport, git_repository *repo); + int (*download_pack)(struct git_transport *transport, git_repository *repo, git_off_t *bytes, git_indexer_stats *stats); /** * Fetch the changes */ diff --git a/src/transports/git.c b/src/transports/git.c index 825f072c8..31bc21c96 100644 --- a/src/transports/git.c +++ b/src/transports/git.c @@ -293,41 +293,22 @@ static int git_negotiate_fetch(git_transport *transport, git_repository *repo, c { transport_git *t = (transport_git *) transport; git_revwalk *walk; - git_reference *ref; - git_strarray refs; git_oid oid; int error; unsigned int i; + git_buf data = GIT_BUF_INIT; gitno_buffer *buf = &t->buf; - if (git_pkt_send_wants(wants, &t->caps, t->socket) < 0) + if (git_pkt_buffer_wants(wants, &t->caps, &data) < 0) return -1; - if (git_reference_listall(&refs, repo, GIT_REF_LISTALL) < 0) - return -1; + if (git_fetch_setup_walk(&walk, repo) < 0) + goto on_error; - if (git_revwalk_new(&walk, repo) < 0) - return -1; - - git_revwalk_sorting(walk, GIT_SORT_TIME); - - for (i = 0; i < refs.count; ++i) { - /* No tags */ - if (!git__prefixcmp(refs.strings[i], GIT_REFS_TAGS_DIR)) - continue; - - if (git_reference_lookup(&ref, repo, refs.strings[i]) < 0) - goto on_error; - - if (git_reference_type(ref) == GIT_REF_SYMBOLIC) - continue; - - if ((error = git_revwalk_push(walk, git_reference_oid(ref))) < 0) - goto on_error; - - } - git_strarray_free(&refs); + if (gitno_send(t->socket, data.ptr, data.size, 0) < 0) + goto on_error; + git_buf_clear(&data); /* * We don't support any kind of ACK extensions, so the negotiation * boils down to sending what we have and listening for an ACK @@ -335,12 +316,18 @@ static int git_negotiate_fetch(git_transport *transport, git_repository *repo, c */ i = 0; while ((error = git_revwalk_next(&oid, walk)) == 0) { - error = git_pkt_send_have(&oid, t->socket); + git_pkt_buffer_have(&oid, &data); i++; if (i % 20 == 0) { int pkt_type; - git_pkt_send_flush(t->socket); + git_pkt_buffer_flush(&data); + if (git_buf_oom(&data)) + goto on_error; + + if (gitno_send(t->socket, data.ptr, data.size, 0) < 0) + goto on_error; + pkt_type = recv_pkt(buf); if (pkt_type == GIT_PKT_ACK) { @@ -354,35 +341,27 @@ static int git_negotiate_fetch(git_transport *transport, git_repository *repo, c } } - if (error != GIT_EREVWALKOVER) + if (error < 0 && error != GIT_EREVWALKOVER) goto on_error; - git_pkt_send_flush(t->socket); - git_pkt_send_done(t->socket); + /* Tell the other end that we're done negotiating */ + git_buf_clear(&data); + git_pkt_buffer_flush(&data); + git_pkt_buffer_done(&data); + if (gitno_send(t->socket, data.ptr, data.size, 0) < 0) + goto on_error; + git_buf_free(&data); git_revwalk_free(walk); return 0; on_error: + git_buf_free(&data); git_revwalk_free(walk); return -1; } -static int git_send_flush(git_transport *transport) -{ - transport_git *t = (transport_git *) transport; - - return git_pkt_send_flush(t->socket); -} - -static int git_send_done(git_transport *transport) -{ - transport_git *t = (transport_git *) transport; - - return git_pkt_send_done(t->socket); -} - -static int git_download_pack(char **out, git_transport *transport, git_repository *repo) +static int git_download_pack(git_transport *transport, git_repository *repo, git_off_t *bytes, git_indexer_stats *stats) { transport_git *t = (transport_git *) transport; int error = 0, read_bytes; @@ -410,7 +389,7 @@ static int git_download_pack(char **out, git_transport *transport, git_repositor if (pkt->type == GIT_PKT_PACK) { git__free(pkt); - return git_fetch__download_pack(out, buf->data, buf->offset, t->socket, repo); + return git_fetch__download_pack(buf->data, buf->offset, t->socket, repo, bytes, stats); } /* For now we don't care about anything */ @@ -424,7 +403,6 @@ static int git_download_pack(char **out, git_transport *transport, git_repositor return read_bytes; } - static int git_close(git_transport *transport) { transport_git *t = (transport_git*) transport; @@ -476,8 +454,6 @@ int git_transport_git(git_transport **out) t->parent.connect = git_connect; t->parent.ls = git_ls; t->parent.negotiate_fetch = git_negotiate_fetch; - t->parent.send_flush = git_send_flush; - t->parent.send_done = git_send_done; t->parent.download_pack = git_download_pack; t->parent.close = git_close; t->parent.free = git_free; diff --git a/src/transports/http.c b/src/transports/http.c index 0938fefff..012e8ffbc 100644 --- a/src/transports/http.c +++ b/src/transports/http.c @@ -410,44 +410,6 @@ static int parse_response(transport_http *t) return ret; } -static int setup_walk(git_revwalk **out, git_repository *repo) -{ - git_revwalk *walk; - git_strarray refs; - unsigned int i; - git_reference *ref; - - if (git_reference_listall(&refs, repo, GIT_REF_LISTALL) < 0) - return -1; - - if (git_revwalk_new(&walk, repo) < 0) - return -1; - - git_revwalk_sorting(walk, GIT_SORT_TIME); - - for (i = 0; i < refs.count; ++i) { - /* No tags */ - if (!git__prefixcmp(refs.strings[i], GIT_REFS_TAGS_DIR)) - continue; - - if (git_reference_lookup(&ref, repo, refs.strings[i]) < 0) - goto on_error; - - if (git_reference_type(ref) == GIT_REF_SYMBOLIC) - continue; - if (git_revwalk_push(walk, git_reference_oid(ref)) < 0) - goto on_error; - } - - git_strarray_free(&refs); - *out = walk; - return 0; - -on_error: - git_strarray_free(&refs); - return -1; -} - static int http_negotiate_fetch(git_transport *transport, git_repository *repo, const git_vector *wants) { transport_http *t = (transport_http *) transport; @@ -470,7 +432,7 @@ static int http_negotiate_fetch(git_transport *transport, git_repository *repo, if (git_vector_init(common, 16, NULL) < 0) return -1; - if (setup_walk(&walk, repo) < 0) + if (git_fetch_setup_walk(&walk, repo) < 0) return -1; do { @@ -529,7 +491,8 @@ cleanup: } typedef struct { - git_filebuf *file; + git_indexer_stream *idx; + git_indexer_stats *stats; transport_http *transport; } download_pack_cbdata; @@ -545,10 +508,10 @@ static int on_body_download_pack(http_parser *parser, const char *str, size_t le { download_pack_cbdata *data = (download_pack_cbdata *) parser->data; transport_http *t = data->transport; - git_filebuf *file = data->file; + git_indexer_stream *idx = data->idx; + git_indexer_stats *stats = data->stats; - - return t->error = git_filebuf_write(file, str, len); + return t->error = git_indexer_stream_add(idx, str, len, stats); } /* @@ -557,30 +520,16 @@ static int on_body_download_pack(http_parser *parser, const char *str, size_t le * the simple downloader. Furthermore, we're using keep-alive * connections, so the simple downloader would just hang. */ -static int http_download_pack(char **out, git_transport *transport, git_repository *repo) +static int http_download_pack(git_transport *transport, git_repository *repo, git_off_t *bytes, git_indexer_stats *stats) { transport_http *t = (transport_http *) transport; git_buf *oldbuf = &t->buf; - int ret = 0; + int recvd; http_parser_settings settings; char buffer[1024]; gitno_buffer buf; + git_indexer_stream *idx = NULL; download_pack_cbdata data; - git_filebuf file = GIT_FILEBUF_INIT; - git_buf path = GIT_BUF_INIT; - char suff[] = "/objects/pack/pack-received\0"; - - /* - * This is part of the previous response, so we don't want to - * re-init the parser, just set these two callbacks. - */ - data.file = &file; - data.transport = t; - t->parser.data = &data; - t->transfer_finished = 0; - memset(&settings, 0x0, sizeof(settings)); - settings.on_message_complete = on_message_complete_download_pack; - settings.on_body = on_body_download_pack; gitno_buffer_setup(&buf, buffer, sizeof(buffer), t->socket); @@ -589,48 +538,50 @@ static int http_download_pack(char **out, git_transport *transport, git_reposito return -1; } - if (git_buf_joinpath(&path, repo->path_repository, suff) < 0) + if (git_indexer_stream_new(&idx, git_repository_path(repo)) < 0) + return -1; + + + /* + * This is part of the previous response, so we don't want to + * re-init the parser, just set these two callbacks. + */ + memset(stats, 0, sizeof(git_indexer_stats)); + data.stats = stats; + data.idx = idx; + data.transport = t; + t->parser.data = &data; + t->transfer_finished = 0; + memset(&settings, 0x0, sizeof(settings)); + settings.on_message_complete = on_message_complete_download_pack; + settings.on_body = on_body_download_pack; + *bytes = oldbuf->size; + + if (git_indexer_stream_add(idx, oldbuf->ptr, oldbuf->size, stats) < 0) goto on_error; - if (git_filebuf_open(&file, path.ptr, GIT_FILEBUF_TEMPORARY) < 0) - goto on_error; - - /* Part of the packfile has been received, don't loose it */ - if (git_filebuf_write(&file, oldbuf->ptr, oldbuf->size) < 0) - goto on_error; - - while(1) { + do { size_t parsed; - ret = gitno_recv(&buf); - if (ret < 0) + if ((recvd = gitno_recv(&buf)) < 0) goto on_error; parsed = http_parser_execute(&t->parser, &settings, buf.data, buf.offset); - /* Both should happen at the same time */ if (parsed != buf.offset || t->error < 0) - return t->error; + goto on_error; + *bytes += recvd; gitno_consume_n(&buf, parsed); + } while (recvd > 0 && !t->transfer_finished); - if (ret == 0 || t->transfer_finished) { - break; - } - } - - *out = git__strdup(file.path_lock); - GITERR_CHECK_ALLOC(*out); - - /* A bit dodgy, but we need to keep the pack at the temporary path */ - ret = git_filebuf_commit_at(&file, file.path_lock, GIT_PACK_FILE_MODE); - - git_buf_free(&path); + if (git_indexer_stream_finalize(idx, stats) < 0) + goto on_error; + git_indexer_stream_free(idx); return 0; on_error: - git_filebuf_cleanup(&file); - git_buf_free(&path); + git_indexer_stream_free(idx); return -1; }