From 7a520f5d8af2aedd5693bf7314527d76d9af2ef4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= <carlos@cmartin.tk>
Date: Fri, 13 Apr 2012 23:19:38 +0200
Subject: [PATCH 1/6] fetch: use the streaming indexer when downloading a pack

This changes the git_remote_download() API, but the existing one is
silly, so you don't get to complain.

The new API allows to know how much data has been downloaded, how many
objects we expect in total and how many we've processed.
---
 include/git2/remote.h |  2 +-
 src/fetch.c           | 60 +++++++++++++-----------------
 src/fetch.h           |  6 +--
 src/remote.c          |  6 +--
 src/transport.h       |  2 +-
 src/transports/git.c  |  4 +-
 src/transports/http.c | 85 +++++++++++++++++++------------------------
 7 files changed, 72 insertions(+), 93 deletions(-)

diff --git a/include/git2/remote.h b/include/git2/remote.h
index e6537ec52..576f5841b 100644
--- a/include/git2/remote.h
+++ b/include/git2/remote.h
@@ -150,7 +150,7 @@ GIT_EXTERN(int) git_remote_ls(git_remote *remote, git_headlist_cb list_cb, void
  * @param filename where to store the temproray filename
  * @return GIT_SUCCESS or an error code
  */
-GIT_EXTERN(int) git_remote_download(char **filename, git_remote *remote);
+GIT_EXTERN(int) git_remote_download(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats);
 
 /**
  * Check whether the remote is connected
diff --git a/src/fetch.c b/src/fetch.c
index 57a6d0265..8da4fd8cd 100644
--- a/src/fetch.c
+++ b/src/fetch.c
@@ -9,6 +9,7 @@
 #include "git2/oid.h"
 #include "git2/refs.h"
 #include "git2/revwalk.h"
+#include "git2/indexer.h"
 
 #include "common.h"
 #include "transport.h"
@@ -101,30 +102,27 @@ int git_fetch_negotiate(git_remote *remote)
 	return t->negotiate_fetch(t, remote->repo, &remote->refs);
 }
 
-int git_fetch_download_pack(char **out, git_remote *remote)
+int git_fetch_download_pack(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats)
 {
-	if(!remote->need_pack) {
-		*out = NULL;
+	if(!remote->need_pack)
 		return 0;
-	}
 
-	return remote->transport->download_pack(out, remote->transport, remote->repo);
+	return remote->transport->download_pack(remote->transport, remote->repo, bytes, stats);
 }
 
 /* Receiving data from a socket and storing it is pretty much the same for git and HTTP */
 int git_fetch__download_pack(
-	char **out,
 	const char *buffered,
 	size_t buffered_size,
 	GIT_SOCKET fd,
-	git_repository *repo)
+	git_repository *repo,
+	git_off_t *bytes,
+	git_indexer_stats *stats)
 {
-	git_filebuf file = GIT_FILEBUF_INIT;
-	int error;
+	int recvd;
 	char buff[1024];
-	git_buf path = GIT_BUF_INIT;
-	static const char suff[] = "/objects/pack/pack-received";
 	gitno_buffer buf;
+	git_indexer_stream *idx;
 
 	gitno_buffer_setup(&buf, buff, sizeof(buff), fd);
 
@@ -133,41 +131,33 @@ int git_fetch__download_pack(
 		return -1;
 	}
 
-	if (git_buf_joinpath(&path, repo->path_repository, suff) < 0)
+	if (git_indexer_stream_new(&idx, git_repository_path(repo)) < 0)
+		return -1;
+
+	memset(stats, 0, sizeof(git_indexer_stats));
+	if (git_indexer_stream_add(idx, buffered, buffered_size, stats) < 0)
 		goto on_error;
 
-	if (git_filebuf_open(&file, path.ptr, GIT_FILEBUF_TEMPORARY) < 0)
-		goto on_error;
+	*bytes = buffered_size;
 
-	/* Part of the packfile has been received, don't loose it */
-	if (git_filebuf_write(&file, buffered, buffered_size) < 0)
-		goto on_error;
-
-	while (1) {
-		if (git_filebuf_write(&file, buf.data, buf.offset) < 0)
+	do {
+		if (git_indexer_stream_add(idx, buf.data, buf.offset, stats) < 0)
 			goto on_error;
 
 		gitno_consume_n(&buf, buf.offset);
-		error = gitno_recv(&buf);
-		if (error < GIT_SUCCESS)
+		if ((recvd = gitno_recv(&buf)) < 0)
 			goto on_error;
-		if (error == 0) /* Orderly shutdown */
-			break;
-	}
 
-	*out = git__strdup(file.path_lock);
-	if (*out == NULL)
+		*bytes += recvd;
+	} while(recvd > 0);
+
+	if (git_indexer_stream_finalize(idx, stats))
 		goto on_error;
 
-	/* A bit dodgy, but we need to keep the pack at the temporary path */
-	if (git_filebuf_commit_at(&file, file.path_lock, GIT_PACK_FILE_MODE) < 0)
-		goto on_error;
-
-	git_buf_free(&path);
-
+	git_indexer_stream_free(idx);
 	return 0;
+
 on_error:
-	git_buf_free(&path);
-	git_filebuf_cleanup(&file);
+	git_indexer_stream_free(idx);
 	return -1;
 }
diff --git a/src/fetch.h b/src/fetch.h
index c1ab84034..03767be8d 100644
--- a/src/fetch.h
+++ b/src/fetch.h
@@ -10,9 +10,9 @@
 #include "netops.h"
 
 int git_fetch_negotiate(git_remote *remote);
-int git_fetch_download_pack(char **out, git_remote *remote);
+int git_fetch_download_pack(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats);
 
-int git_fetch__download_pack(char **out, const char *buffered, size_t buffered_size,
-                             GIT_SOCKET fd, git_repository *repo);
+int git_fetch__download_pack(const char *buffered, size_t buffered_size, GIT_SOCKET fd,
+			     git_repository *repo, git_off_t *bytes, git_indexer_stats *stats);
 
 #endif
diff --git a/src/remote.c b/src/remote.c
index b48a23339..bbb491dd8 100644
--- a/src/remote.c
+++ b/src/remote.c
@@ -297,16 +297,16 @@ int git_remote_ls(git_remote *remote, git_headlist_cb list_cb, void *payload)
 	return remote->transport->ls(remote->transport, list_cb, payload);
 }
 
-int git_remote_download(char **filename, git_remote *remote)
+int git_remote_download(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats)
 {
 	int error;
 
-	assert(filename && remote);
+	assert(remote && bytes && stats);
 
 	if ((error = git_fetch_negotiate(remote)) < 0)
 		return error;
 
-	return git_fetch_download_pack(filename, remote);
+	return git_fetch_download_pack(remote, bytes, stats);
 }
 
 int git_remote_update_tips(git_remote *remote)
diff --git a/src/transport.h b/src/transport.h
index 4c123571d..1cea32bee 100644
--- a/src/transport.h
+++ b/src/transport.h
@@ -81,7 +81,7 @@ struct git_transport {
 	/**
 	 * Download the packfile
 	 */
-	int (*download_pack)(char **out, struct git_transport *transport, git_repository *repo);
+	int (*download_pack)(struct git_transport *transport, git_repository *repo, git_off_t *bytes, git_indexer_stats *stats);
 	/**
 	 * Fetch the changes
 	 */
diff --git a/src/transports/git.c b/src/transports/git.c
index 825f072c8..62106de22 100644
--- a/src/transports/git.c
+++ b/src/transports/git.c
@@ -382,7 +382,7 @@ static int git_send_done(git_transport *transport)
 	return git_pkt_send_done(t->socket);
 }
 
-static int git_download_pack(char **out, git_transport *transport, git_repository *repo)
+static int git_download_pack(git_transport *transport, git_repository *repo, git_off_t *bytes, git_indexer_stats *stats)
 {
 	transport_git *t = (transport_git *) transport;
 	int error = 0, read_bytes;
@@ -410,7 +410,7 @@ static int git_download_pack(char **out, git_transport *transport, git_repositor
 
 			if (pkt->type == GIT_PKT_PACK) {
 				git__free(pkt);
-				return git_fetch__download_pack(out, buf->data, buf->offset, t->socket, repo);
+				return git_fetch__download_pack(buf->data, buf->offset, t->socket, repo, bytes, stats);
 			}
 
 			/* For now we don't care about anything */
diff --git a/src/transports/http.c b/src/transports/http.c
index 0938fefff..e6a709403 100644
--- a/src/transports/http.c
+++ b/src/transports/http.c
@@ -529,7 +529,8 @@ cleanup:
 }
 
 typedef struct {
-	git_filebuf *file;
+	git_indexer_stream *idx;
+	git_indexer_stats *stats;
 	transport_http *transport;
 } download_pack_cbdata;
 
@@ -545,10 +546,10 @@ static int on_body_download_pack(http_parser *parser, const char *str, size_t le
 {
 	download_pack_cbdata *data = (download_pack_cbdata *) parser->data;
 	transport_http *t = data->transport;
-	git_filebuf *file = data->file;
+	git_indexer_stream *idx = data->idx;
+	git_indexer_stats *stats = data->stats;
 
-
-	return t->error = git_filebuf_write(file, str, len);
+	return t->error = git_indexer_stream_add(idx, str, len, stats);
 }
 
 /*
@@ -557,30 +558,16 @@ static int on_body_download_pack(http_parser *parser, const char *str, size_t le
  * the simple downloader. Furthermore, we're using keep-alive
  * connections, so the simple downloader would just hang.
  */
-static int http_download_pack(char **out, git_transport *transport, git_repository *repo)
+static int http_download_pack(git_transport *transport, git_repository *repo, git_off_t *bytes, git_indexer_stats *stats)
 {
 	transport_http *t = (transport_http *) transport;
 	git_buf *oldbuf = &t->buf;
-	int ret = 0;
+	int recvd;
 	http_parser_settings settings;
 	char buffer[1024];
 	gitno_buffer buf;
+	git_indexer_stream *idx = NULL;
 	download_pack_cbdata data;
-	git_filebuf file = GIT_FILEBUF_INIT;
-	git_buf path = GIT_BUF_INIT;
-	char suff[] = "/objects/pack/pack-received\0";
-
-	/*
-	 * This is part of the previous response, so we don't want to
-	 * re-init the parser, just set these two callbacks.
-	 */
-	data.file = &file;
-	data.transport = t;
-	t->parser.data = &data;
-	t->transfer_finished = 0;
-	memset(&settings, 0x0, sizeof(settings));
-	settings.on_message_complete = on_message_complete_download_pack;
-	settings.on_body = on_body_download_pack;
 
 	gitno_buffer_setup(&buf, buffer, sizeof(buffer), t->socket);
 
@@ -589,48 +576,50 @@ static int http_download_pack(char **out, git_transport *transport, git_reposito
 		return -1;
 	}
 
-	if (git_buf_joinpath(&path, repo->path_repository, suff) < 0)
+	if (git_indexer_stream_new(&idx, git_repository_path(repo)) < 0)
+		return -1;
+
+
+	/*
+	 * This is part of the previous response, so we don't want to
+	 * re-init the parser, just set these two callbacks.
+	 */
+	memset(stats, 0, sizeof(git_indexer_stats));
+	data.stats = stats;
+	data.idx = idx;
+	data.transport = t;
+	t->parser.data = &data;
+	t->transfer_finished = 0;
+	memset(&settings, 0x0, sizeof(settings));
+	settings.on_message_complete = on_message_complete_download_pack;
+	settings.on_body = on_body_download_pack;
+	*bytes = oldbuf->size;
+
+	if (git_indexer_stream_add(idx, oldbuf->ptr, oldbuf->size, stats) < 0)
 		goto on_error;
 
-	if (git_filebuf_open(&file, path.ptr, GIT_FILEBUF_TEMPORARY) < 0)
-		goto on_error;
-
-	/* Part of the packfile has been received, don't loose it */
-	if (git_filebuf_write(&file, oldbuf->ptr, oldbuf->size) < 0)
-		goto on_error;
-
-	while(1) {
+	do {
 		size_t parsed;
 
-		ret = gitno_recv(&buf);
-		if (ret < 0)
+		if ((recvd = gitno_recv(&buf)) < 0)
 			goto on_error;
 
 		parsed = http_parser_execute(&t->parser, &settings, buf.data, buf.offset);
-		/* Both should happen at the same time */
 		if (parsed != buf.offset || t->error < 0)
-			return t->error;
+			goto on_error;
 
+		*bytes += recvd;
 		gitno_consume_n(&buf, parsed);
+	} while (recvd > 0 && !t->transfer_finished);
 
-		if (ret == 0 || t->transfer_finished) {
-			break;
-		}
-	}
-
-	*out = git__strdup(file.path_lock);
-	GITERR_CHECK_ALLOC(*out);
-
-	/* A bit dodgy, but we need to keep the pack at the temporary path */
-	ret = git_filebuf_commit_at(&file, file.path_lock, GIT_PACK_FILE_MODE);
-
-	git_buf_free(&path);
+	if (git_indexer_stream_finalize(idx, stats) < 0)
+		goto on_error;
 
+	git_indexer_stream_free(idx);
 	return 0;
 
 on_error:
-	git_filebuf_cleanup(&file);
-	git_buf_free(&path);
+	git_indexer_stream_free(idx);
 	return -1;
 }
 

From db0f96a6aff33612d88ab5d9263bcad9daf6b11a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= <carlos@cmartin.tk>
Date: Fri, 13 Apr 2012 23:37:55 +0200
Subject: [PATCH 2/6] examples: port 'fetch' to the new API

---
 examples/network/fetch.c | 97 ++++++++++------------------------------
 1 file changed, 24 insertions(+), 73 deletions(-)

diff --git a/examples/network/fetch.c b/examples/network/fetch.c
index cdd4a4662..3bba1698c 100644
--- a/examples/network/fetch.c
+++ b/examples/network/fetch.c
@@ -4,94 +4,45 @@
 #include <stdlib.h>
 #include <string.h>
 
-static int rename_packfile(char *packname, git_indexer *idx)
-{
-  char path[GIT_PATH_MAX], oid[GIT_OID_HEXSZ + 1], *slash;
-  int ret;
-
-  strcpy(path, packname);
-  slash = strrchr(path, '/');
-
-  if (!slash)
-	  return GIT_EINVALIDARGS;
-
-  memset(oid, 0x0, sizeof(oid));
-  // The name of the packfile is given by it's hash which you can get
-  // with git_indexer_hash after the index has been written out to
-  // disk. Rename the packfile to its "real" name in the same
-  // directory as it was originally (libgit2 stores it in the folder
-  // where the packs go, so a rename in place is the right thing to do here
-  git_oid_fmt(oid, git_indexer_hash(idx));
-  ret = sprintf(slash + 1, "pack-%s.pack", oid);
-  if(ret < 0)
-	  return GIT_EOSERR;
-
-  printf("Renaming pack to %s\n", path);
-  return rename(packname, path);
-}
-
 int fetch(git_repository *repo, int argc, char **argv)
 {
   git_remote *remote = NULL;
-  git_indexer *idx = NULL;
+  git_off_t bytes = 0;
   git_indexer_stats stats;
-  int error;
   char *packname = NULL;
 
   // Get the remote and connect to it
   printf("Fetching %s\n", argv[1]);
-  error = git_remote_new(&remote, repo, argv[1], NULL);
-  if (error < GIT_SUCCESS)
-    return error;
-
-  error = git_remote_connect(remote, GIT_DIR_FETCH);
-  if (error < GIT_SUCCESS)
-    return error;
-
-  // Download the packfile from the server. As we don't know its hash
-  // yet, it will get a temporary filename
-  error = git_remote_download(&packname, remote);
-  if (error < GIT_SUCCESS)
-    return error;
-
-  // No error and a NULL packname means no packfile was needed
-  if (packname != NULL) {
-	  printf("The packname is %s\n", packname);
-
-	  // Create a new instance indexer
-	  error = git_indexer_new(&idx, packname);
-	  if (error < GIT_SUCCESS)
-		  return error;
-
-	  // This should be run in paralel, but it'd be too complicated for the example
-	  error = git_indexer_run(idx, &stats);
-	  if (error < GIT_SUCCESS)
-		  return error;
-
-	  printf("Received %d objects\n", stats.total);
-
-	  // Write the index file. The index will be stored with the
-	  // correct filename
-	  error = git_indexer_write(idx);
-	  if (error < GIT_SUCCESS)
-		  return error;
-
-	  error = rename_packfile(packname, idx);
-	  if (error < GIT_SUCCESS)
-		  return error;
+  if (git_remote_load(&remote, repo, argv[1]) == GIT_ENOTFOUND) {
+	  if (git_remote_new(&remote, repo, argv[1], NULL) < 0)
+		  return -1;
   }
 
+  if (git_remote_connect(remote, GIT_DIR_FETCH) < 0)
+	  return -1;
+
+  // Download the packfile and index it
+  // Doing this in a background thread and printing out what bytes
+  // and stats.{processed,total} say would make the UI friendlier
+  if (git_remote_download(remote, &bytes, &stats) < 0) {
+	  git_remote_free(remote);
+	  return -1;
+  }
+
+  printf("Received %d objects in %d bytes\n", stats.total, bytes);
+
   // Update the references in the remote's namespace to point to the
   // right commits. This may be needed even if there was no packfile
   // to download, which can happen e.g. when the branches have been
   // changed but all the neede objects are available locally.
-  error = git_remote_update_tips(remote);
-  if (error < GIT_SUCCESS)
-    return error;
+  if (git_remote_update_tips(remote) < 0)
+	  return -1;
 
-  free(packname);
-  git_indexer_free(idx);
   git_remote_free(remote);
 
-  return GIT_SUCCESS;
+  return 0;
+
+on_error:
+  git_remote_free(remote);
+  return -1;
 }

From dee5515a237b2d4182e454986025199064193376 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= <carlos@cmartin.tk>
Date: Sat, 14 Apr 2012 18:34:50 +0200
Subject: [PATCH 3/6] transports: buffer the git requests before sending them

Trying to send every single line immediately won't give us any speed
improvement and duplicates the code we need for other transports. Make
the git transport use the same buffer functions as HTTP.
---
 include/git2/indexer.h |  2 +-
 include/git2/remote.h  |  1 +
 src/fetch.c            | 41 ++++++++++++++++++++++++
 src/fetch.h            |  1 +
 src/indexer.c          |  2 +-
 src/pkt.c              | 68 ---------------------------------------
 src/pkt.h              |  3 --
 src/transport.h        |  9 +-----
 src/transports/git.c   | 72 ++++++++++++++----------------------------
 src/transports/http.c  | 40 +----------------------
 10 files changed, 71 insertions(+), 168 deletions(-)

diff --git a/include/git2/indexer.h b/include/git2/indexer.h
index a70fab214..14bd0e402 100644
--- a/include/git2/indexer.h
+++ b/include/git2/indexer.h
@@ -41,7 +41,7 @@ GIT_EXTERN(int) git_indexer_stream_new(git_indexer_stream **out, const char *git
  * @param size the size of the data
  * @param stats stat storage
  */
-GIT_EXTERN(int) git_indexer_stream_add(git_indexer_stream *idx, void *data, size_t size, git_indexer_stats *stats);
+GIT_EXTERN(int) git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t size, git_indexer_stats *stats);
 
 /**
  * Finalize the pack and index
diff --git a/include/git2/remote.h b/include/git2/remote.h
index 576f5841b..8f49fddf1 100644
--- a/include/git2/remote.h
+++ b/include/git2/remote.h
@@ -11,6 +11,7 @@
 #include "repository.h"
 #include "refspec.h"
 #include "net.h"
+#include "indexer.h"
 
 /**
  * @file git2/remote.h
diff --git a/src/fetch.c b/src/fetch.c
index 8da4fd8cd..6fe1b5676 100644
--- a/src/fetch.c
+++ b/src/fetch.c
@@ -161,3 +161,44 @@ on_error:
 	git_indexer_stream_free(idx);
 	return -1;
 }
+
+int git_fetch_setup_walk(git_revwalk **out, git_repository *repo)
+{
+	git_revwalk *walk;
+	git_strarray refs;
+	unsigned int i;
+	git_reference *ref;
+
+	if (git_reference_listall(&refs, repo, GIT_REF_LISTALL) < 0)
+		return -1;
+
+	if (git_revwalk_new(&walk, repo) < 0)
+		return -1;
+
+	git_revwalk_sorting(walk, GIT_SORT_TIME);
+
+	for (i = 0; i < refs.count; ++i) {
+		/* No tags */
+		if (!git__prefixcmp(refs.strings[i], GIT_REFS_TAGS_DIR))
+			continue;
+
+		if (git_reference_lookup(&ref, repo, refs.strings[i]) < 0)
+			goto on_error;
+
+		if (git_reference_type(ref) == GIT_REF_SYMBOLIC)
+			continue;
+		if (git_revwalk_push(walk, git_reference_oid(ref)) < 0)
+			goto on_error;
+
+		git_reference_free(ref);
+	}
+
+	git_strarray_free(&refs);
+	*out = walk;
+	return 0;
+
+on_error:
+	git_reference_free(ref);
+	git_strarray_free(&refs);
+	return -1;
+}
diff --git a/src/fetch.h b/src/fetch.h
index 03767be8d..b3192a563 100644
--- a/src/fetch.h
+++ b/src/fetch.h
@@ -14,5 +14,6 @@ int git_fetch_download_pack(git_remote *remote, git_off_t *bytes, git_indexer_st
 
 int git_fetch__download_pack(const char *buffered, size_t buffered_size, GIT_SOCKET fd,
 			     git_repository *repo, git_off_t *bytes, git_indexer_stats *stats);
+int git_fetch_setup_walk(git_revwalk **out, git_repository *repo);
 
 #endif
diff --git a/src/indexer.c b/src/indexer.c
index 1834d9884..1f8b512c2 100644
--- a/src/indexer.c
+++ b/src/indexer.c
@@ -288,7 +288,7 @@ on_error:
 	return -1;
 }
 
-int git_indexer_stream_add(git_indexer_stream *idx, void *data, size_t size, git_indexer_stats *stats)
+int git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t size, git_indexer_stats *stats)
 {
 	int error;
 	struct git_pack_header hdr;
diff --git a/src/pkt.c b/src/pkt.c
index ee113cd46..2c9fe27da 100644
--- a/src/pkt.c
+++ b/src/pkt.c
@@ -287,19 +287,6 @@ static int buffer_want_with_caps(git_remote_head *head, git_transport_caps *caps
 	return git_buf_printf(buf, "%04xwant %s%c%s\n", len, oid, 0, capstr);
 }
 
-static int send_want_with_caps(git_remote_head *head, git_transport_caps *caps, GIT_SOCKET fd)
-{
-	git_buf buf = GIT_BUF_INIT;
-	int ret;
-
-	if (buffer_want_with_caps(head, caps, &buf) < 0)
-		return -1;
-
-	ret = gitno_send(fd, buf.ptr, buf.size, 0);
-	git_buf_free(&buf);
-	return ret;
-}
-
 /*
  * All "want" packets have the same length and format, so what we do
  * is overwrite the OID each time.
@@ -341,47 +328,6 @@ int git_pkt_buffer_wants(const git_vector *refs, git_transport_caps *caps, git_b
 	return git_pkt_buffer_flush(buf);
 }
 
-int git_pkt_send_wants(const git_vector *refs, git_transport_caps *caps, GIT_SOCKET fd)
-{
-	unsigned int i = 0;
-	char buf[sizeof(pkt_want_prefix) + GIT_OID_HEXSZ + 1];
-	git_remote_head *head;
-
-	memcpy(buf, pkt_want_prefix, strlen(pkt_want_prefix));
-	buf[sizeof(buf) - 2] = '\n';
-	buf[sizeof(buf) - 1] = '\0';
-
-	/* If there are common caps, find the first one */
-	if (caps->common) {
-		for (; i < refs->length; ++i) {
-			head = refs->contents[i];
-			if (head->local)
-				continue;
-			else
-				break;
-		}
-
-		if (send_want_with_caps(refs->contents[i], caps, fd) < 0)
-			return -1;
-
-		/* Increase it here so it's correct whether we run this or not */
-		i++;
-	}
-
-	/* Continue from where we left off */
-	for (; i < refs->length; ++i) {
-		head = refs->contents[i];
-		if (head->local)
-			continue;
-
-		git_oid_fmt(buf + strlen(pkt_want_prefix), &head->oid);
-		if (gitno_send(fd, buf, strlen(buf), 0) < 0)
-			return -1;
-	}
-
-	return git_pkt_send_flush(fd);
-}
-
 int git_pkt_buffer_have(git_oid *oid, git_buf *buf)
 {
 	char oidhex[GIT_OID_HEXSZ + 1];
@@ -391,21 +337,7 @@ int git_pkt_buffer_have(git_oid *oid, git_buf *buf)
 	return git_buf_printf(buf, "%s%s\n", pkt_have_prefix, oidhex);
 }
 
-int git_pkt_send_have(git_oid *oid, GIT_SOCKET fd)
-{
-	char buf[] = "0032have 0000000000000000000000000000000000000000\n";
-
-	git_oid_fmt(buf + strlen(pkt_have_prefix), oid);
-	return gitno_send(fd, buf, strlen(buf), 0);
-}
-
-
 int git_pkt_buffer_done(git_buf *buf)
 {
 	return git_buf_puts(buf, pkt_done_str);
 }
-
-int git_pkt_send_done(GIT_SOCKET fd)
-{
-	return gitno_send(fd, pkt_done_str, strlen(pkt_done_str), 0);
-}
diff --git a/src/pkt.h b/src/pkt.h
index 1f8d62e1a..7e696f70f 100644
--- a/src/pkt.h
+++ b/src/pkt.h
@@ -68,11 +68,8 @@ int git_pkt_parse_line(git_pkt **head, const char *line, const char **out, size_
 int git_pkt_buffer_flush(git_buf *buf);
 int git_pkt_send_flush(GIT_SOCKET s);
 int git_pkt_buffer_done(git_buf *buf);
-int git_pkt_send_done(GIT_SOCKET s);
 int git_pkt_buffer_wants(const git_vector *refs, git_transport_caps *caps, git_buf *buf);
-int git_pkt_send_wants(const git_vector *refs, git_transport_caps *caps, GIT_SOCKET fd);
 int git_pkt_buffer_have(git_oid *oid, git_buf *buf);
-int git_pkt_send_have(git_oid *oid, GIT_SOCKET fd);
 void git_pkt_free(git_pkt *pkt);
 
 #endif
diff --git a/src/transport.h b/src/transport.h
index 1cea32bee..9be96fed6 100644
--- a/src/transport.h
+++ b/src/transport.h
@@ -8,6 +8,7 @@
 #define INCLUDE_transport_h__
 
 #include "git2/net.h"
+#include "git2/indexer.h"
 #include "vector.h"
 
 #define GIT_CAP_OFS_DELTA "ofs-delta"
@@ -65,19 +66,11 @@ struct git_transport {
 	 * Push the changes over
 	 */
 	int (*push)(struct git_transport *transport);
-	/**
-	 * Send a 'done' message
-	 */
-	int (*send_done)(struct git_transport *transport);
 	/**
 	 * Negotiate the minimal amount of objects that need to be
 	 * retrieved
 	 */
 	int (*negotiate_fetch)(struct git_transport *transport, git_repository *repo, const git_vector *wants);
-	/**
-	 * Send a flush
-	 */
-	int (*send_flush)(struct git_transport *transport);
 	/**
 	 * Download the packfile
 	 */
diff --git a/src/transports/git.c b/src/transports/git.c
index 62106de22..31bc21c96 100644
--- a/src/transports/git.c
+++ b/src/transports/git.c
@@ -293,41 +293,22 @@ static int git_negotiate_fetch(git_transport *transport, git_repository *repo, c
 {
 	transport_git *t = (transport_git *) transport;
 	git_revwalk *walk;
-	git_reference *ref;
-	git_strarray refs;
 	git_oid oid;
 	int error;
 	unsigned int i;
+	git_buf data = GIT_BUF_INIT;
 	gitno_buffer *buf = &t->buf;
 
-	if (git_pkt_send_wants(wants, &t->caps, t->socket) < 0)
+	if (git_pkt_buffer_wants(wants, &t->caps, &data) < 0)
 		return -1;
 
-	if (git_reference_listall(&refs, repo, GIT_REF_LISTALL) < 0)
-		return -1;
+	if (git_fetch_setup_walk(&walk, repo) < 0)
+		goto on_error;
 
-	if (git_revwalk_new(&walk, repo) < 0)
-		return -1;
-
-	git_revwalk_sorting(walk, GIT_SORT_TIME);
-
-	for (i = 0; i < refs.count; ++i) {
-		/* No tags */
-		if (!git__prefixcmp(refs.strings[i], GIT_REFS_TAGS_DIR))
-			continue;
-
-		if (git_reference_lookup(&ref, repo, refs.strings[i]) < 0)
-			goto on_error;
-
-		if (git_reference_type(ref) == GIT_REF_SYMBOLIC)
-			continue;
-
-		if ((error = git_revwalk_push(walk, git_reference_oid(ref))) < 0)
-			goto on_error;
-
-	}
-	git_strarray_free(&refs);
+	if (gitno_send(t->socket, data.ptr, data.size, 0) < 0)
+		goto on_error;
 
+	git_buf_clear(&data);
 	/*
 	 * We don't support any kind of ACK extensions, so the negotiation
 	 * boils down to sending what we have and listening for an ACK
@@ -335,12 +316,18 @@ static int git_negotiate_fetch(git_transport *transport, git_repository *repo, c
 	 */
 	i = 0;
 	while ((error = git_revwalk_next(&oid, walk)) == 0) {
-		error = git_pkt_send_have(&oid, t->socket);
+		git_pkt_buffer_have(&oid, &data);
 		i++;
 		if (i % 20 == 0) {
 			int pkt_type;
 
-			git_pkt_send_flush(t->socket);
+			git_pkt_buffer_flush(&data);
+			if (git_buf_oom(&data))
+				goto on_error;
+
+			if (gitno_send(t->socket, data.ptr, data.size, 0) < 0)
+				goto on_error;
+
 			pkt_type = recv_pkt(buf);
 
 			if (pkt_type == GIT_PKT_ACK) {
@@ -354,34 +341,26 @@ static int git_negotiate_fetch(git_transport *transport, git_repository *repo, c
 
 		}
 	}
-	if (error != GIT_EREVWALKOVER)
+	if (error < 0 && error != GIT_EREVWALKOVER)
 		goto on_error;
 
-	git_pkt_send_flush(t->socket);
-	git_pkt_send_done(t->socket);
+	/* Tell the other end that we're done negotiating */
+	git_buf_clear(&data);
+	git_pkt_buffer_flush(&data);
+	git_pkt_buffer_done(&data);
+	if (gitno_send(t->socket, data.ptr, data.size, 0) < 0)
+		goto on_error;
 
+	git_buf_free(&data);
 	git_revwalk_free(walk);
 	return 0;
 
 on_error:
+	git_buf_free(&data);
 	git_revwalk_free(walk);
 	return -1;
 }
 
-static int git_send_flush(git_transport *transport)
-{
-	transport_git *t = (transport_git *) transport;
-
-	return git_pkt_send_flush(t->socket);
-}
-
-static int git_send_done(git_transport *transport)
-{
-	transport_git *t = (transport_git *) transport;
-
-	return git_pkt_send_done(t->socket);
-}
-
 static int git_download_pack(git_transport *transport, git_repository *repo, git_off_t *bytes, git_indexer_stats *stats)
 {
 	transport_git *t = (transport_git *) transport;
@@ -424,7 +403,6 @@ static int git_download_pack(git_transport *transport, git_repository *repo, git
 	return read_bytes;
 }
 
-
 static int git_close(git_transport *transport)
 {
 	transport_git *t = (transport_git*) transport;
@@ -476,8 +454,6 @@ int git_transport_git(git_transport **out)
 	t->parent.connect = git_connect;
 	t->parent.ls = git_ls;
 	t->parent.negotiate_fetch = git_negotiate_fetch;
-	t->parent.send_flush = git_send_flush;
-	t->parent.send_done = git_send_done;
 	t->parent.download_pack = git_download_pack;
 	t->parent.close = git_close;
 	t->parent.free = git_free;
diff --git a/src/transports/http.c b/src/transports/http.c
index e6a709403..012e8ffbc 100644
--- a/src/transports/http.c
+++ b/src/transports/http.c
@@ -410,44 +410,6 @@ static int parse_response(transport_http *t)
 	return ret;
 }
 
-static int setup_walk(git_revwalk **out, git_repository *repo)
-{
-	git_revwalk *walk;
-	git_strarray refs;
-	unsigned int i;
-	git_reference *ref;
-
-	if (git_reference_listall(&refs, repo, GIT_REF_LISTALL) < 0)
-		return -1;
-
-	if (git_revwalk_new(&walk, repo) < 0)
-		return -1;
-
-	git_revwalk_sorting(walk, GIT_SORT_TIME);
-
-	for (i = 0; i < refs.count; ++i) {
-		/* No tags */
-		if (!git__prefixcmp(refs.strings[i], GIT_REFS_TAGS_DIR))
-			continue;
-
-		if (git_reference_lookup(&ref, repo, refs.strings[i]) < 0)
-			goto on_error;
-
-		if (git_reference_type(ref) == GIT_REF_SYMBOLIC)
-			continue;
-		if (git_revwalk_push(walk, git_reference_oid(ref)) < 0)
-			goto on_error;
-	}
-
-	git_strarray_free(&refs);
-	*out = walk;
-	return 0;
-
-on_error:
-	git_strarray_free(&refs);
-	return -1;
-}
-
 static int http_negotiate_fetch(git_transport *transport, git_repository *repo, const git_vector *wants)
 {
 	transport_http *t = (transport_http *) transport;
@@ -470,7 +432,7 @@ static int http_negotiate_fetch(git_transport *transport, git_repository *repo,
 	if (git_vector_init(common, 16, NULL) < 0)
 		return -1;
 
-	if (setup_walk(&walk, repo) < 0)
+	if (git_fetch_setup_walk(&walk, repo) < 0)
 		return -1;
 
 	do {

From bf4ef0c567c3add37aa1744467692c49a534d264 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= <carlos@cmartin.tk>
Date: Mon, 16 Apr 2012 05:02:41 +0200
Subject: [PATCH 4/6] examples: run fetch in a background thread

This allows us to give updates on how it's doing
---
 examples/network/fetch.c | 69 ++++++++++++++++++++++++++++++++--------
 1 file changed, 56 insertions(+), 13 deletions(-)

diff --git a/examples/network/fetch.c b/examples/network/fetch.c
index 3bba1698c..f7a60640e 100644
--- a/examples/network/fetch.c
+++ b/examples/network/fetch.c
@@ -3,33 +3,76 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <pthread.h>
+
+struct dl_data {
+	git_remote *remote;
+	git_off_t *bytes;
+	git_indexer_stats *stats;
+	int ret;
+	int finished;
+};
+
+static void *download(void *ptr)
+{
+	struct dl_data *data = (struct dl_data *)ptr;
+
+	// Connect to the remote end specifying that we want to fetch
+	// information from it.
+	if (git_remote_connect(data->remote, GIT_DIR_FETCH) < 0) {
+		data->ret = -1;
+		goto exit;
+	}
+
+	// Download the packfile and index it. This function updates the
+	// amount of received data and the indexer stats which lets you
+	// inform the user about progress.
+	if (git_remote_download(data->remote, data->bytes, data->stats) < 0) {
+		data->ret = -1;
+		goto exit;
+	}
+
+	data->ret = 0;
+
+exit:
+	data->finished = 1;
+	pthread_exit(&data->ret);
+}
 
 int fetch(git_repository *repo, int argc, char **argv)
 {
   git_remote *remote = NULL;
   git_off_t bytes = 0;
   git_indexer_stats stats;
-  char *packname = NULL;
+  pthread_t worker;
+  struct dl_data data;
 
-  // Get the remote and connect to it
+  // Figure out whether it's a named remote or a URL
   printf("Fetching %s\n", argv[1]);
-  if (git_remote_load(&remote, repo, argv[1]) == GIT_ENOTFOUND) {
+  if (git_remote_load(&remote, repo, argv[1]) < 0) {
 	  if (git_remote_new(&remote, repo, argv[1], NULL) < 0)
 		  return -1;
   }
 
-  if (git_remote_connect(remote, GIT_DIR_FETCH) < 0)
-	  return -1;
+  // Set up the information for the background worker thread
+  data.remote = remote;
+  data.bytes = &bytes;
+  data.stats = &stats;
+  data.ret = 0;
+  data.finished = 0;
+  memset(&stats, 0, sizeof(stats));
 
-  // Download the packfile and index it
-  // Doing this in a background thread and printing out what bytes
-  // and stats.{processed,total} say would make the UI friendlier
-  if (git_remote_download(remote, &bytes, &stats) < 0) {
-	  git_remote_free(remote);
-	  return -1;
-  }
+  pthread_create(&worker, NULL, download, &data);
 
-  printf("Received %d objects in %d bytes\n", stats.total, bytes);
+  // Loop while the worker thread is still running. Here we show processed
+  // and total objects in the pack and the amount of received
+  // data. Most frontends will probably want to show a percentage and
+  // the download rate.
+  do {
+	usleep(10000);
+	printf("\rReceived %d/%d objects in %d bytes", stats.processed, stats.total, bytes);
+  } while (!data.finished);
+  printf("\rReceived %d/%d objects in %d bytes\n", stats.processed, stats.total, bytes);
 
   // Update the references in the remote's namespace to point to the
   // right commits. This may be needed even if there was no packfile

From 2e3a0055d136d13fba365bf2a26638f84bd32d02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= <carlos@cmartin.tk>
Date: Mon, 16 Apr 2012 11:58:46 +0200
Subject: [PATCH 5/6] revwalk: return GIT_EREVWALKER earlier if no references
 were pushed

In the case that walk->one is NULL, we know that we have no positive
references, so we already know that the revwalk is over.
---
 src/revwalk.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/revwalk.c b/src/revwalk.c
index a62576038..041dc1a1c 100644
--- a/src/revwalk.c
+++ b/src/revwalk.c
@@ -689,6 +689,13 @@ static int prepare_walk(git_revwalk *walk)
 	commit_object *next, *two;
 	commit_list *bases = NULL;
 
+	/*
+	 * If walk->one is NULL, there were no positive references,
+	 * so we know that the walk is already over.
+	 */
+	if (walk->one == NULL)
+		return GIT_EREVWALKOVER;
+
 	/* first figure out what the merge bases are */
 	if (merge_bases_many(&bases, walk, walk->one, &walk->twos) < 0)
 		return -1;

From f184836bd281efe8a656e3a9c6c2f9c040b88119 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= <carlos@cmartin.tk>
Date: Wed, 25 Apr 2012 12:13:20 +0200
Subject: [PATCH 6/6] remote: run a callback when updating the branch tips

This allows the caller to update an internal structure or update the
user output with the tips that were updated.

While in the area, only try to update the ref if the value is
different from its old one.
---
 examples/network/Makefile |  2 +-
 examples/network/fetch.c  | 21 ++++++++++++++++++++-
 include/git2/remote.h     |  6 ++----
 src/remote.c              | 26 +++++++++++++++++++++++---
 4 files changed, 46 insertions(+), 9 deletions(-)

diff --git a/examples/network/Makefile b/examples/network/Makefile
index ed0c2099f..c21869ac9 100644
--- a/examples/network/Makefile
+++ b/examples/network/Makefile
@@ -2,7 +2,7 @@ default: all
 
 CC = gcc
 CFLAGS += -g
-CFLAGS += -I../../include -L../../ -lgit2
+CFLAGS += -I../../include -L../../ -lgit2 -lpthread
 
 OBJECTS = \
   git2.o \
diff --git a/examples/network/fetch.c b/examples/network/fetch.c
index f7a60640e..d4a39746f 100644
--- a/examples/network/fetch.c
+++ b/examples/network/fetch.c
@@ -39,6 +39,25 @@ exit:
 	pthread_exit(&data->ret);
 }
 
+int update_cb(const char *refname, const git_oid *a, const git_oid *b)
+{
+	const char *action;
+	char a_str[GIT_OID_HEXSZ+1], b_str[GIT_OID_HEXSZ+1];
+
+	git_oid_fmt(b_str, b);
+	b_str[GIT_OID_HEXSZ] = '\0';
+
+	if (git_oid_iszero(a)) {
+		printf("[new]     %.20s %s\n", b_str, refname);
+	} else {
+		git_oid_fmt(a_str, a);
+		a_str[GIT_OID_HEXSZ] = '\0';
+		printf("[updated] %.10s..%.10s %s\n", a_str, b_str, refname);
+	}
+
+	return 0;
+}
+
 int fetch(git_repository *repo, int argc, char **argv)
 {
   git_remote *remote = NULL;
@@ -78,7 +97,7 @@ int fetch(git_repository *repo, int argc, char **argv)
   // right commits. This may be needed even if there was no packfile
   // to download, which can happen e.g. when the branches have been
   // changed but all the neede objects are available locally.
-  if (git_remote_update_tips(remote) < 0)
+  if (git_remote_update_tips(remote, update_cb) < 0)
 	  return -1;
 
   git_remote_free(remote);
diff --git a/include/git2/remote.h b/include/git2/remote.h
index 8f49fddf1..09b927e28 100644
--- a/include/git2/remote.h
+++ b/include/git2/remote.h
@@ -183,12 +183,10 @@ GIT_EXTERN(void) git_remote_free(git_remote *remote);
 /**
  * Update the tips to the new state
  *
- * Make sure that you only call this once you've successfully indexed
- * or expanded the packfile.
- *
  * @param remote the remote to update
+ * @param cb callback to run on each ref update. 'a' is the old value, 'b' is then new value
  */
-GIT_EXTERN(int) git_remote_update_tips(git_remote *remote);
+GIT_EXTERN(int) git_remote_update_tips(git_remote *remote, int (*cb)(const char *refname, const git_oid *a, const git_oid *b));
 
 /**
  * Return whether a string is a valid remote URL
diff --git a/src/remote.c b/src/remote.c
index bbb491dd8..e1937df85 100644
--- a/src/remote.c
+++ b/src/remote.c
@@ -309,11 +309,12 @@ int git_remote_download(git_remote *remote, git_off_t *bytes, git_indexer_stats
 	return git_fetch_download_pack(remote, bytes, stats);
 }
 
-int git_remote_update_tips(git_remote *remote)
+int git_remote_update_tips(git_remote *remote, int (*cb)(const char *refname, const git_oid *a, const git_oid *b))
 {
 	int error = 0;
 	unsigned int i = 0;
 	git_buf refname = GIT_BUF_INIT;
+	git_oid old;
 	git_vector *refs = &remote->refs;
 	git_remote_head *head;
 	git_reference *ref;
@@ -338,17 +339,36 @@ int git_remote_update_tips(git_remote *remote)
 		head = refs->contents[i];
 
 		if (git_refspec_transform_r(&refname, spec, head->name) < 0)
-			break;
+			goto on_error;
+
+		error = git_reference_name_to_oid(&old, remote->repo, refname.ptr);
+		if (error < 0 && error != GIT_ENOTFOUND)
+			goto on_error;
+
+		if (error == GIT_ENOTFOUND)
+			memset(&old, 0, GIT_OID_RAWSZ);
+
+		if (!git_oid_cmp(&old, &head->oid))
+			continue;
 
 		if (git_reference_create_oid(&ref, remote->repo, refname.ptr, &head->oid, 1) < 0)
 			break;
 
 		git_reference_free(ref);
+
+		if (cb != NULL) {
+			if (cb(refname.ptr, &old, &head->oid) < 0)
+				goto on_error;
+		}
 	}
 
 	git_buf_free(&refname);
+	return 0;
+
+on_error:
+	git_buf_free(&refname);
+	return -1;
 
-	return error;
 }
 
 int git_remote_connected(git_remote *remote)