network: add sideband support

This lets us notify the user of what the remote end is doing while we
wait for it to start sending us the packfile.

include/git2/remote.h

@@ -287,7 +287,7 @@ typedef enum git_remote_completion_type {
  * Set the calbacks to be called by the remote.
  */
 struct git_remote_callbacks {
-	int (*progress)(const char *str, void *data);
+	void (*progress)(const char *str, int len, void *data);
 	int (*completion)(git_remote_completion_type type, void *data);
 	int (*update_tips)(const char *refname, const git_oid *a, const git_oid *b, void *data);
 	void *data;

src/fetch.c

@@ -292,6 +292,31 @@ int git_fetch_download_pack(git_remote *remote, git_off_t *bytes, git_indexer_st
 
 }
 
+static int no_sideband(git_indexer_stream *idx, gitno_buffer *buf, git_off_t *bytes, git_indexer_stats *stats)
+{
+	int recvd;
+
+	do {
+		if (git_indexer_stream_add(idx, buf->data, buf->offset, stats) < 0)
+			return -1;
+
+		gitno_consume_n(buf, buf->offset);
+
+		if ((recvd = gitno_recv(buf)) < 0)
+			return -1;
+
+		*bytes += recvd;
+	} while(recvd > 0 && stats->data_received);
+
+	if (!stats->data_received)
+		giterr_set(GITERR_NET, "Early EOF while downloading packfile");
+
+	if (git_indexer_stream_finalize(idx, stats))
+		return -1;
+
+	return 0;
+}
+
 /* Receiving data from a socket and storing it is pretty much the same for git and HTTP */
 int git_fetch__download_pack(
 	git_transport *t,
@@ -299,7 +324,6 @@ int git_fetch__download_pack(
 	git_off_t *bytes,
 	git_indexer_stats *stats)
 {
-	int recvd;
 	git_buf path = GIT_BUF_INIT;
 	gitno_buffer *buf = &t->buffer;
 	git_indexer_stream *idx = NULL;
@@ -314,23 +338,49 @@ int git_fetch__download_pack(
 	memset(stats, 0, sizeof(git_indexer_stats));
 	*bytes = 0;
 
+	/*
+	 * If the remote doesn't support the side-band, we can feed
+	 * the data directly to the indexer. Otherwise, we need to
+	 * check which one belongs there.
+	 */
+	if (!t->caps.side_band && !t->caps.side_band_64k) {
+		if (no_sideband(idx, buf, bytes, stats) < 0)
+			goto on_error;
+
+		git_indexer_stream_free(idx);
+		return 0;
+	}
+
 	do {
-		if (git_indexer_stream_add(idx, buf->data, buf->offset, stats) < 0)
+		git_pkt *pkt;
+		if (recv_pkt(&pkt, buf) < 0)
 			goto on_error;
 
-		gitno_consume_n(buf, buf->offset);
+		if (pkt->type == GIT_PKT_PROGRESS) {
+			if (t->progress_cb) {
+				git_pkt_progress *p = (git_pkt_progress *) pkt;
+				t->progress_cb(p->data, p->len, t->cb_data);
+			}
+			git__free(pkt);
+		} else if (pkt->type == GIT_PKT_DATA) {
+			git_pkt_data *p = (git_pkt_data *) pkt;
+			*bytes += p->len;
+			if (git_indexer_stream_add(idx, p->data, p->len, stats) < 0)
+				goto on_error;
 
-		if ((recvd = gitno_recv(buf)) < 0)
+			git__free(pkt);
-			goto on_error;
+		} else if (pkt->type == GIT_PKT_FLUSH) {
-
+			/* A flush indicates the end of the packfile */
-		*bytes += recvd;
+			git__free(pkt);
-	} while(recvd > 0 && !stats->data_received);
+			break;
+		}
+	} while (!stats->data_received);
 
 	if (!stats->data_received)
 		giterr_set(GITERR_NET, "Early EOF while downloading packfile");
 
 	if (git_indexer_stream_finalize(idx, stats))
-		goto on_error;
+		return -1;
 
 	git_indexer_stream_free(idx);
 	return 0;

src/pkt.c

@@ -17,6 +17,7 @@
 #include "netops.h"
 #include "posix.h"
 #include "buffer.h"
+#include "protocol.h"
 
 #include <ctype.h>
 
@@ -130,6 +131,42 @@ static int err_pkt(git_pkt **out, const char *line, size_t len)
 	return 0;
 }
 
+static int data_pkt(git_pkt **out, const char *line, size_t len)
+{
+	git_pkt_data *pkt;
+
+	line++;
+	len--;
+	pkt = git__malloc(sizeof(git_pkt_data) + len);
+	GITERR_CHECK_ALLOC(pkt);
+
+	pkt->type = GIT_PKT_DATA;
+	pkt->len = (int) len;
+	memcpy(pkt->data, line, len);
+
+	*out = (git_pkt *) pkt;
+
+	return 0;
+}
+
+static int progress_pkt(git_pkt **out, const char *line, size_t len)
+{
+	git_pkt_progress *pkt;
+
+	line++;
+	len--;
+	pkt = git__malloc(sizeof(git_pkt_progress) + len);
+	GITERR_CHECK_ALLOC(pkt);
+
+	pkt->type = GIT_PKT_PROGRESS;
+	pkt->len = (int) len;
+	memcpy(pkt->data, line, len);
+
+	*out = (git_pkt *) pkt;
+
+	return 0;
+}
+
 /*
  * Parse an other-ref line.
  */
@@ -263,8 +300,11 @@ int git_pkt_parse_line(
 
 	len -= PKT_LEN_SIZE; /* the encoded length includes its own size */
 
-	/* Assming the minimal size is actually 4 */
+	if (*line == GIT_SIDE_BAND_DATA)
-	if (!git__prefixcmp(line, "ACK"))
+		ret = data_pkt(head, line, len);
+	else if (*line == GIT_SIDE_BAND_PROGRESS)
+		ret = progress_pkt(head, line, len);
+	else if (!git__prefixcmp(line, "ACK"))
 		ret = ack_pkt(head, line, len);
 	else if (!git__prefixcmp(line, "NAK"))
 		ret = nak_pkt(head);
@@ -301,6 +341,13 @@ static int buffer_want_with_caps(git_remote_head *head, git_transport_caps *caps
 	char oid[GIT_OID_HEXSZ +1] = {0};
 	unsigned int len;
 
+	/* Prefer side-band-64k if the server supports both */
+	if (caps->side_band) {
+		if (caps->side_band_64k)
+			git_buf_printf(&str, "%s ", GIT_CAP_SIDE_BAND_64K);
+		else
+			git_buf_printf(&str, "%s ", GIT_CAP_SIDE_BAND);
+	}
 	if (caps->ofs_delta)
 		git_buf_puts(&str, GIT_CAP_OFS_DELTA " ");
 

src/pkt.h

@@ -24,6 +24,8 @@ enum git_pkt_type {
 	GIT_PKT_PACK,
 	GIT_PKT_COMMENT,
 	GIT_PKT_ERR,
+	GIT_PKT_DATA,
+	GIT_PKT_PROGRESS,
 };
 
 /* Used for multi-ack */
@@ -65,6 +67,14 @@ typedef struct {
 	char comment[GIT_FLEX_ARRAY];
 } git_pkt_comment;
 
+typedef struct {
+	enum git_pkt_type type;
+	int len;
+	char data[GIT_FLEX_ARRAY];
+} git_pkt_data;
+
+typedef git_pkt_data git_pkt_progress;
+
 typedef struct {
 	enum git_pkt_type type;
 	char error[GIT_FLEX_ARRAY];

src/protocol.c

@@ -80,6 +80,20 @@ int git_protocol_detect_caps(git_pkt_ref *pkt, git_transport_caps *caps)
 			continue;
 		}
 
+		/* Keep side-band check after side-band-64k */
+		if(!git__prefixcmp(ptr, GIT_CAP_SIDE_BAND_64K)) {
+			caps->common = caps->side_band_64k = 1;
+			ptr += strlen(GIT_CAP_SIDE_BAND_64K);
+			continue;
+		}
+
+		if(!git__prefixcmp(ptr, GIT_CAP_SIDE_BAND)) {
+			caps->common = caps->side_band = 1;
+			ptr += strlen(GIT_CAP_SIDE_BAND);
+			continue;
+		}
+
+
 		/* We don't know this capability, so skip it */
 		ptr = strchr(ptr, ' ');
 	}

src/protocol.h

@@ -14,4 +14,8 @@
 int git_protocol_store_refs(git_transport *t, int flushes);
 int git_protocol_detect_caps(git_pkt_ref *pkt, git_transport_caps *caps);
 
+#define GIT_SIDE_BAND_DATA     1
+#define GIT_SIDE_BAND_PROGRESS 2
+#define GIT_SIDE_BAND_ERROR    3
+
 #endif

src/remote.c

@@ -386,6 +386,9 @@ int git_remote_connect(git_remote *remote, int direction)
 	if (git_transport_new(&t, url) < 0)
 		return -1;
 
+	t->progress_cb = remote->callbacks.progress;
+	t->cb_data = remote->callbacks.data;
+
 	t->check_cert = remote->check_cert;
 	if (t->connect(t, direction) < 0) {
 		goto on_error;
@@ -646,4 +649,9 @@ void git_remote_set_callbacks(git_remote *remote, git_remote_callbacks *callback
 	assert(remote && callbacks);
 
 	memcpy(&remote->callbacks, callbacks, sizeof(git_remote_callbacks));
+
+	if (remote->transport) {
+		remote->transport->progress_cb = remote->callbacks.progress;
+		remote->transport->cb_data = remote->callbacks.data;
+	}
 }

src/transport.h

@@ -21,11 +21,15 @@
 
 #define GIT_CAP_OFS_DELTA "ofs-delta"
 #define GIT_CAP_MULTI_ACK "multi_ack"
+#define GIT_CAP_SIDE_BAND "side-band"
+#define GIT_CAP_SIDE_BAND_64K "side-band-64k"
 
 typedef struct git_transport_caps {
 	int common:1,
 		ofs_delta:1,
-		multi_ack: 1;
+		multi_ack: 1,
+		side_band:1,
+		side_band_64k:1;
 } git_transport_caps;
 
 #ifdef GIT_SSL
@@ -84,6 +88,7 @@ struct git_transport {
 	gitno_buffer buffer;
 	GIT_SOCKET socket;
 	git_transport_caps caps;
+	void *cb_data;
 	/**
 	 * Connect and store the remote heads
 	 */
@@ -113,6 +118,11 @@ struct git_transport {
 	 * Free the associated resources
 	 */
 	void (*free)(struct git_transport *transport);
+	/**
+	 * Callbacks for the progress and error output
+	 */
+	void (*progress_cb)(const char *str, int len, void *data);
+	void (*error_cb)(const char *str, int len, void *data);
 };
 
 

src/transports/git.c

@@ -24,7 +24,7 @@
 
 typedef struct {
 	git_transport parent;
-	char buff[1024];
+	char buff[65536];
 #ifdef GIT_WIN32
 	WSADATA wsd;
 #endif