Merge remote-tracking branch 'ben/local-transport' into development

src/transports/local.c

@@ -10,11 +10,19 @@
 #include "git2/repository.h"
 #include "git2/object.h"
 #include "git2/tag.h"
-#include "refs.h"
 #include "git2/transport.h"
+#include "git2/revwalk.h"
+#include "git2/odb_backend.h"
+#include "git2/pack.h"
+#include "git2/commit.h"
+#include "git2/revparse.h"
+#include "pack-objects.h"
+#include "refs.h"
 #include "posix.h"
 #include "path.h"
 #include "buffer.h"
+#include "repository.h"
+#include "odb.h"
 
 typedef struct {
 	git_transport parent;
@@ -34,7 +42,7 @@ static int add_ref(transport_local *t, const char *name)
 	git_object *obj = NULL, *target = NULL;
 	git_buf buf = GIT_BUF_INIT;
 
-	head = (git_remote_head *)git__malloc(sizeof(git_remote_head));
+	head = (git_remote_head *)git__calloc(1, sizeof(git_remote_head));
 	GITERR_CHECK_ALLOC(head);
 
 	head->name = git__strdup(name);
@@ -200,15 +208,134 @@ static int local_ls(git_transport *transport, git_headlist_cb list_cb, void *pay
 static int local_negotiate_fetch(
 	git_transport *transport,
 	git_repository *repo,
-	const git_remote_head * const *refs, size_t count)
+	const git_remote_head * const *refs,
+	size_t count)
 {
-	GIT_UNUSED(transport);
-	GIT_UNUSED(repo);
+	transport_local *t = (transport_local*)transport;
+	git_remote_head *rhead;
+	unsigned int i;
+
 	GIT_UNUSED(refs);
 	GIT_UNUSED(count);
 
-	giterr_set(GITERR_NET, "Fetch via local transport isn't implemented. Sorry");
-	return -1;
+	/* Fill in the loids */
+	git_vector_foreach(&t->refs, i, rhead) {
+		git_object *obj;
+
+		int error = git_revparse_single(&obj, repo, rhead->name);
+		if (!error)
+			git_oid_cpy(&rhead->loid, git_object_id(obj));
+		else if (error != GIT_ENOTFOUND)
+			return error;
+		giterr_clear();
+	}
+
+	return 0;
+}
+
+typedef struct foreach_data {
+	git_transfer_progress *stats;
+	git_transfer_progress_callback progress_cb;
+	void *progress_payload;
+	git_odb_writepack *writepack;
+} foreach_data;
+
+static int foreach_cb(void *buf, size_t len, void *payload)
+{
+	foreach_data *data = (foreach_data*)payload;
+
+	data->stats->received_bytes += len;
+	return data->writepack->add(data->writepack, buf, len, data->stats);
+}
+
+static int local_download_pack(
+		git_transport *transport,
+		git_repository *repo,
+		git_transfer_progress *stats,
+		git_transfer_progress_callback progress_cb,
+		void *progress_payload)
+{
+	transport_local *t = (transport_local*)transport;
+	git_revwalk *walk = NULL;
+	git_remote_head *rhead;
+	unsigned int i;
+	int error = -1;
+	git_oid oid;
+	git_packbuilder *pack = NULL;
+	git_odb_writepack *writepack = NULL;
+	git_odb *odb = NULL;
+
+	if ((error = git_revwalk_new(&walk, t->repo)) < 0)
+		goto cleanup;
+	git_revwalk_sorting(walk, GIT_SORT_TIME);
+
+	if ((error = git_packbuilder_new(&pack, t->repo)) < 0)
+		goto cleanup;
+
+	stats->total_objects = 0;
+	stats->indexed_objects = 0;
+	stats->received_objects = 0;
+	stats->received_bytes = 0;
+
+	git_vector_foreach(&t->refs, i, rhead) {
+		git_object *obj;
+		if ((error = git_object_lookup(&obj, t->repo, &rhead->oid, GIT_OBJ_ANY)) < 0)
+			goto cleanup;
+
+		if (git_object_type(obj) == GIT_OBJ_COMMIT) {
+			/* Revwalker includes only wanted commits */
+			error = git_revwalk_push(walk, &rhead->oid);
+			if (!git_oid_iszero(&rhead->loid))
+				error = git_revwalk_hide(walk, &rhead->loid);
+		} else {
+			/* Tag or some other wanted object. Add it on its own */
+			error = git_packbuilder_insert(pack, &rhead->oid, rhead->name);
+		}
+      git_object_free(obj);
+	}
+
+	/* Walk the objects, building a packfile */
+	if ((error = git_repository_odb__weakptr(&odb, repo)) < 0)
+		goto cleanup;
+
+	while ((error = git_revwalk_next(&oid, walk)) == 0) {
+		git_commit *commit;
+
+		/* Skip commits we already have */
+		if (git_odb_exists(odb, &oid)) continue;
+
+		if (!git_object_lookup((git_object**)&commit, t->repo, &oid, GIT_OBJ_COMMIT)) {
+			const git_oid *tree_oid = git_commit_tree_oid(commit);
+			git_commit_free(commit);
+
+			/* Add the commit and its tree */
+			if ((error = git_packbuilder_insert(pack, &oid, NULL)) < 0 ||
+				 (error = git_packbuilder_insert_tree(pack, tree_oid)) < 0)
+				goto cleanup;
+		}
+	}
+
+	if ((error = git_odb_write_pack(&writepack, odb, progress_cb, progress_payload)) < 0)
+		goto cleanup;
+
+	/* Write the data to the ODB */
+	{
+		foreach_data data = {0};
+		data.stats = stats;
+		data.progress_cb = progress_cb;
+		data.progress_payload = progress_payload;
+		data.writepack = writepack;
+
+		if ((error = git_packbuilder_foreach(pack, foreach_cb, &data)) < 0)
+			goto cleanup;
+	}
+	error = writepack->commit(writepack, stats);
+
+cleanup:
+	if (writepack) writepack->free(writepack);
+	git_packbuilder_free(pack);
+	git_revwalk_free(walk);
+	return error;
 }
 
 static int local_is_connected(git_transport *transport, int *connected)
@@ -283,6 +410,7 @@ int git_transport_local(git_transport **out, void *param)
 		
 	t->parent.connect = local_connect;
 	t->parent.negotiate_fetch = local_negotiate_fetch;
+	t->parent.download_pack = local_download_pack;
 	t->parent.close = local_close;
 	t->parent.free = local_free;
 	t->parent.ls = local_ls;

tests-clar/clar_helpers.c

@@ -1,5 +1,6 @@
 #include "clar_libgit2.h"
 #include "posix.h"
+#include "path.h"
 
 void clar_on_init(void)
 {
@@ -222,3 +223,51 @@ bool cl_is_chmod_supported(void)
 	return _is_supported;
 }
 
+const char* cl_git_fixture_url(const char *fixturename)
+{
+	return cl_git_path_url(cl_fixture(fixturename));
+}
+
+const char* cl_git_path_url(const char *path)
+{
+	static char url[4096];
+
+	const char *in_buf;
+	git_buf path_buf = GIT_BUF_INIT;
+	git_buf url_buf = GIT_BUF_INIT;
+
+	cl_git_pass(git_path_prettify_dir(&path_buf, path, NULL));
+	cl_git_pass(git_buf_puts(&url_buf, "file://"));
+
+#ifdef _MSC_VER
+	/*
+	 * A FILE uri matches the following format: file://[host]/path
+	 * where "host" can be empty and "path" is an absolute path to the resource.
+	 *
+	 * In this test, no hostname is used, but we have to ensure the leading triple slashes:
+	 *
+	 * *nix: file:///usr/home/...
+	 * Windows: file:///C:/Users/...
+	 */
+	cl_git_pass(git_buf_putc(url_buf, '/'));
+#endif
+
+	in_buf = git_buf_cstr(&path_buf);
+
+	/*
+	 * A very hacky Url encoding that only takes care of escaping the spaces
+	 */
+	while (*in_buf) {
+		if (*in_buf == ' ')
+			cl_git_pass(git_buf_puts(&url_buf, "%20"));
+		else
+			cl_git_pass(git_buf_putc(&url_buf, *in_buf));
+
+		in_buf++;
+	}
+
+	strncpy(url, git_buf_cstr(&url_buf), 4096);
+	git_buf_free(&url_buf);
+	git_buf_free(&path_buf);
+	return url;
+}

tests-clar/clar_libgit2.h

@@ -57,4 +57,8 @@ int cl_rename(const char *source, const char *dest);
 git_repository *cl_git_sandbox_init(const char *sandbox);
 void cl_git_sandbox_cleanup(void);
 
+/* Local-repo url helpers */
+const char* cl_git_fixture_url(const char *fixturename);
+const char* cl_git_path_url(const char *path);
+
 #endif

tests-clar/clone/nonetwork.c

@@ -3,7 +3,6 @@
 #include "git2/clone.h"
 #include "repository.h"
 
-#define DO_LOCAL_TEST 0
 #define LIVE_REPO_URL "git://github.com/libgit2/TestGitRepository"
 
 static git_repository *g_repo;
@@ -20,46 +19,6 @@ static void cleanup_repository(void *path)
 	cl_fixture_cleanup((const char *)path);
 }
 
-// TODO: This is copy/pasted from network/remotelocal.c.
-static void build_local_file_url(git_buf *out, const char *fixture)
-{
-	const char *in_buf;
-
-	git_buf path_buf = GIT_BUF_INIT;
-
-	cl_git_pass(git_path_prettify_dir(&path_buf, fixture, NULL));
-	cl_git_pass(git_buf_puts(out, "file://"));
-
-#ifdef GIT_WIN32
-	/*
-	 * A FILE uri matches the following format: file://[host]/path
-	 * where "host" can be empty and "path" is an absolute path to the resource.
-	 *
-	 * In this test, no hostname is used, but we have to ensure the leading triple slashes:
-	 *
-	 * *nix: file:///usr/home/...
-	 * Windows: file:///C:/Users/...
-	 */
-	cl_git_pass(git_buf_putc(out, '/'));
-#endif
-
-	in_buf = git_buf_cstr(&path_buf);
-
-	/*
-	 * A very hacky Url encoding that only takes care of escaping the spaces
-	 */
-	while (*in_buf) {
-		if (*in_buf == ' ')
-			cl_git_pass(git_buf_puts(out, "%20"));
-		else
-			cl_git_pass(git_buf_putc(out, *in_buf));
-
-		in_buf++;
-	}
-
-	git_buf_free(&path_buf);
-}
-
 void test_clone_nonetwork__bad_url(void)
 {
 	/* Clone should clean up the mess if the URL isn't a git repository */
@@ -71,30 +30,18 @@ void test_clone_nonetwork__bad_url(void)
 
 void test_clone_nonetwork__local(void)
 {
-	git_buf src = GIT_BUF_INIT;
-	build_local_file_url(&src, cl_fixture("testrepo.git"));
-
-#if DO_LOCAL_TEST
+	const char *src = cl_git_fixture_url("testrepo.git");
 	cl_set_cleanup(&cleanup_repository, "./local");
 
-	cl_git_pass(git_clone(&g_repo, git_buf_cstr(&src), "./local", NULL, NULL, NULL));
-#endif
-
-	git_buf_free(&src);
+	cl_git_pass(git_clone(&g_repo, src, "./local", NULL, NULL, NULL));
 }
 
 void test_clone_nonetwork__local_bare(void)
 {
-	git_buf src = GIT_BUF_INIT;
-	build_local_file_url(&src, cl_fixture("testrepo.git"));
-
-#if DO_LOCAL_TEST
+	const char *src = cl_git_fixture_url("testrepo.git");
 	cl_set_cleanup(&cleanup_repository, "./local.git");
 
-	cl_git_pass(git_clone_bare(&g_repo, git_buf_cstr(&src), "./local.git", NULL, NULL));
-#endif
-
-	git_buf_free(&src);
+	cl_git_pass(git_clone_bare(&g_repo, src, "./local.git", NULL, NULL));
 }
 
 void test_clone_nonetwork__fail_when_the_target_is_a_file(void)

tests-clar/network/fetchlocal.c

@@ -0,0 +1,63 @@
+#include "clar_libgit2.h"
+
+#include "buffer.h"
+#include "path.h"
+#include "remote.h"
+
+static void transfer_cb(const git_transfer_progress *stats, void *payload)
+{
+	int *callcount = (int*)payload;
+	GIT_UNUSED(stats);
+	(*callcount)++;
+}
+
+void test_network_fetchlocal__complete(void)
+{
+	git_repository *repo;
+	git_remote *origin;
+	int callcount = 0;
+	git_strarray refnames = {0};
+
+	const char *url = cl_git_fixture_url("testrepo.git");
+	cl_git_pass(git_repository_init(&repo, "foo", true));
+
+	cl_git_pass(git_remote_add(&origin, repo, GIT_REMOTE_ORIGIN, url));
+	cl_git_pass(git_remote_connect(origin, GIT_DIR_FETCH));
+	cl_git_pass(git_remote_download(origin, transfer_cb, &callcount));
+	cl_git_pass(git_remote_update_tips(origin));
+
+	cl_git_pass(git_reference_list(&refnames, repo, GIT_REF_LISTALL));
+	cl_assert_equal_i(18, refnames.count);
+	cl_assert(callcount > 0);
+
+	git_strarray_free(&refnames);
+	git_remote_free(origin);
+	git_repository_free(repo);
+}
+
+void test_network_fetchlocal__partial(void)
+{
+	git_repository *repo = cl_git_sandbox_init("partial-testrepo");
+	git_remote *origin;
+	int callcount = 0;
+	git_strarray refnames = {0};
+	const char *url;
+
+	cl_git_pass(git_reference_list(&refnames, repo, GIT_REF_LISTALL));
+	cl_assert_equal_i(1, refnames.count);
+
+	url = cl_git_fixture_url("testrepo.git");
+	cl_git_pass(git_remote_add(&origin, repo, GIT_REMOTE_ORIGIN, url));
+	cl_git_pass(git_remote_connect(origin, GIT_DIR_FETCH));
+	cl_git_pass(git_remote_download(origin, transfer_cb, &callcount));
+	cl_git_pass(git_remote_update_tips(origin));
+
+	cl_git_pass(git_reference_list(&refnames, repo, GIT_REF_LISTALL));
+	cl_assert_equal_i(19, refnames.count); /* 18 remote + 1 local */
+	cl_assert(callcount > 0);
+
+	git_strarray_free(&refnames);
+	git_remote_free(origin);
+
+	cl_git_sandbox_cleanup();
+}

tests-clar/network/remotelocal.c

@@ -7,45 +7,6 @@ static git_repository *repo;
 static git_buf file_path_buf = GIT_BUF_INIT;
 static git_remote *remote;
 
-static void build_local_file_url(git_buf *out, const char *fixture)
-{
-	const char *in_buf;
-
-	git_buf path_buf = GIT_BUF_INIT;
-
-	cl_git_pass(git_path_prettify_dir(&path_buf, fixture, NULL));
-	cl_git_pass(git_buf_puts(out, "file://"));
-
-#ifdef _MSC_VER
-	/*
-	 * A FILE uri matches the following format: file://[host]/path
-	 * where "host" can be empty and "path" is an absolute path to the resource.
-	 * 
-	 * In this test, no hostname is used, but we have to ensure the leading triple slashes:
-	 * 
-	 * *nix: file:///usr/home/...
-	 * Windows: file:///C:/Users/...
-	 */
-	cl_git_pass(git_buf_putc(out, '/'));
-#endif
-
-	in_buf = git_buf_cstr(&path_buf);
-
-	/*
-	 * A very hacky Url encoding that only takes care of escaping the spaces
-	 */
-	while (*in_buf) {
-		if (*in_buf == ' ')
-			cl_git_pass(git_buf_puts(out, "%20"));
-		else
-			cl_git_pass(git_buf_putc(out, *in_buf));
-
-		in_buf++;
-	}
-
-	git_buf_free(&path_buf);
-}
-
 void test_network_remotelocal__initialize(void)
 {
 	cl_git_pass(git_repository_init(&repo, "remotelocal/", 0));
@@ -82,7 +43,7 @@ static int ensure_peeled__cb(git_remote_head *head, void *payload)
 
 static void connect_to_local_repository(const char *local_repository)
 {
-	build_local_file_url(&file_path_buf, local_repository);
+	git_buf_sets(&file_path_buf, cl_git_path_url(local_repository));
 
 	cl_git_pass(git_remote_new(&remote, repo, NULL, git_buf_cstr(&file_path_buf), NULL));
 	cl_git_pass(git_remote_connect(remote, GIT_DIR_FETCH));

tests-clar/resources/partial-testrepo/.gitted/HEAD

@@ -0,0 +1 @@
+ref: refs/heads/dir

tests-clar/resources/partial-testrepo/.gitted/config

@@ -0,0 +1,7 @@
+[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = false
+	logallrefupdates = true
+	ignorecase = true
+[branch "dir"]

tests-clar/resources/partial-testrepo/.gitted/objects/4a/202b346bb0fb0db7eff3cffeb3c70babbd2045

@@ -0,0 +1,2 @@
+x
<01>ŽQ
+Â0DýÎ)öÊ6›Í¦ "xO°‰‰-ØFb¼¿EoàÏ0¼Ç¤º,ske×[ÎPn8R,EpD?±gŸ}Ê^3²âÙ<µåµGŽhYKÄèÒ8ЖDA<44>É)¿ÉÈ;gôݧÚàšjïp™4ÕŽ¯ô-çû¢óã<C3B3>êr‚ÁŠ;°s°GA4Ûº=ìùÖ(ôin7øIÌKÍFE

tests-clar/resources/partial-testrepo/.gitted/objects/5b/5b025afb0b4c913b4c338a42934a3863bf3644

@@ -0,0 +1,2 @@
+x
ťŽŰ	1EýNi@™Ék2 "X‚$ŮYW0Yc˙íŔżĂ…s¸ĄŐzďÚÚőMDĎ
€0ćś8!¶†ÉĚŢs‰XŠŞgÚdí::@X0»P˘wŮ"F/‰‰śÍRŕ<>Uz÷ĄmúZZďú˛¤ŇV}|•/śo5݇ŇęIŁ!¬1z Ć:vůÇUim}ę/˘>
+öF-

tests-clar/resources/partial-testrepo/.gitted/objects/9f/d738e8f7967c078dceed8190330fc8648ee56a

@@ -0,0 +1,3 @@
+x
<01><>[
+Т0E§Ю*fЪфеЄ "ИW0<57><30>-иFтtџн<><EFBFBD>чpS[<5B>YР<59>x<EFBFBD>^
+Dэb	CLhutЩ<18>}Ѕ8X*4ZэЌsYНЈ<D09D>UР<55>AУж
ЬX3<>RЋMЕЖ) s6шМЂMІжс<D0B6><D181>м&Jm<4A>ѓ;}ЧѕБаќ<ЅЖ\@<40>р<EFBFBD>бо<D0B1>pФ<70>ЈvК?<3F>ђ<EFBFBD>ЋjлКL№ЋЈи?Hх

tests-clar/resources/partial-testrepo/.gitted/objects/a4/a7dce85cf63874e984719f4fdd239f5145052f

@@ -0,0 +1,2 @@
+x
ťŹ;j1DëťmdÓú·Ŕ<C2B7>ÇŽ|M«µ3`ŤŚV{>€łâQŻ ¸·vL0I?Í!š4–Z=Ę! ×¦8˛F˘Ă’!rÖsQßyČ9<C48C>]$DŽ&„l6AÇ>jFWüҵIKNiűë§Z˘%ˇS<>Ś‘
+‹Ň	­Ĺʉř<E280B0>U~Ě˝řä>'Ľď™ű	Żwţ
×[ËÇ×÷öÚDGÚˇ±đŚQ-şMůŹ«>dܶ‘OŢáŇň}í\ŕ8g_ШÂoYr

tests-clar/resources/partial-testrepo/.gitted/objects/c4/7800c7266a2be04c571c04d5a6614691ea99bd

@@ -0,0 +1,3 @@
+x
<01>ŽQ
+Â0DýÎ)öʦ»I<‚'ØlR+˜Fj¼¿EoàÏ0<xÃh«õÞa Üõµ]È™­åXUlÞPF)Åz‘4yó”µ,\r	'SÂÄ-mI4
+‘Xhô”&òÌFÞ}n+\µõ—Y´-p|é·œoUî<55>ƒ¶z;-<2D>‘a<E28098>
Ñlt{ØË?®I«,:ÃoÚR̳cHK

tests-clar/resources/partial-testrepo/.gitted/refs/heads/dir

@@ -0,0 +1 @@
+144344043ba4d4a405da03de3844aa829ae8be0e