diff --git a/src/fetch.c b/src/fetch.c
index e83dc4add..295b036ef 100644
--- a/src/fetch.c
+++ b/src/fetch.c
@@ -23,7 +23,6 @@ struct filter_payload {
 	git_remote *remote;
 	const git_refspec *spec, *tagspec;
 	git_odb *odb;
-	int want_head;
 };
 
 static int filter_ref__cb(git_remote_head *head, void *payload)
@@ -34,9 +33,7 @@ static int filter_ref__cb(git_remote_head *head, void *payload)
 	if (!git_reference_is_valid_name(head->name))
 		return 0;
 
-	if ((strcmp(head->name, GIT_HEAD_FILE) == 0) && p->want_head) {
-		match = 1;
-	} else if (p->remote->download_tags == GIT_REMOTE_DOWNLOAD_TAGS_ALL) {
+	if (p->remote->download_tags == GIT_REMOTE_DOWNLOAD_TAGS_ALL) {
 		/*
 		 * If tagopt is --tags, then we only use the default
 		 * tags refspec and ignore the remote's
@@ -63,10 +60,10 @@ static int filter_ref__cb(git_remote_head *head, void *payload)
 static int filter_wants(git_remote *remote)
 {
 	struct filter_payload p;
-	git_refspec tagspec;
+	git_refspec tagspec, head;
 	int error = -1;
 
-	git_vector_clear(&remote->refs);
+	//git_vector_clear(&remote->refs);
 	if (git_refspec__parse(&tagspec, GIT_REFSPEC_TAGS, true) < 0)
 		return error;
 
@@ -78,8 +75,15 @@ static int filter_wants(git_remote *remote)
 	 */
 	p.tagspec = &tagspec;
 	p.remote = remote;
-	if (remote->refspecs.length == 0)
-		p.want_head = 1;
+	if (remote->active_refspecs.length == 0) {
+		if ((error = git_refspec__parse(&head, "HEAD", true)) < 0)
+			goto cleanup;
+
+		if ((error = git_refspec__dwim_one(&remote->active_refspecs, &head, &remote->refs)) < 0)
+			goto cleanup;
+	}
+
+	git_vector_clear(&remote->refs);
 
 	if (git_repository_odb__weakptr(&p.odb, remote->repo) < 0)
 		goto cleanup;
@@ -107,7 +111,7 @@ int git_fetch_negotiate(git_remote *remote)
 	}
 
 	/* Don't try to negotiate when we don't want anything */
-	if (remote->refs.length == 0 || !remote->need_pack)
+	if (!remote->need_pack)
 		return 0;
 
 	/*
diff --git a/src/refspec.c b/src/refspec.c
index 492c6ed3f..a97340071 100644
--- a/src/refspec.c
+++ b/src/refspec.c
@@ -12,6 +12,7 @@
 #include "util.h"
 #include "posix.h"
 #include "refs.h"
+#include "vector.h"
 
 int git_refspec__parse(git_refspec *refspec, const char *input, bool is_fetch)
 {
@@ -287,3 +288,70 @@ git_direction git_refspec_direction(const git_refspec *spec)
 
 	return spec->push;
 }
+
+int git_refspec__dwim_one(git_vector *out, git_refspec *spec, git_vector *refs)
+{
+	git_buf buf = GIT_BUF_INIT;
+	size_t j, pos;
+	git_remote_head key;
+
+	const char* formatters[] = {
+		GIT_REFS_DIR "%s",
+		GIT_REFS_TAGS_DIR "%s",
+		GIT_REFS_HEADS_DIR "%s",
+		NULL
+	};
+
+	git_refspec *cur = git__calloc(1, sizeof(git_refspec));
+	GITERR_CHECK_ALLOC(cur);
+
+	cur->force = spec->force;
+	cur->push = spec->push;
+	cur->pattern = spec->pattern;
+	cur->matching = spec->matching;
+	cur->string = git__strdup(spec->string);
+
+	/* shorthand on the lhs */
+	if (git__prefixcmp(spec->src, GIT_REFS_DIR)) {
+		for (j = 0; formatters[j]; j++) {
+			git_buf_clear(&buf);
+			if (git_buf_printf(&buf, formatters[j], spec->src) < 0)
+				return -1;
+
+			key.name = (char *) git_buf_cstr(&buf);
+			if (!git_vector_search(&pos, refs, &key)) {
+				/* we found something to match the shorthand, set src to that */
+				cur->src = git_buf_detach(&buf);
+			}
+		}
+	}
+
+	/* No shorthands found, copy over the name */
+	if (cur->src == NULL && spec->src != NULL) {
+		cur->src = git__strdup(spec->src);
+		GITERR_CHECK_ALLOC(cur->src);
+	}
+
+	if (spec->dst && git__prefixcmp(spec->dst, GIT_REFS_DIR)) {
+		/* if it starts with "remotes" then we just prepend "refs/" */
+		if (!git__prefixcmp(spec->dst, "remotes/")) {
+			git_buf_puts(&buf, GIT_REFS_DIR);
+		} else {
+			git_buf_puts(&buf, GIT_REFS_HEADS_DIR);
+		}
+
+		if (git_buf_puts(&buf, spec->dst) < 0)
+			return -1;
+
+		cur->dst = git_buf_detach(&buf);
+	}
+
+	git_buf_free(&buf);
+
+	if (cur->dst == NULL && spec->dst != NULL) {
+		cur->dst = git__strdup(spec->dst);
+		GITERR_CHECK_ALLOC(cur->dst);
+	}
+
+	return git_vector_insert(out, cur);
+}
diff --git a/src/refspec.h b/src/refspec.h
index 44d484c7b..51b7bfee9 100644
--- a/src/refspec.h
+++ b/src/refspec.h
@@ -9,6 +9,7 @@
 
 #include "git2/refspec.h"
 #include "buffer.h"
+#include "vector.h"
 
 struct git_refspec {
 	char *string;
@@ -17,7 +18,6 @@ struct git_refspec {
 	unsigned int force :1,
 		push : 1,
 		pattern :1,
-		dwim :1,
 		matching :1;
 };
 
@@ -63,4 +63,10 @@ int git_refspec__serialize(git_buf *out, const git_refspec *refspec);
  */
 int git_refspec_is_wildcard(const git_refspec *spec);
 
+/**
+ * DWIM `spec` with `refs` existing on the remote, append the dwim'ed
+ * result in `out`.
+ */
+int git_refspec__dwim_one(git_vector *out, git_refspec *spec, git_vector *refs);
+
 #endif
diff --git a/src/remote.c b/src/remote.c
index 2b611aabc..49de5fa25 100644
--- a/src/remote.c
+++ b/src/remote.c
@@ -19,6 +19,8 @@
 #include "refspec.h"
 #include "fetchhead.h"
 
+static int dwim_refspecs(git_vector *out, git_vector *refspecs, git_vector *refs);
+
 static int add_refspec(git_remote *remote, const char *string, bool is_fetch)
 {
 	git_refspec *spec;
@@ -288,7 +290,8 @@ int git_remote_load(git_remote **out, git_repository *repo, const char *name)
 	GITERR_CHECK_ALLOC(remote->name);
 
 	if ((git_vector_init(&remote->refs, 32, NULL) < 0) ||
-	    (git_vector_init(&remote->refspecs, 2, NULL))) {
+	    (git_vector_init(&remote->refspecs, 2, NULL) < 0) ||
+	    (git_vector_init(&remote->active_refspecs, 2, NULL) < 0)) {
 		error = -1;
 		goto cleanup;
 	}
@@ -347,6 +350,10 @@ int git_remote_load(git_remote **out, git_repository *repo, const char *name)
 	if (download_tags_value(remote, config) < 0)
 		goto cleanup;
 
+	/* Move the data over to where the matching functions can find them */
+	if (dwim_refspecs(&remote->active_refspecs, &remote->refspecs, &remote->refs) < 0)
+		goto cleanup;
+
 	*out = remote;
 
 cleanup:
@@ -684,62 +691,33 @@ static int store_refs(git_remote_head *head, void *payload)
 	return git_vector_insert(refs, head);
 }
 
-static int dwim_refspecs(git_vector *refspecs, git_vector *refs)
+/* DWIM `refspecs` based on `refs` and append the output to `out` */
+static int dwim_refspecs(git_vector *out, git_vector *refspecs, git_vector *refs)
 {
-	git_buf buf = GIT_BUF_INIT;
+	size_t i;
 	git_refspec *spec;
-	size_t i, j, pos;
-	git_remote_head key;
-
-	const char* formatters[] = {
-		GIT_REFS_DIR "%s",
-		GIT_REFS_TAGS_DIR "%s",
-		GIT_REFS_HEADS_DIR "%s",
-		NULL
-	};
 
 	git_vector_foreach(refspecs, i, spec) {
-		if (spec->dwim)
-			continue;
-
-		/* shorthand on the lhs */
-		if (git__prefixcmp(spec->src, GIT_REFS_DIR)) {
-			for (j = 0; formatters[j]; j++) {
-				git_buf_clear(&buf);
-				if (git_buf_printf(&buf, formatters[j], spec->src) < 0)
-					return -1;
-
-				key.name = (char *) git_buf_cstr(&buf);
-				if (!git_vector_search(&pos, refs, &key)) {
-					/* we found something to match the shorthand, set src to that */
-					git__free(spec->src);
-					spec->src = git_buf_detach(&buf);
-				}
-			}
-		}
-
-		if (spec->dst && git__prefixcmp(spec->dst, GIT_REFS_DIR)) {
-			/* if it starts with "remotes" then we just prepend "refs/" */
-			if (!git__prefixcmp(spec->dst, "remotes/")) {
-				git_buf_puts(&buf, GIT_REFS_DIR);
-			} else {
-				git_buf_puts(&buf, GIT_REFS_HEADS_DIR);
-			}
-
-			if (git_buf_puts(&buf, spec->dst) < 0)
-				return -1;
-
-			git__free(spec->dst);
-			spec->dst = git_buf_detach(&buf);
-		}
-
-		spec->dwim = 1;
+		if (git_refspec__dwim_one(out, spec, refs) < 0)
+			return -1;
 	}
 
-	git_buf_free(&buf);
 	return 0;
 }
 
+static void free_refspecs(git_vector *vec)
+{
+	size_t i;
+	git_refspec *spec;
+
+	git_vector_foreach(vec, i, spec) {
+		git_refspec__free(spec);
+		git__free(spec);
+	}
+
+	git_vector_clear(vec);
+}
+
 static int remote_head_cmp(const void *_a, const void *_b)
 {
 	const git_remote_head *a = (git_remote_head *) _a;
@@ -755,14 +733,16 @@ int git_remote_download(git_remote *remote)
 
 	assert(remote);
 
-	if (git_vector_init(&refs, 16, remote_head_cmp) < 0)
+	if (git_vector_init(&refs, 8, remote_head_cmp) < 0)
 		return -1;
 
 	if (git_remote_ls(remote, store_refs, &refs) < 0) {
 		return -1;
 	}
 
-	error = dwim_refspecs(&remote->refspecs, &refs);
+	free_refspecs(&remote->active_refspecs);
+
+	error = dwim_refspecs(&remote->active_refspecs, &remote->refspecs, &refs);
 	git_vector_free(&refs);
 	if (error < 0)
 		return -1;
@@ -1024,7 +1004,7 @@ int git_remote_update_tips(git_remote *remote)
 		goto out;
 	}
 
-	git_vector_foreach(&remote->refspecs, i, spec) {
+	git_vector_foreach(&remote->active_refspecs, i, spec) {
 		if (spec->push)
 			continue;
 
@@ -1032,27 +1012,6 @@ int git_remote_update_tips(git_remote *remote)
 			goto out;
 	}
 
-	/* If we have no refspecs, update HEAD -> FETCH_HEAD manually */
-	if (remote->refspecs.length == 0 && refs.length > 0 && git_remote_update_fetchhead(remote)) {
-		git_vector vec;
-		git_refspec headspec;
-
-		if (git_refspec__parse(&headspec, "HEAD", true) < 0)
-			goto out;
-
-		if (git_vector_init(&vec, 1, NULL) < 0) {
-			goto out;
-		}
-
-		if (git_vector_insert(&vec, git_vector_get(&refs, 0)) < 0) {
-			git_vector_free(&vec);
-			goto out;
-		}
-
-		error = git_remote_write_fetchhead(remote, &headspec, &vec);
-		git_vector_free(&vec);
-	}
-
 out:
 	git_refspec__free(&tagspec);
 	git_vector_free(&refs);
@@ -1103,12 +1062,12 @@ void git_remote_free(git_remote *remote)
 
 	git_vector_free(&remote->refs);
 
-	git_vector_foreach(&remote->refspecs, i, spec) {
-		git_refspec__free(spec);
-		git__free(spec);
-	}
+	free_refspecs(&remote->refspecs);
 	git_vector_free(&remote->refspecs);
 
+	free_refspecs(&remote->active_refspecs);
+	git_vector_free(&remote->active_refspecs);
+
 	git__free(remote->url);
 	git__free(remote->pushurl);
 	git__free(remote->name);
@@ -1517,7 +1476,7 @@ git_refspec *git_remote__matching_refspec(git_remote *remote, const char *refnam
 	git_refspec *spec;
 	size_t i;
 
-	git_vector_foreach(&remote->refspecs, i, spec) {
+	git_vector_foreach(&remote->active_refspecs, i, spec) {
 		if (spec->push)
 			continue;
 
@@ -1533,7 +1492,7 @@ git_refspec *git_remote__matching_dst_refspec(git_remote *remote, const char *re
 	git_refspec *spec;
 	size_t i;
 
-	git_vector_foreach(&remote->refspecs, i, spec) {
+	git_vector_foreach(&remote->active_refspecs, i, spec) {
 		if (spec->push)
 			continue;
 
diff --git a/src/remote.h b/src/remote.h
index 269584d96..33e4d68f8 100644
--- a/src/remote.h
+++ b/src/remote.h
@@ -21,6 +21,7 @@ struct git_remote {
 	char *pushurl;
 	git_vector refs;
 	git_vector refspecs;
+	git_vector active_refspecs;
 	git_transport *transport;
 	git_repository *repo;
 	git_remote_callbacks callbacks;
diff --git a/src/transports/smart_protocol.c b/src/transports/smart_protocol.c
index 5d7b9bcba..ce3f115ee 100644
--- a/src/transports/smart_protocol.c
+++ b/src/transports/smart_protocol.c
@@ -269,7 +269,7 @@ static int wait_while_ack(gitno_buffer *buf)
 	return 0;
 }
 
-int git_smart__negotiate_fetch(git_transport *transport, git_repository *repo, const git_remote_head * const *refs, size_t count)
+int git_smart__negotiate_fetch(git_transport *transport, git_repository *repo, const git_remote_head * const *wants, size_t count)
 {
 	transport_smart *t = (transport_smart *)transport;
 	gitno_buffer *buf = &t->buffer;
@@ -279,7 +279,7 @@ int git_smart__negotiate_fetch(git_transport *transport, git_repository *repo, c
 	unsigned int i;
 	git_oid oid;
 
-	if ((error = git_pkt_buffer_wants(refs, count, &t->caps, &data)) < 0)
+	if ((error = git_pkt_buffer_wants(wants, count, &t->caps, &data)) < 0)
 		return error;
 
 	if ((error = fetch_setup_walk(&walk, repo)) < 0)
@@ -350,7 +350,7 @@ int git_smart__negotiate_fetch(git_transport *transport, git_repository *repo, c
 			git_pkt_ack *pkt;
 			unsigned int i;
 
-			if ((error = git_pkt_buffer_wants(refs, count, &t->caps, &data)) < 0)
+			if ((error = git_pkt_buffer_wants(wants, count, &t->caps, &data)) < 0)
 				goto on_error;
 
 			git_vector_foreach(&t->common, i, pkt) {
@@ -370,7 +370,7 @@ int git_smart__negotiate_fetch(git_transport *transport, git_repository *repo, c
 		git_pkt_ack *pkt;
 		unsigned int i;
 
-		if ((error = git_pkt_buffer_wants(refs, count, &t->caps, &data)) < 0)
+		if ((error = git_pkt_buffer_wants(wants, count, &t->caps, &data)) < 0)
 			goto on_error;
 
 		git_vector_foreach(&t->common, i, pkt) {