diff --git a/src/fetch.c b/src/fetch.c
index 1944bd005..08c789ddb 100644
--- a/src/fetch.c
+++ b/src/fetch.c
@@ -36,6 +36,10 @@ static int filter_ref__cb(git_remote_head *head, void *payload)
 		/* If it doesn't match the refpec, we don't want it */
 		if (!git_refspec_src_matches(p->spec, head->name))
 			return 0;
+
+		/* Don't even try to ask for the annotation target */
+		if (!git__suffixcmp(head->name, "^{}"))
+			return 0;
 	}
 
 	/* If we have the object, mark it so we don't ask for it */
diff --git a/src/remote.c b/src/remote.c
index 1857d328e..a5cfc822e 100644
--- a/src/remote.c
+++ b/src/remote.c
@@ -353,10 +353,6 @@ int git_remote_update_tips(git_remote *remote, int (*cb)(const char *refname, co
 	for (; i < refs->length; ++i) {
 		head = refs->contents[i];
 
-		/* Skip tag annotations */
-		if (!git__suffixcmp(head->name, "^{}"))
-			continue;
-
 		if (git_refspec_transform_r(&refname, spec, head->name) < 0)
 			goto on_error;