diff --git a/src/iterator.c b/src/iterator.c
index a312afb3a..cf51a340d 100644
--- a/src/iterator.c
+++ b/src/iterator.c
@@ -1027,8 +1027,11 @@ static int dirload_with_stat(
 	strncomp = (flags & GIT_PATH_DIR_IGNORE_CASE) != 0 ?
 		git__strncasecmp : git__strncmp;
 
-	if ((error = git_path_diriter_init(&diriter, dirpath, flags)) < 0)
+	/* Any error here is equivalent to the dir not existing, skip over it */
+	if ((error = git_path_diriter_init(&diriter, dirpath, flags)) < 0) {
+		error = GIT_ENOTFOUND;
 		goto done;
+	}
 
 	while ((error = git_path_diriter_next(&diriter)) == 0) {
 		if ((error = git_path_diriter_fullpath(&path, &path_len, &diriter)) < 0)
diff --git a/tests/repo/iterator.c b/tests/repo/iterator.c
index 26e8954fe..bb2d3a186 100644
--- a/tests/repo/iterator.c
+++ b/tests/repo/iterator.c
@@ -928,7 +928,7 @@ void test_repo_iterator__fs2(void)
 	git_iterator_free(i);
 }
 
-void test_repo_iterator__fs_preserves_error(void)
+void test_repo_iterator__unreadable_dir(void)
 {
 	git_iterator *i;
 	const git_index_entry *e;
@@ -951,10 +951,6 @@ void test_repo_iterator__fs_preserves_error(void)
 
 	cl_git_pass(git_iterator_advance(&e, i)); /* a */
 	cl_git_fail(git_iterator_advance(&e, i)); /* b */
-	cl_assert(giterr_last());
-	cl_assert(giterr_last()->message != NULL);
-	/* skip 'c/' empty directory */
-	cl_git_pass(git_iterator_advance(&e, i)); /* d */
 	cl_assert_equal_i(GIT_ITEROVER, git_iterator_advance(&e, i));
 
 	cl_must_pass(p_chmod("empty_standard_repo/r/b", 0777));