Wrap malloc and friends and report out of memory as GIT_ENOMEM

We now forbid direct use of malloc, strdup or calloc within the
library and instead use wrapper functions git__malloc, etc. to
invoke the underlying library malloc and set git_errno to a no
memory error code if the allocation fails.

In the future once we have pack objects in memory we are likely
to enhance these routines with garbage collection logic to purge
cached pack data when allocations fail.  Because the size of the
function will grow somewhat large, we don't want to mark them for
inline as gcc tends to aggressively inline, creating larger than
expected executables.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>

src/commit.c

@@ -23,6 +23,7 @@
  * Boston, MA 02110-1301, USA.
  */
 
+#include "common.h"
 #include "commit.h"
 
 const git_oid *git_commit_id(git_commit *c)

src/common.h

@@ -2,7 +2,6 @@
 #define INCLUDE_common_h__
 
 #include "cc-compat.h"
-#include "util.h"
 #include "errors.h"
 
 #ifdef GIT_HAS_PTHREAD
@@ -19,6 +18,7 @@
 # define PRIuPTR "lu"
 #endif
 
+#include "util.h"
 #include "git/common.h"
 
 #define GIT_PATH_MAX 4096

src/errors.c

@@ -19,7 +19,9 @@ int *git__errno_storage(void)
 {
 	int *e = pthread_getspecific(errno_key);
 	if (!e) {
+#undef calloc
 		e = calloc(1, sizeof(*e));
+#define calloc(a,b) GIT__FORBID_MALLOC
 		pthread_setspecific(errno_key, e);
 	}
 	return e;
@@ -33,6 +35,7 @@ static struct {
 } error_codes[] = {
 	{ GIT_ENOTOID, "Not a git oid" },
 	{ GIT_ENOTFOUND, "Object does not exist in the scope searched" },
+	{ GIT_ENOMEM, "Not enough space" },
 };
 
 const char *git_strerror(int num)

src/fileops.c

@@ -58,17 +58,18 @@ int gitfo_read_file(gitfo_buf *obj, const char *path)
 	assert(obj && path && *path);
 
 	if ((fd = gitfo_open(path, O_RDONLY)) < 0)
-		return GIT_ERROR;  /* TODO: error handling */
+		return GIT_ERROR;
 
-	if (((len = gitfo_size(fd)) < 0) || ((buff = malloc(len+1)) == NULL)) {
+	if (((len = gitfo_size(fd)) < 0) 
+		|| ((buff = git__malloc(len + 1)) == NULL)) {
 		gitfo_close(fd);
-		return GIT_ERROR;  /* TODO: error handling */
+		return GIT_ERROR;
 	}
 
 	if (gitfo_read(fd, buff, len) < 0) {
 		gitfo_close(fd);
 		free(buff);
-		return GIT_ERROR;  /* TODO: error handling */
+		return GIT_ERROR;
 	}
 	buff[len] = '\0';
 
@@ -98,13 +99,13 @@ gitfo_cache *gitfo_enable_caching(git_file fd, size_t cache_size)
 {
 	gitfo_cache *ioc;
 
-	ioc = malloc(sizeof(*ioc));
+	ioc = git__malloc(sizeof(*ioc));
 	if (!ioc)
 		return NULL;
 
 	ioc->pos = 0;
 	ioc->cache_size = cache_size;
-	ioc->cache = malloc(cache_size);
+	ioc->cache = git__malloc(cache_size);
 	if (!ioc->cache) {
 		free(ioc);
 		return NULL;

src/git/common.h

@@ -55,6 +55,9 @@
 /** Input does not exist in the scope searched. */
 #define GIT_ENOTFOUND (GIT_ERROR - 2)
 
+/** Not enough space available. */
+#define GIT_ENOMEM (GIT_ERROR - 3)
+
 GIT_BEGIN_DECL
 
 /** A revision traversal pool. */

src/hash.c

@@ -33,7 +33,7 @@ struct git_hash_ctx {
 
 git_hash_ctx *git_hash_new_ctx(void)
 {
-	git_hash_ctx *ctx = malloc(sizeof(*ctx));
+	git_hash_ctx *ctx = git__malloc(sizeof(*ctx));
 
 	if (!ctx)
 		return NULL;

src/odb.c

@@ -279,7 +279,7 @@ static void *inflate_tail(z_stream *s, void *hb, size_t used, obj_hdr *hdr)
 	 * initial sequence of inflated data from the tail of the
 	 * head buffer, if any.
 	 */
-	if ((buf = malloc(hdr->size + 1)) == NULL)
+	if ((buf = git__malloc(hdr->size + 1)) == NULL)
 		return NULL;
 	tail = s->total_out - used;
 	if (used > 0 && tail > 0) {
@@ -352,7 +352,10 @@ static int inflate_packlike_loose_disk_obj(git_obj *out, gitfo_buf *obj)
 	/*
 	 * allocate a buffer and inflate the data into it
 	 */
-	buf = malloc(hdr.size+1);
+	buf = git__malloc(hdr.size + 1);
+	if (!buf)
+		return GIT_ERROR;
+
 	in  = ((unsigned char *)obj->data) + used;
 	len = obj->len - used;
 	if (inflate_buffer(in, len, buf, hdr.size)) {
@@ -414,7 +417,7 @@ static int open_alternates(git_odb *db)
 {
 	unsigned n = 0;
 
-	db->alternates = malloc(sizeof(*db->alternates) * (n + 1));
+	db->alternates = git__malloc(sizeof(*db->alternates) * (n + 1));
 	if (!db->alternates)
 		return GIT_ERROR;
 
@@ -424,11 +427,11 @@ static int open_alternates(git_odb *db)
 
 int git_odb_open(git_odb **out, const char *objects_dir)
 {
-	git_odb *db = malloc(sizeof(*db));
+	git_odb *db = git__malloc(sizeof(*db));
 	if (!db)
 		return GIT_ERROR;
 
-	db->objects_dir = strdup(objects_dir);
+	db->objects_dir = git__strdup(objects_dir);
 	if (!db->objects_dir) {
 		free(db);
 		return GIT_ERROR;

src/oid.c

@@ -87,7 +87,7 @@ void git_oid_pathfmt(char *str, const git_oid *oid)
 
 char *git_oid_allocfmt(const git_oid *oid)
 {
-	char *str = malloc(GIT_OID_HEXSZ + 1);
+	char *str = git__malloc(GIT_OID_HEXSZ + 1);
 	if (!str)
 		return NULL;
 	git_oid_fmt(str, oid);

src/revwalk.c

@@ -23,11 +23,12 @@
  * Boston, MA 02110-1301, USA.
  */
 
+#include "common.h"
 #include "revwalk.h"
 
 git_revpool *gitrp_alloc(git_odb *db)
 {
-	git_revpool *walk = malloc(sizeof(*walk));
+	git_revpool *walk = git__malloc(sizeof(*walk));
 	if (!walk)
 		return NULL;
 

src/thread-utils.c

@@ -1,3 +1,4 @@
+#include "common.h"
 #include "thread-utils.h"
 
 #ifdef _WIN32

src/util.c

@@ -0,0 +1,26 @@
+#define GIT__NO_HIDE_MALLOC
+#include "common.h"
+
+void *git__malloc(size_t n)
+{
+	void *r = malloc(n);
+	if (!r)
+		return git_ptr_error(GIT_ENOMEM);
+	return r;
+}
+
+void *git__calloc(size_t a, size_t b)
+{
+	void *r = calloc(a, b);
+	if (!r)
+		return git_ptr_error(GIT_ENOMEM);
+	return r;
+}
+
+char *git__strdup(const char *s)
+{
+	char *r = strdup(s);
+	if (!s)
+		return git_ptr_error(GIT_ENOMEM);
+	return r;
+}

src/util.h

@@ -3,6 +3,17 @@
 
 #define ARRAY_SIZE(x) (sizeof(x)/sizeof(x[0]))
 
+extern void *git__malloc(size_t);
+extern void *git__calloc(size_t, size_t);
+extern char *git__strdup(const char *);
+
+#ifndef GIT__NO_HIDE_MALLOC
+# define GIT__FORBID_MALLOC do_not_use_malloc_directly
+# define malloc(a)          GIT__FORBID_MALLOC
+# define calloc(a,b)        GIT__FORBID_MALLOC
+# define strdup(a)          GIT__FORBID_MALLOC
+#endif
+
 /*
  * Realloc the buffer pointed at by variable 'x' so that it can hold
  * at least 'nr' entries; the number of entries currently allocated