From f9774eea3a37e3a2a44d3c7e9dbc895280c3d90a Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Mon, 22 Apr 2013 17:22:31 +0200 Subject: [PATCH 1/4] atomic: Add an atomic type for 64-bit operations --- src/thread-utils.h | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/thread-utils.h b/src/thread-utils.h index dafe70ad6..28ecd297e 100644 --- a/src/thread-utils.h +++ b/src/thread-utils.h @@ -18,6 +18,14 @@ typedef struct { #endif } git_atomic; +typedef struct { +#if defined(GIT_WIN32) + __int64 val; +#else + int64_t val; +#endif +} git_atomic64; + GIT_INLINE(void) git_atomic_set(git_atomic *a, int val) { a->val = val; @@ -57,6 +65,17 @@ GIT_INLINE(int) git_atomic_inc(git_atomic *a) #endif } +GIT_INLINE(int) git_atomic_add(git_atomic *a, int32_t addend) +{ +#if defined(GIT_WIN32) + return _InterlockedExchangeAdd(&a->val, addend); +#elif defined(__GNUC__) + return __sync_add_and_fetch(&a->val, addend); +#else +# error "Unsupported architecture for atomic operations" +#endif +} + GIT_INLINE(int) git_atomic_dec(git_atomic *a) { #if defined(GIT_WIN32) @@ -82,6 +101,17 @@ GIT_INLINE(void *) git___compare_and_swap( return (foundval == oldval) ? oldval : newval; } +GIT_INLINE(int) git_atomic64_add(git_atomic64 *a, int64_t addend) +{ +#if defined(GIT_WIN32) + return _InterlockedExchangeAdd64(&a->val, addend); +#elif defined(__GNUC__) + return __sync_add_and_fetch(&a->val, addend); +#else +# error "Unsupported architecture for atomic operations" +#endif +} + #else #define git_thread unsigned int @@ -110,6 +140,12 @@ GIT_INLINE(int) git_atomic_inc(git_atomic *a) return ++a->val; } +GIT_INLINE(int) git_atomic_add(git_atomic *a, int32_t addend) +{ + a->val += addend; + return a->val; +} + GIT_INLINE(int) git_atomic_dec(git_atomic *a) { return --a->val; @@ -125,6 +161,12 @@ GIT_INLINE(void *) git___compare_and_swap( return oldval; } +GIT_INLINE(int) git_atomic64_add(git_atomic64 *a, int64_t addend) +{ + a->val += addend; + return a->val; +} + #endif /* Atomically replace oldval with newval From a14163a79d644f0fd2856b083f355f2df19f6bdd Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Mon, 22 Apr 2013 17:30:49 +0200 Subject: [PATCH 2/4] cache: Shared meter for memory usage --- src/cache.c | 16 ++++++++++++---- src/cache.h | 4 ++-- src/util.c | 3 +-- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/cache.c b/src/cache.c index ca122fb77..4b26d4351 100644 --- a/src/cache.c +++ b/src/cache.c @@ -18,7 +18,9 @@ GIT__USE_OIDMAP bool git_cache__enabled = true; -size_t git_cache__max_storage = (4 * 1024 * 1024); +int64_t git_cache__max_storage = (4 * 1024 * 1024); + +static git_atomic64 git_cache__current_storage = {0}; static size_t git_cache__max_object_size[8] = { 0, /* GIT_OBJ__EXT1 */ @@ -81,6 +83,7 @@ static void clear_cache(git_cache *cache) }); kh_clear(oid, cache->map); + git_atomic64_add(&git_cache__current_storage, -cache->used_memory); cache->used_memory = 0; } @@ -106,7 +109,7 @@ void git_cache_free(git_cache *cache) static void cache_evict_entries(git_cache *cache) { uint32_t seed = rand(); - size_t evict_count = 8; + int64_t evicted_memory = 0, evict_count = 8; /* do not infinite loop if there's not enough entries to evict */ if (evict_count > kh_size(cache->map)) { @@ -121,12 +124,15 @@ static void cache_evict_entries(git_cache *cache) git_cached_obj *evict = kh_val(cache->map, pos); evict_count--; - cache->used_memory -= evict->size; + evicted_memory += evict->size; git_cached_obj_decref(evict); kh_del(oid, cache->map, pos); } } + + cache->used_memory -= evicted_memory; + git_atomic64_add(&git_cache__current_storage, -evicted_memory); } static bool cache_should_store(git_otype object_type, size_t object_size) @@ -171,7 +177,8 @@ static void *cache_store(git_cache *cache, git_cached_obj *entry) if (git_mutex_lock(&cache->lock) < 0) return entry; - if (cache->used_memory > git_cache__max_storage) + /* soften the load on the cache */ + if (git_cache__current_storage.val > git_cache__max_storage) cache_evict_entries(cache); pos = kh_get(oid, cache->map, &entry->oid); @@ -186,6 +193,7 @@ static void *cache_store(git_cache *cache, git_cached_obj *entry) kh_val(cache->map, pos) = entry; git_cached_obj_incref(entry); cache->used_memory += entry->size; + git_atomic64_add(&git_cache__current_storage, (int64_t)entry->size); } } /* found */ diff --git a/src/cache.h b/src/cache.h index 1715c7220..f74fddfc9 100644 --- a/src/cache.h +++ b/src/cache.h @@ -31,11 +31,11 @@ typedef struct { typedef struct { git_oidmap *map; git_mutex lock; - size_t used_memory; + int64_t used_memory; } git_cache; extern bool git_cache__enabled; -extern size_t git_cache__max_storage; +extern int64_t git_cache__max_storage; int git_cache_set_max_object_size(git_otype type, size_t size); diff --git a/src/util.c b/src/util.c index c3fc69756..2dec49f17 100644 --- a/src/util.c +++ b/src/util.c @@ -39,7 +39,6 @@ int git_libgit2_capabilities() /* Declarations for tuneable settings */ extern size_t git_mwindow__window_size; extern size_t git_mwindow__mapped_limit; -extern size_t git_odb__cache_size; static int config_level_to_futils_dir(int config_level) { @@ -104,7 +103,7 @@ int git_libgit2_opts(int key, ...) } case GIT_OPT_SET_CACHE_MAX_SIZE: - git_cache__max_storage = va_arg(ap, size_t); + git_cache__max_storage = va_arg(ap, int64_t); break; case GIT_OPT_ENABLE_CACHING: From 920cbc9846c526958f6b4a7f914bdde2da1d34ec Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Mon, 22 Apr 2013 17:31:47 +0200 Subject: [PATCH 3/4] cache: More aggressive default --- src/cache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cache.c b/src/cache.c index 4b26d4351..b462af408 100644 --- a/src/cache.c +++ b/src/cache.c @@ -18,7 +18,7 @@ GIT__USE_OIDMAP bool git_cache__enabled = true; -int64_t git_cache__max_storage = (4 * 1024 * 1024); +int64_t git_cache__max_storage = (256 * 1024 * 1024); static git_atomic64 git_cache__current_storage = {0}; From a2378ae4fee55c95eb9a1f6b44f5a837d39fa724 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Tue, 23 Apr 2013 20:42:29 +0200 Subject: [PATCH 4/4] opts: Add getter for cached memory --- include/git2/common.h | 3 ++- src/cache.c | 3 +-- src/cache.h | 1 + src/util.c | 5 +++++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/include/git2/common.h b/include/git2/common.h index ccd252fda..6101e13bc 100644 --- a/include/git2/common.h +++ b/include/git2/common.h @@ -133,7 +133,8 @@ enum { GIT_OPT_SET_SEARCH_PATH, GIT_OPT_SET_CACHE_OBJECT_LIMIT, GIT_OPT_SET_CACHE_MAX_SIZE, - GIT_OPT_ENABLE_CACHING + GIT_OPT_ENABLE_CACHING, + GIT_OPT_GET_CACHED_MEMORY }; /** diff --git a/src/cache.c b/src/cache.c index b462af408..88f643b35 100644 --- a/src/cache.c +++ b/src/cache.c @@ -19,8 +19,7 @@ GIT__USE_OIDMAP bool git_cache__enabled = true; int64_t git_cache__max_storage = (256 * 1024 * 1024); - -static git_atomic64 git_cache__current_storage = {0}; +git_atomic64 git_cache__current_storage = {0}; static size_t git_cache__max_object_size[8] = { 0, /* GIT_OBJ__EXT1 */ diff --git a/src/cache.h b/src/cache.h index f74fddfc9..1fb87dcea 100644 --- a/src/cache.h +++ b/src/cache.h @@ -36,6 +36,7 @@ typedef struct { extern bool git_cache__enabled; extern int64_t git_cache__max_storage; +extern git_atomic64 git_cache__current_storage; int git_cache_set_max_object_size(git_otype type, size_t size); diff --git a/src/util.c b/src/util.c index 2dec49f17..ce67c7e62 100644 --- a/src/util.c +++ b/src/util.c @@ -109,6 +109,11 @@ int git_libgit2_opts(int key, ...) case GIT_OPT_ENABLE_CACHING: git_cache__enabled = (va_arg(ap, int) != 0); break; + + case GIT_OPT_GET_CACHED_MEMORY: + *(va_arg(ap, int64_t *)) = git_cache__current_storage.val; + *(va_arg(ap, int64_t *)) = git_cache__max_storage; + break; } va_end(ap);