mirror of
https://git.proxmox.com/git/libgit2
synced 2025-05-03 01:58:32 +00:00
packbuilder: size_t all the things
After 1cd65991, we were passing a pointer to an `unsigned long` to
a function that now expected a pointer to a `size_t`. These types
differ on 64-bit Windows, which means that we trash the stack.
Use `size_t`s in the packbuilder to avoid this.
This commit is contained in:
Edward Thomson
parent
581a4d3942
commit
60e15ecd55
@ -51,6 +51,13 @@ v0.24 + 1
|
||||
|
||||
### Breaking API changes
|
||||
|
||||
* `git_packbuilder_object_count` and `git_packbuilder_written` now
|
||||
return a `size_t` instead of a `uint32_t` for more thorough
|
||||
compatibility with the rest of the library.
|
||||
|
||||
* `git_packbuiler_progress` now provides explicitly sized `uint32_t`
|
||||
values instead of `unsigned int`.
|
||||
|
||||
v0.24
|
||||
-------
|
||||
|
||||
|
||||
@ -196,7 +196,7 @@ GIT_EXTERN(int) git_packbuilder_foreach(git_packbuilder *pb, git_packbuilder_for
|
||||
* @param pb the packbuilder
|
||||
* @return the number of objects in the packfile
|
||||
*/
|
||||
GIT_EXTERN(uint32_t) git_packbuilder_object_count(git_packbuilder *pb);
|
||||
GIT_EXTERN(size_t) git_packbuilder_object_count(git_packbuilder *pb);
|
||||
|
||||
/**
|
||||
* Get the number of objects the packbuilder has already written out
|
||||
@ -204,13 +204,13 @@ GIT_EXTERN(uint32_t) git_packbuilder_object_count(git_packbuilder *pb);
|
||||
* @param pb the packbuilder
|
||||
* @return the number of objects which have already been written
|
||||
*/
|
||||
GIT_EXTERN(uint32_t) git_packbuilder_written(git_packbuilder *pb);
|
||||
GIT_EXTERN(size_t) git_packbuilder_written(git_packbuilder *pb);
|
||||
|
||||
/** Packbuilder progress notification function */
|
||||
typedef int (*git_packbuilder_progress)(
|
||||
int stage,
|
||||
unsigned int current,
|
||||
unsigned int total,
|
||||
uint32_t current,
|
||||
uint32_t total,
|
||||
void *payload);
|
||||
|
||||
/**
|
||||
|
||||
@ -28,7 +28,7 @@ struct unpacked {
|
||||
git_pobject *object;
|
||||
void *data;
|
||||
struct git_delta_index *index;
|
||||
int depth;
|
||||
size_t depth;
|
||||
};
|
||||
|
||||
struct tree_walk_context {
|
||||
@ -99,8 +99,15 @@ static int packbuilder_config(git_packbuilder *pb)
|
||||
|
||||
#define config_get(KEY,DST,DFLT) do { \
|
||||
ret = git_config_get_int64(&val, config, KEY); \
|
||||
if (!ret) (DST) = val; \
|
||||
else if (ret == GIT_ENOTFOUND) { \
|
||||
if (!ret) { \
|
||||
if (!git__is_sizet(val)) { \
|
||||
giterr_set(GITERR_CONFIG, \
|
||||
"configuration value '%s' is too large", KEY); \
|
||||
ret = -1; \
|
||||
goto out; \
|
||||
} \
|
||||
(DST) = (size_t)val; \
|
||||
} else if (ret == GIT_ENOTFOUND) { \
|
||||
(DST) = (DFLT); \
|
||||
ret = 0; \
|
||||
} else if (ret < 0) goto out; } while (0)
|
||||
@ -187,7 +194,7 @@ static void rehash(git_packbuilder *pb)
|
||||
{
|
||||
git_pobject *po;
|
||||
khiter_t pos;
|
||||
unsigned int i;
|
||||
size_t i;
|
||||
int ret;
|
||||
|
||||
kh_clear(oid, pb->object_ix);
|
||||
@ -222,7 +229,7 @@ int git_packbuilder_insert(git_packbuilder *pb, const git_oid *oid,
|
||||
return -1;
|
||||
}
|
||||
|
||||
pb->nr_alloc = (uint32_t)newsize;
|
||||
pb->nr_alloc = newsize;
|
||||
|
||||
pb->object_list = git__reallocarray(pb->object_list,
|
||||
pb->nr_alloc, sizeof(*po));
|
||||
@ -272,7 +279,7 @@ int git_packbuilder_insert(git_packbuilder *pb, const git_oid *oid,
|
||||
static int get_delta(void **out, git_odb *odb, git_pobject *po)
|
||||
{
|
||||
git_odb_object *src = NULL, *trg = NULL;
|
||||
unsigned long delta_size;
|
||||
size_t delta_size;
|
||||
void *delta_buf;
|
||||
int error;
|
||||
|
||||
@ -441,8 +448,8 @@ static int write_one(
|
||||
return write_object(pb, po, write_cb, cb_data);
|
||||
}
|
||||
|
||||
GIT_INLINE(void) add_to_write_order(git_pobject **wo, unsigned int *endp,
|
||||
git_pobject *po)
|
||||
GIT_INLINE(void) add_to_write_order(git_pobject **wo, size_t *endp,
|
||||
git_pobject *po)
|
||||
{
|
||||
if (po->filled)
|
||||
return;
|
||||
@ -450,8 +457,8 @@ GIT_INLINE(void) add_to_write_order(git_pobject **wo, unsigned int *endp,
|
||||
po->filled = 1;
|
||||
}
|
||||
|
||||
static void add_descendants_to_write_order(git_pobject **wo, unsigned int *endp,
|
||||
git_pobject *po)
|
||||
static void add_descendants_to_write_order(git_pobject **wo, size_t *endp,
|
||||
git_pobject *po)
|
||||
{
|
||||
int add_to_order = 1;
|
||||
while (po) {
|
||||
@ -492,8 +499,8 @@ static void add_descendants_to_write_order(git_pobject **wo, unsigned int *endp,
|
||||
};
|
||||
}
|
||||
|
||||
static void add_family_to_write_order(git_pobject **wo, unsigned int *endp,
|
||||
git_pobject *po)
|
||||
static void add_family_to_write_order(git_pobject **wo, size_t *endp,
|
||||
git_pobject *po)
|
||||
{
|
||||
git_pobject *root;
|
||||
|
||||
@ -524,7 +531,7 @@ static int cb_tag_foreach(const char *name, git_oid *oid, void *data)
|
||||
|
||||
static git_pobject **compute_write_order(git_packbuilder *pb)
|
||||
{
|
||||
unsigned int i, wo_end, last_untagged;
|
||||
size_t i, wo_end, last_untagged;
|
||||
git_pobject **wo;
|
||||
|
||||
if ((wo = git__mallocarray(pb->nr_objects, sizeof(*wo))) == NULL)
|
||||
@ -629,13 +636,18 @@ static int write_pack(git_packbuilder *pb,
|
||||
enum write_one_status status;
|
||||
struct git_pack_header ph;
|
||||
git_oid entry_oid;
|
||||
unsigned int i = 0;
|
||||
size_t i = 0;
|
||||
int error = 0;
|
||||
|
||||
write_order = compute_write_order(pb);
|
||||
if (write_order == NULL)
|
||||
return -1;
|
||||
|
||||
if (!git__is_uint32(pb->nr_objects)) {
|
||||
giterr_set(GITERR_INVALID, "too many objects");
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* Write pack header */
|
||||
ph.hdr_signature = htonl(PACK_SIGNATURE);
|
||||
ph.hdr_version = htonl(PACK_VERSION);
|
||||
@ -711,11 +723,18 @@ static int type_size_sort(const void *_a, const void *_b)
|
||||
return a < b ? -1 : (a > b); /* newest first */
|
||||
}
|
||||
|
||||
static int delta_cacheable(git_packbuilder *pb, unsigned long src_size,
|
||||
unsigned long trg_size, unsigned long delta_size)
|
||||
static int delta_cacheable(
|
||||
git_packbuilder *pb,
|
||||
size_t src_size,
|
||||
size_t trg_size,
|
||||
size_t delta_size)
|
||||
{
|
||||
if (pb->max_delta_cache_size &&
|
||||
pb->delta_cache_size + delta_size > pb->max_delta_cache_size)
|
||||
size_t new_size;
|
||||
|
||||
if (git__add_sizet_overflow(&new_size, pb->delta_cache_size, delta_size))
|
||||
return 0;
|
||||
|
||||
if (pb->max_delta_cache_size && new_size > pb->max_delta_cache_size)
|
||||
return 0;
|
||||
|
||||
if (delta_size < pb->cache_max_small_delta_size)
|
||||
@ -729,15 +748,14 @@ static int delta_cacheable(git_packbuilder *pb, unsigned long src_size,
|
||||
}
|
||||
|
||||
static int try_delta(git_packbuilder *pb, struct unpacked *trg,
|
||||
struct unpacked *src, int max_depth,
|
||||
unsigned long *mem_usage, int *ret)
|
||||
struct unpacked *src, size_t max_depth,
|
||||
size_t *mem_usage, int *ret)
|
||||
{
|
||||
git_pobject *trg_object = trg->object;
|
||||
git_pobject *src_object = src->object;
|
||||
git_odb_object *obj;
|
||||
unsigned long trg_size, src_size, delta_size,
|
||||
sizediff, max_size, sz;
|
||||
unsigned int ref_depth;
|
||||
size_t trg_size, src_size, delta_size, sizediff, max_size, sz;
|
||||
size_t ref_depth;
|
||||
void *delta_buf;
|
||||
|
||||
/* Don't bother doing diffs between different types */
|
||||
@ -755,7 +773,7 @@ static int try_delta(git_packbuilder *pb, struct unpacked *trg,
|
||||
return 0;
|
||||
|
||||
/* Now some size filtering heuristics. */
|
||||
trg_size = (unsigned long)trg_object->size;
|
||||
trg_size = trg_object->size;
|
||||
if (!trg_object->delta) {
|
||||
max_size = trg_size/2 - 20;
|
||||
ref_depth = 1;
|
||||
@ -769,7 +787,7 @@ static int try_delta(git_packbuilder *pb, struct unpacked *trg,
|
||||
if (max_size == 0)
|
||||
return 0;
|
||||
|
||||
src_size = (unsigned long)src_object->size;
|
||||
src_size = src_object->size;
|
||||
sizediff = src_size < trg_size ? trg_size - src_size : 0;
|
||||
if (sizediff >= max_size)
|
||||
return 0;
|
||||
@ -781,7 +799,7 @@ static int try_delta(git_packbuilder *pb, struct unpacked *trg,
|
||||
if (git_odb_read(&obj, pb->odb, &trg_object->id) < 0)
|
||||
return -1;
|
||||
|
||||
sz = (unsigned long)git_odb_object_size(obj);
|
||||
sz = git_odb_object_size(obj);
|
||||
trg->data = git__malloc(sz);
|
||||
GITERR_CHECK_ALLOC(trg->data);
|
||||
memcpy(trg->data, git_odb_object_data(obj), sz);
|
||||
@ -803,7 +821,7 @@ static int try_delta(git_packbuilder *pb, struct unpacked *trg,
|
||||
!git__is_ulong(obj_sz = git_odb_object_size(obj)))
|
||||
return -1;
|
||||
|
||||
sz = (unsigned long)obj_sz;
|
||||
sz = obj_sz;
|
||||
src->data = git__malloc(sz);
|
||||
GITERR_CHECK_ALLOC(src->data);
|
||||
memcpy(src->data, git_odb_object_data(obj), sz);
|
||||
@ -841,11 +859,12 @@ static int try_delta(git_packbuilder *pb, struct unpacked *trg,
|
||||
git_packbuilder__cache_lock(pb);
|
||||
if (trg_object->delta_data) {
|
||||
git__free(trg_object->delta_data);
|
||||
assert(pb->delta_cache_size >= trg_object->delta_size);
|
||||
pb->delta_cache_size -= trg_object->delta_size;
|
||||
trg_object->delta_data = NULL;
|
||||
}
|
||||
if (delta_cacheable(pb, src_size, trg_size, delta_size)) {
|
||||
bool overflow = git__add_uint64_overflow(
|
||||
bool overflow = git__add_sizet_overflow(
|
||||
&pb->delta_cache_size, pb->delta_cache_size, delta_size);
|
||||
|
||||
git_packbuilder__cache_unlock(pb);
|
||||
@ -871,13 +890,13 @@ static int try_delta(git_packbuilder *pb, struct unpacked *trg,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static unsigned int check_delta_limit(git_pobject *me, unsigned int n)
|
||||
static size_t check_delta_limit(git_pobject *me, size_t n)
|
||||
{
|
||||
git_pobject *child = me->delta_child;
|
||||
unsigned int m = n;
|
||||
size_t m = n;
|
||||
|
||||
while (child) {
|
||||
unsigned int c = check_delta_limit(child, n + 1);
|
||||
size_t c = check_delta_limit(child, n + 1);
|
||||
if (m < c)
|
||||
m = c;
|
||||
child = child->delta_sibling;
|
||||
@ -885,9 +904,9 @@ static unsigned int check_delta_limit(git_pobject *me, unsigned int n)
|
||||
return m;
|
||||
}
|
||||
|
||||
static unsigned long free_unpacked(struct unpacked *n)
|
||||
static size_t free_unpacked(struct unpacked *n)
|
||||
{
|
||||
unsigned long freed_mem = 0;
|
||||
size_t freed_mem = 0;
|
||||
|
||||
if (n->index) {
|
||||
freed_mem += git_delta_index_size(n->index);
|
||||
@ -896,7 +915,7 @@ static unsigned long free_unpacked(struct unpacked *n)
|
||||
n->index = NULL;
|
||||
|
||||
if (n->data) {
|
||||
freed_mem += (unsigned long)n->object->size;
|
||||
freed_mem += n->object->size;
|
||||
git__free(n->data);
|
||||
n->data = NULL;
|
||||
}
|
||||
@ -905,7 +924,8 @@ static unsigned long free_unpacked(struct unpacked *n)
|
||||
return freed_mem;
|
||||
}
|
||||
|
||||
static int report_delta_progress(git_packbuilder *pb, uint32_t count, bool force)
|
||||
static int report_delta_progress(
|
||||
git_packbuilder *pb, uint32_t count, bool force)
|
||||
{
|
||||
int ret;
|
||||
|
||||
@ -929,15 +949,14 @@ static int report_delta_progress(git_packbuilder *pb, uint32_t count, bool force
|
||||
}
|
||||
|
||||
static int find_deltas(git_packbuilder *pb, git_pobject **list,
|
||||
unsigned int *list_size, unsigned int window,
|
||||
int depth)
|
||||
size_t *list_size, size_t window, size_t depth)
|
||||
{
|
||||
git_pobject *po;
|
||||
git_buf zbuf = GIT_BUF_INIT;
|
||||
struct unpacked *array;
|
||||
uint32_t idx = 0, count = 0;
|
||||
unsigned long mem_usage = 0;
|
||||
unsigned int i;
|
||||
size_t idx = 0, count = 0;
|
||||
size_t mem_usage = 0;
|
||||
size_t i;
|
||||
int error = -1;
|
||||
|
||||
array = git__calloc(window, sizeof(struct unpacked));
|
||||
@ -945,7 +964,7 @@ static int find_deltas(git_packbuilder *pb, git_pobject **list,
|
||||
|
||||
for (;;) {
|
||||
struct unpacked *n = array + idx;
|
||||
int max_depth, j, best_base = -1;
|
||||
size_t max_depth, j, best_base = SIZE_MAX;
|
||||
|
||||
git_packbuilder__progress_lock(pb);
|
||||
if (!*list_size) {
|
||||
@ -966,7 +985,7 @@ static int find_deltas(git_packbuilder *pb, git_pobject **list,
|
||||
while (pb->window_memory_limit &&
|
||||
mem_usage > pb->window_memory_limit &&
|
||||
count > 1) {
|
||||
uint32_t tail = (idx + window - count) % window;
|
||||
size_t tail = (idx + window - count) % window;
|
||||
mem_usage -= free_unpacked(array + tail);
|
||||
count--;
|
||||
}
|
||||
@ -978,15 +997,18 @@ static int find_deltas(git_packbuilder *pb, git_pobject **list,
|
||||
*/
|
||||
max_depth = depth;
|
||||
if (po->delta_child) {
|
||||
max_depth -= check_delta_limit(po, 0);
|
||||
if (max_depth <= 0)
|
||||
size_t delta_limit = check_delta_limit(po, 0);
|
||||
|
||||
if (delta_limit > max_depth)
|
||||
goto next;
|
||||
|
||||
max_depth -= delta_limit;
|
||||
}
|
||||
|
||||
j = window;
|
||||
while (--j > 0) {
|
||||
int ret;
|
||||
uint32_t other_idx = idx + j;
|
||||
size_t other_idx = idx + j;
|
||||
struct unpacked *m;
|
||||
|
||||
if (other_idx >= window)
|
||||
@ -1027,7 +1049,7 @@ static int find_deltas(git_packbuilder *pb, git_pobject **list,
|
||||
GITERR_CHECK_ALLOC(po->delta_data);
|
||||
|
||||
memcpy(po->delta_data, zbuf.ptr, zbuf.size);
|
||||
po->z_delta_size = (unsigned long)zbuf.size;
|
||||
po->z_delta_size = zbuf.size;
|
||||
git_buf_clear(&zbuf);
|
||||
|
||||
git_packbuilder__cache_lock(pb);
|
||||
@ -1051,10 +1073,10 @@ static int find_deltas(git_packbuilder *pb, git_pobject **list,
|
||||
*/
|
||||
if (po->delta) {
|
||||
struct unpacked swap = array[best_base];
|
||||
int dist = (window + idx - best_base) % window;
|
||||
int dst = best_base;
|
||||
size_t dist = (window + idx - best_base) % window;
|
||||
size_t dst = best_base;
|
||||
while (dist--) {
|
||||
int src = (dst + 1) % window;
|
||||
size_t src = (dst + 1) % window;
|
||||
array[dst] = array[src];
|
||||
dst = src;
|
||||
}
|
||||
@ -1092,13 +1114,13 @@ struct thread_params {
|
||||
git_cond cond;
|
||||
git_mutex mutex;
|
||||
|
||||
unsigned int list_size;
|
||||
unsigned int remaining;
|
||||
size_t list_size;
|
||||
size_t remaining;
|
||||
|
||||
int window;
|
||||
int depth;
|
||||
int working;
|
||||
int data_ready;
|
||||
size_t window;
|
||||
size_t depth;
|
||||
size_t working;
|
||||
size_t data_ready;
|
||||
};
|
||||
|
||||
static void *threaded_find_deltas(void *arg)
|
||||
@ -1140,11 +1162,11 @@ static void *threaded_find_deltas(void *arg)
|
||||
}
|
||||
|
||||
static int ll_find_deltas(git_packbuilder *pb, git_pobject **list,
|
||||
unsigned int list_size, unsigned int window,
|
||||
int depth)
|
||||
size_t list_size, size_t window, size_t depth)
|
||||
{
|
||||
struct thread_params *p;
|
||||
int i, ret, active_threads = 0;
|
||||
size_t i;
|
||||
int ret, active_threads = 0;
|
||||
|
||||
if (!pb->nr_threads)
|
||||
pb->nr_threads = git_online_cpus();
|
||||
@ -1159,7 +1181,7 @@ static int ll_find_deltas(git_packbuilder *pb, git_pobject **list,
|
||||
|
||||
/* Partition the work among the threads */
|
||||
for (i = 0; i < pb->nr_threads; ++i) {
|
||||
unsigned sub_size = list_size / (pb->nr_threads - i);
|
||||
size_t sub_size = list_size / (pb->nr_threads - i);
|
||||
|
||||
/* don't use too small segments or no deltas will be found */
|
||||
if (sub_size < 2*window && i+1 < pb->nr_threads)
|
||||
@ -1213,7 +1235,7 @@ static int ll_find_deltas(git_packbuilder *pb, git_pobject **list,
|
||||
while (active_threads) {
|
||||
struct thread_params *target = NULL;
|
||||
struct thread_params *victim = NULL;
|
||||
unsigned sub_size = 0;
|
||||
size_t sub_size = 0;
|
||||
|
||||
/* Start by locating a thread that has transitioned its
|
||||
* 'working' flag from 1 -> 0. This indicates that it is
|
||||
@ -1293,7 +1315,7 @@ static int ll_find_deltas(git_packbuilder *pb, git_pobject **list,
|
||||
static int prepare_pack(git_packbuilder *pb)
|
||||
{
|
||||
git_pobject **delta_list;
|
||||
unsigned int i, n = 0;
|
||||
size_t i, n = 0;
|
||||
|
||||
if (pb->nr_objects == 0 || pb->done)
|
||||
return 0; /* nothing to do */
|
||||
@ -1480,12 +1502,12 @@ cleanup:
|
||||
return error;
|
||||
}
|
||||
|
||||
uint32_t git_packbuilder_object_count(git_packbuilder *pb)
|
||||
size_t git_packbuilder_object_count(git_packbuilder *pb)
|
||||
{
|
||||
return pb->nr_objects;
|
||||
}
|
||||
|
||||
uint32_t git_packbuilder_written(git_packbuilder *pb)
|
||||
size_t git_packbuilder_written(git_packbuilder *pb)
|
||||
{
|
||||
return pb->nr_written;
|
||||
}
|
||||
|
||||
@ -42,8 +42,8 @@ typedef struct git_pobject {
|
||||
* me */
|
||||
|
||||
void *delta_data;
|
||||
unsigned long delta_size;
|
||||
unsigned long z_delta_size;
|
||||
size_t delta_size;
|
||||
size_t z_delta_size;
|
||||
|
||||
int written:1,
|
||||
recursing:1,
|
||||
@ -65,10 +65,11 @@ struct git_packbuilder {
|
||||
git_zstream zstream;
|
||||
|
||||
uint32_t nr_objects,
|
||||
nr_deltified,
|
||||
nr_alloc,
|
||||
nr_written,
|
||||
nr_remaining;
|
||||
nr_deltified,
|
||||
nr_written,
|
||||
nr_remaining;
|
||||
|
||||
size_t nr_alloc;
|
||||
|
||||
git_pobject *object_list;
|
||||
|
||||
@ -85,13 +86,13 @@ struct git_packbuilder {
|
||||
git_cond progress_cond;
|
||||
|
||||
/* configs */
|
||||
uint64_t delta_cache_size;
|
||||
uint64_t max_delta_cache_size;
|
||||
uint64_t cache_max_small_delta_size;
|
||||
uint64_t big_file_threshold;
|
||||
uint64_t window_memory_limit;
|
||||
size_t delta_cache_size;
|
||||
size_t max_delta_cache_size;
|
||||
size_t cache_max_small_delta_size;
|
||||
size_t big_file_threshold;
|
||||
size_t window_memory_limit;
|
||||
|
||||
int nr_threads; /* nr of threads to use */
|
||||
unsigned int nr_threads; /* nr of threads to use */
|
||||
|
||||
git_packbuilder_progress progress_cb;
|
||||
void *progress_cb_payload;
|
||||
|
||||
@ -284,7 +284,7 @@ static int create_binary(
|
||||
size_t b_datalen)
|
||||
{
|
||||
git_buf deflate = GIT_BUF_INIT, delta = GIT_BUF_INIT;
|
||||
unsigned long delta_data_len;
|
||||
size_t delta_data_len;
|
||||
int error;
|
||||
|
||||
/* The git_delta function accepts unsigned long only */
|
||||
@ -310,7 +310,7 @@ static int create_binary(
|
||||
|
||||
if (error == 0) {
|
||||
error = git_zstream_deflatebuf(
|
||||
&delta, delta_data, (size_t)delta_data_len);
|
||||
&delta, delta_data, delta_data_len);
|
||||
|
||||
git__free(delta_data);
|
||||
} else if (error == GIT_EBUFS) {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user