diff --git a/include/git2.h b/include/git2.h index 35e48b240..96de524e7 100644 --- a/include/git2.h +++ b/include/git2.h @@ -60,5 +60,6 @@ #include "git2/net.h" #include "git2/transport.h" #include "git2/status.h" +#include "git2/indexer.h" #endif diff --git a/include/git2/indexer.h b/include/git2/indexer.h new file mode 100644 index 000000000..6c31956b1 --- /dev/null +++ b/include/git2/indexer.h @@ -0,0 +1,67 @@ +#ifndef _INCLUDE_git_indexer_h__ +#define _INCLUDE_git_indexer_h__ + +#include "git2/common.h" +#include "git2/oid.h" + +/** + * This is passed as the first argument to the callback to allow the + * user to see the progress. + */ +typedef struct git_indexer_stats { + unsigned int total; + unsigned int processed; +} git_indexer_stats; + + +typedef struct git_indexer git_indexer; + +/** + * Create a new indexer instance + * + * @param out where to store the indexer instance + * @param packname the absolute filename of the packfile to index + */ +GIT_EXTERN(int) git_indexer_new(git_indexer **out, const char *packname); + +/** + * Iterate over the objects in the packfile and extract the information + * + * Indexing a packfile can be very expensive so this function is + * expected to be run in a worker thread and the stats used to provide + * feedback the user. + * + * @param idx the indexer instance + * @param stats storage for the running state + */ +GIT_EXTERN(int) git_indexer_run(git_indexer *idx, git_indexer_stats *stats); + +/** + * Write the index file to disk. + * + * The file will be stored as pack-$hash.idx in the same directory as + * the packfile. + * + * @param idx the indexer instance + */ +GIT_EXTERN(int) git_indexer_write(git_indexer *idx); + +/** + * Get the packfile's hash + * + * A packfile's name is derived from the sorted hashing of all object + * names. This is only correct after the index has been written to disk. + * + * @param idx the indexer instance + */ +GIT_EXTERN(const git_oid *) git_indexer_hash(git_indexer *idx); + +/** + * Free the indexer and its resources + * + * @param idx the indexer to free + */ +GIT_EXTERN(void) git_indexer_free(git_indexer *idx); + + +#endif diff --git a/src/indexer.c b/src/indexer.c new file mode 100644 index 000000000..7a2b28ae3 --- /dev/null +++ b/src/indexer.c @@ -0,0 +1,380 @@ +/* + * This file is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, version 2, + * as published by the Free Software Foundation. + * + * In addition to the permissions in the GNU General Public License, + * the authors give you unlimited permission to link the compiled + * version of this file into combinations with other programs, + * and to distribute those combinations without any restriction + * coming from the use of this file. (The General Public License + * restrictions do apply in other respects; for example, they cover + * modification of the file, and distribution when not linked into + * a combined executable.) + * + * This file is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; see the file COPYING. If not, write to + * the Free Software Foundation, 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "git2/indexer.h" +#include "git2/object.h" +#include "git2/zlib.h" +#include "git2/oid.h" + +#include "common.h" +#include "pack.h" +#include "mwindow.h" +#include "posix.h" +#include "pack.h" +#include "filebuf.h" +#include "sha1.h" + +#define UINT31_MAX (0x7FFFFFFF) + +struct entry { + git_oid oid; + uint32_t crc; + uint32_t offset; + uint64_t offset_long; +}; + +typedef struct git_indexer { + struct git_pack_file *pack; + struct stat st; + struct git_pack_header hdr; + size_t nr_objects; + git_vector objects; + git_filebuf file; + unsigned int fanout[256]; + git_oid hash; +} git_indexer; + +const git_oid *git_indexer_hash(git_indexer *idx) +{ + return &idx->hash; +} + +static int parse_header(git_indexer *idx) +{ + int error; + + /* Verify we recognize this pack file format. */ + if ((error = p_read(idx->pack->mwf.fd, &idx->hdr, sizeof(idx->hdr))) < GIT_SUCCESS) + return git__rethrow(error, "Failed to read in pack header"); + + if (idx->hdr.hdr_signature != ntohl(PACK_SIGNATURE)) + return git__throw(GIT_EOBJCORRUPTED, "Wrong pack signature"); + + if (!pack_version_ok(idx->hdr.hdr_version)) + return git__throw(GIT_EOBJCORRUPTED, "Wrong pack version"); + + + return GIT_SUCCESS; +} + +int objects_cmp(const void *a, const void *b) +{ + const struct entry *entrya = a; + const struct entry *entryb = b; + + return git_oid_cmp(&entrya->oid, &entryb->oid); +} + +int git_indexer_new(git_indexer **out, const char *packname) +{ + git_indexer *idx; + unsigned int namelen; + int ret, error; + + if (git_path_root(packname) < 0) + return git__throw(GIT_EINVALIDPATH, "Path is not absolute"); + + idx = git__malloc(sizeof(git_indexer)); + if (idx == NULL) + return GIT_ENOMEM; + + memset(idx, 0x0, sizeof(*idx)); + + namelen = strlen(packname); + idx->pack = git__malloc(sizeof(struct git_pack_file) + namelen + 1); + if (idx->pack == NULL) + goto cleanup; + + memset(idx->pack, 0x0, sizeof(struct git_pack_file)); + memcpy(idx->pack->pack_name, packname, namelen + 1); + + ret = p_stat(packname, &idx->st); + if (ret < 0) { + if (errno == ENOENT) + error = git__throw(GIT_ENOTFOUND, "Failed to stat packfile. File not found"); + else + error = git__throw(GIT_EOSERR, "Failed to stat packfile."); + + goto cleanup; + } + + ret = p_open(idx->pack->pack_name, O_RDONLY); + if (ret < 0) { + error = git__throw(GIT_EOSERR, "Failed to open packfile"); + goto cleanup; + } + + idx->pack->mwf.fd = ret; + idx->pack->mwf.size = idx->st.st_size; + + error = parse_header(idx); + if (error < GIT_SUCCESS) { + error = git__rethrow(error, "Failed to parse packfile header"); + goto cleanup; + } + + idx->nr_objects = ntohl(idx->hdr.hdr_entries); + + error = git_vector_init(&idx->objects, idx->nr_objects, objects_cmp); + if (error < GIT_SUCCESS) { + goto cleanup; + } + + *out = idx; + + return GIT_SUCCESS; + +cleanup: + git_indexer_free(idx); + + return error; +} + +static void index_path(char *path, git_indexer *idx) +{ + char *ptr; + const char prefix[] = "pack-", suffix[] = ".idx\0"; + + ptr = strrchr(path, '/') + 1; + + memcpy(ptr, prefix, STRLEN(prefix)); + ptr += STRLEN(prefix); + git_oid_fmt(ptr, &idx->hash); + ptr += GIT_OID_HEXSZ; + memcpy(ptr, suffix, STRLEN(suffix)); +} + +int git_indexer_write(git_indexer *idx) +{ + git_mwindow *w = NULL; + int error, namelen; + unsigned int i, long_offsets, left; + struct git_pack_idx_header hdr; + char filename[GIT_PATH_MAX]; + struct entry *entry; + void *packfile_hash; + git_oid file_hash; + SHA_CTX ctx; + + git_vector_sort(&idx->objects); + + namelen = strlen(idx->pack->pack_name); + memcpy(filename, idx->pack->pack_name, namelen); + memcpy(filename + namelen - STRLEN("pack"), "idx\0", STRLEN("idx\0")); + + error = git_filebuf_open(&idx->file, filename, GIT_FILEBUF_HASH_CONTENTS); + + /* Write out the header */ + hdr.idx_signature = htonl(PACK_IDX_SIGNATURE); + hdr.idx_version = htonl(2); + error = git_filebuf_write(&idx->file, &hdr, sizeof(hdr)); + + /* Write out the fanout table */ + for (i = 0; i < 256; ++i) { + uint32_t n = htonl(idx->fanout[i]); + error = git_filebuf_write(&idx->file, &n, sizeof(n)); + if (error < GIT_SUCCESS) + goto cleanup; + } + + /* Write out the object names (SHA-1 hashes) */ + SHA1_Init(&ctx); + git_vector_foreach(&idx->objects, i, entry) { + error = git_filebuf_write(&idx->file, &entry->oid, sizeof(git_oid)); + SHA1_Update(&ctx, &entry->oid, GIT_OID_RAWSZ); + if (error < GIT_SUCCESS) + goto cleanup; + } + SHA1_Final(idx->hash.id, &ctx); + + /* Write out the CRC32 values */ + git_vector_foreach(&idx->objects, i, entry) { + error = git_filebuf_write(&idx->file, &entry->crc, sizeof(uint32_t)); + if (error < GIT_SUCCESS) + goto cleanup; + } + + /* Write out the offsets */ + git_vector_foreach(&idx->objects, i, entry) { + uint32_t n; + + if (entry->offset == UINT32_MAX) + n = htonl(0x80000000 | long_offsets++); + else + n = htonl(entry->offset); + + error = git_filebuf_write(&idx->file, &n, sizeof(uint32_t)); + if (error < GIT_SUCCESS) + goto cleanup; + } + + /* Write out the long offsets */ + git_vector_foreach(&idx->objects, i, entry) { + uint32_t split[2]; + + if (entry->offset != UINT32_MAX) + continue; + + split[0] = htonl(entry->offset_long >> 32); + split[1] = htonl(entry->offset_long & 0xffffffff); + + error = git_filebuf_write(&idx->file, &split, sizeof(uint32_t) * 2); + if (error < GIT_SUCCESS) + goto cleanup; + } + + /* Write out the packfile trailer */ + + packfile_hash = git_mwindow_open(&idx->pack->mwf, &w, idx->st.st_size - GIT_OID_RAWSZ, GIT_OID_RAWSZ, &left); + if (packfile_hash == NULL) { + error = git__rethrow(GIT_ENOMEM, "Failed to open window to packfile hash"); + goto cleanup; + } + + memcpy(&file_hash, packfile_hash, GIT_OID_RAWSZ); + + git_mwindow_close(&w); + + error = git_filebuf_write(&idx->file, &file_hash, sizeof(git_oid)); + + /* Write out the index sha */ + error = git_filebuf_hash(&file_hash, &idx->file); + if (error < GIT_SUCCESS) + goto cleanup; + + error = git_filebuf_write(&idx->file, &file_hash, sizeof(git_oid)); + if (error < GIT_SUCCESS) + goto cleanup; + + /* Figure out what the final name should be */ + index_path(filename, idx); + /* Commit file */ + error = git_filebuf_commit_at(&idx->file, filename); + +cleanup: + if (error < GIT_SUCCESS) + git_filebuf_cleanup(&idx->file); + + return error; +} + +int git_indexer_run(git_indexer *idx, git_indexer_stats *stats) +{ + git_mwindow_file *mwf; + off_t off = sizeof(struct git_pack_header); + int error; + struct entry *entry; + unsigned int left, processed; + + assert(idx && stats); + + mwf = &idx->pack->mwf; + error = git_mwindow_file_register(mwf); + if (error < GIT_SUCCESS) + return git__rethrow(error, "Failed to register mwindow file"); + + stats->total = idx->nr_objects; + stats->processed = processed = 0; + + while (processed < idx->nr_objects) { + git_rawobj obj; + git_oid oid; + git_mwindow *w = NULL; + char hdr[512] = {0}; /* FIXME: How long should this be? */ + int i, hdr_len; + off_t entry_start = off; + void *packed; + size_t entry_size; + + entry = git__malloc(sizeof(struct entry)); + memset(entry, 0x0, sizeof(struct entry)); + + if (off > UINT31_MAX) { + entry->offset = UINT32_MAX; + entry->offset_long = off; + } else { + entry->offset = off; + } + + error = git_packfile_unpack(&obj, idx->pack, &off); + if (error < GIT_SUCCESS) { + error = git__rethrow(error, "Failed to unpack object"); + goto cleanup; + } + + error = git_odb__hash_obj(&oid, hdr, sizeof(hdr), &hdr_len, &obj); + if (error < GIT_SUCCESS) { + error = git__rethrow(error, "Failed to hash object"); + goto cleanup; + } + + git_oid_cpy(&entry->oid, &oid); + entry->crc = crc32(0L, Z_NULL, 0); + + entry_size = off - entry_start; + packed = git_mwindow_open(mwf, &w, entry_start, entry_size, &left); + if (packed == NULL) { + error = git__rethrow(error, "Failed to open window to read packed data"); + goto cleanup; + } + entry->crc = htonl(crc32(entry->crc, packed, entry_size)); + git_mwindow_close(&w); + + /* Add the object to the list */ + error = git_vector_insert(&idx->objects, entry); + if (error < GIT_SUCCESS) { + error = git__rethrow(error, "Failed to add entry to list"); + goto cleanup; + } + + for (i = oid.id[0]; i < 256; ++i) { + idx->fanout[i]++; + } + + free(obj.data); + + stats->processed = ++processed; + } + +cleanup: + git_mwindow_free_all(mwf); + + return error; + +} + +void git_indexer_free(git_indexer *idx) +{ + unsigned int i; + struct entry *e; + + p_close(idx->pack->mwf.fd); + git_vector_foreach(&idx->objects, i, e) + free(e); + git_vector_free(&idx->objects); + free(idx->pack); + free(idx); +} + diff --git a/src/mwindow.c b/src/mwindow.c new file mode 100644 index 000000000..2f7fc7f7d --- /dev/null +++ b/src/mwindow.c @@ -0,0 +1,271 @@ +/* + * This file is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, version 2, + * as published by the Free Software Foundation. + * + * In addition to the permissions in the GNU General Public License, + * the authors give you unlimited permission to link the compiled + * version of this file into combinations with other programs, + * and to distribute those combinations without any restriction + * coming from the use of this file. (The General Public License + * restrictions do apply in other respects; for example, they cover + * modification of the file, and distribution when not linked into + * a combined executable.) + * + * This file is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; see the file COPYING. If not, write to + * the Free Software Foundation, 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "common.h" +#include "mwindow.h" +#include "vector.h" +#include "fileops.h" +#include "map.h" + +#define DEFAULT_WINDOW_SIZE \ + (sizeof(void*) >= 8 \ + ? 1 * 1024 * 1024 * 1024 \ + : 32 * 1024 * 1024) + +#define DEFAULT_MAPPED_LIMIT \ + ((1024L * 1024L) * (sizeof(void*) >= 8 ? 8192 : 256)) + +/* + * We need this because each process is only allowed a specific amount + * of memory. Making it writable should generate one instance per + * process, but we still need to set a couple of variables. + */ + +static git_mwindow_ctl ctl = { + .window_size = DEFAULT_WINDOW_SIZE, + .mapped_limit = DEFAULT_MAPPED_LIMIT +}; + +/* + * Free all the windows in a sequence, typically because we're done + * with the file + */ +void git_mwindow_free_all(git_mwindow_file *mwf) +{ + unsigned int i; + /* + * Remove these windows from the global list + */ + for (i = 0; i < ctl.windowfiles.length; ++i){ + if (git_vector_get(&ctl.windowfiles, i) == mwf) { + git_vector_remove(&ctl.windowfiles, i); + break; + } + } + + if (ctl.windowfiles.length == 0) { + git_vector_free(&ctl.windowfiles); + ctl.windowfiles.contents = NULL; + } + + while (mwf->windows) { + git_mwindow *w = mwf->windows; + assert(w->inuse_cnt == 0); + + ctl.mapped -= w->window_map.len; + ctl.open_windows--; + + git_futils_mmap_free(&w->window_map); + + mwf->windows = w->next; + free(w); + } +} + +/* + * Check if a window 'win' contains the address 'offset' + */ +int git_mwindow_contains(git_mwindow *win, off_t offset) +{ + off_t win_off = win->offset; + return win_off <= offset + && offset <= (off_t)(win_off + win->window_map.len); +} + +/* + * Find the least-recently-used window in a file + */ +void git_mwindow_scan_lru( + git_mwindow_file *mwf, + git_mwindow **lru_w, + git_mwindow **lru_l) +{ + git_mwindow *w, *w_l; + + for (w_l = NULL, w = mwf->windows; w; w = w->next) { + if (!w->inuse_cnt) { + /* + * If the current one is more recent than the last one, + * store it in the output parameter. If lru_w is NULL, + * it's the first loop, so store it as well. + */ + if (!*lru_w || w->last_used < (*lru_w)->last_used) { + *lru_w = w; + *lru_l = w_l; + } + } + w_l = w; + } +} + +/* + * Close the least recently used window. You should check to see if + * the file descriptors need closing from time to time. + */ +int git_mwindow_close_lru(git_mwindow_file *mwf) +{ + unsigned int i; + git_mwindow *lru_w = NULL, *lru_l = NULL; + + /* FIMXE: Does this give us any advantage? */ + if(mwf->windows) + git_mwindow_scan_lru(mwf, &lru_w, &lru_l); + + for (i = 0; i < ctl.windowfiles.length; ++i) { + git_mwindow_scan_lru(git_vector_get(&ctl.windowfiles, i), &lru_w, &lru_l); + } + + if (lru_w) { + git_mwindow_close(&lru_w); + ctl.mapped -= lru_w->window_map.len; + git_futils_mmap_free(&lru_w->window_map); + + if (lru_l) + lru_l->next = lru_w->next; + else + mwf->windows = lru_w->next; + + free(lru_w); + ctl.open_windows--; + + return GIT_SUCCESS; + } + + return git__throw(GIT_ERROR, "Failed to close memory window. Couln't find LRU"); +} + +static git_mwindow *new_window(git_mwindow_file *mwf, git_file fd, size_t size, off_t offset) +{ + size_t walign = ctl.window_size / 2; + size_t len; + git_mwindow *w; + + w = git__malloc(sizeof(*w)); + if (w == NULL) + return w; + + memset(w, 0x0, sizeof(*w)); + w->offset = (offset / walign) * walign; + + len = size - w->offset; + if (len > ctl.window_size) + len = ctl.window_size; + + ctl.mapped += len; + + while(ctl.mapped_limit < ctl.mapped && + git_mwindow_close_lru(mwf) == GIT_SUCCESS) {} + + /* FIXME: Shouldn't we error out if there's an error in closing lru? */ + + if (git_futils_mmap_ro(&w->window_map, fd, w->offset, len) < GIT_SUCCESS) + goto cleanup; + + ctl.mmap_calls++; + ctl.open_windows++; + + if (ctl.mapped > ctl.peak_mapped) + ctl.peak_mapped = ctl.mapped; + + if (ctl.open_windows > ctl.peak_open_windows) + ctl.peak_open_windows = ctl.open_windows; + + return w; + +cleanup: + free(w); + return NULL; +} + +/* + * Open a new window, closing the least recenty used until we have + * enough space. Don't forget to add it to your list + */ +unsigned char *git_mwindow_open(git_mwindow_file *mwf, git_mwindow **cursor, + off_t offset, int extra, unsigned int *left) +{ + git_mwindow *w = *cursor; + + if (!w || !git_mwindow_contains(w, offset + extra)) { + if (w) { + w->inuse_cnt--; + } + + for (w = mwf->windows; w; w = w->next) { + if (git_mwindow_contains(w, offset + extra)) + break; + } + + /* + * If there isn't a suitable window, we need to create a new + * one. + */ + if (!w) { + w = new_window(mwf, mwf->fd, mwf->size, offset); + if (w == NULL) + return NULL; + w->next = mwf->windows; + mwf->windows = w; + } + } + + /* If we changed w, store it in the cursor */ + if (w != *cursor) { + w->last_used = ctl.used_ctr++; + w->inuse_cnt++; + *cursor = w; + } + + offset -= w->offset; + assert(git__is_sizet(offset)); + + if (left) + *left = w->window_map.len - offset; + + return (unsigned char *) w->window_map.data + offset; + + free(w); + return NULL; +} + +int git_mwindow_file_register(git_mwindow_file *mwf) +{ + int error; + + if (ctl.windowfiles.length == 0 && + (error = git_vector_init(&ctl.windowfiles, 8, NULL)) < GIT_SUCCESS) + return error; + + return git_vector_insert(&ctl.windowfiles, mwf); +} + +void git_mwindow_close(git_mwindow **window) +{ + git_mwindow *w = *window; + if (w) { + w->inuse_cnt--; + *window = NULL; + } +} diff --git a/src/mwindow.h b/src/mwindow.h new file mode 100644 index 000000000..6c29307a7 --- /dev/null +++ b/src/mwindow.h @@ -0,0 +1,66 @@ +/* + * This file is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, version 2, + * as published by the Free Software Foundation. + * + * In addition to the permissions in the GNU General Public License, + * the authors give you unlimited permission to link the compiled + * version of this file into combinations with other programs, + * and to distribute those combinations without any restriction + * coming from the use of this file. (The General Public License + * restrictions do apply in other respects; for example, they cover + * modification of the file, and distribution when not linked into + * a combined executable.) + * + * This file is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; see the file COPYING. If not, write to + * the Free Software Foundation, 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef INCLUDE_mwindow__ +#define INCLUDE_mwindow__ + +#include "map.h" +#include "vector.h" +#include "fileops.h" + +typedef struct git_mwindow { + struct git_mwindow *next; + git_map window_map; + off_t offset; + unsigned int last_used; + unsigned int inuse_cnt; +} git_mwindow; + +typedef struct git_mwindow_file { + git_mwindow *windows; + int fd; + off_t size; +} git_mwindow_file; + +typedef struct git_mwindow_ctl { + size_t mapped; + unsigned int open_windows; + size_t window_size; /* needs default value */ + size_t mapped_limit; /* needs default value */ + unsigned int mmap_calls; + unsigned int peak_open_windows; + size_t peak_mapped; + size_t used_ctr; + git_vector windowfiles; +} git_mwindow_ctl; + +int git_mwindow_contains(git_mwindow *win, off_t offset); +void git_mwindow_free_all(git_mwindow_file *mwf); +unsigned char *git_mwindow_open(git_mwindow_file *mwf, git_mwindow **cursor, off_t offset, int extra, unsigned int *left); +void git_mwindow_scan_lru(git_mwindow_file *mwf, git_mwindow **lru_w, git_mwindow **lru_l); +int git_mwindow_file_register(git_mwindow_file *mwf); +void git_mwindow_close(git_mwindow **w_cursor); + +#endif diff --git a/src/odb_pack.c b/src/odb_pack.c index 67b983da0..0d6bb05cc 100644 --- a/src/odb_pack.c +++ b/src/odb_pack.c @@ -32,103 +32,17 @@ #include "odb.h" #include "delta-apply.h" #include "sha1_lookup.h" +#include "mwindow.h" +#include "pack.h" #include "git2/odb_backend.h" -#define DEFAULT_WINDOW_SIZE \ - (sizeof(void*) >= 8 \ - ? 1 * 1024 * 1024 * 1024 \ - : 32 * 1024 * 1024) - -#define DEFAULT_MAPPED_LIMIT \ - ((1024L * 1024L) * (sizeof(void*) >= 8 ? 8192 : 256)) - -#define PACK_SIGNATURE 0x5041434b /* "PACK" */ -#define PACK_VERSION 2 -#define pack_version_ok(v) ((v) == htonl(2) || (v) == htonl(3)) -struct pack_header { - uint32_t hdr_signature; - uint32_t hdr_version; - uint32_t hdr_entries; -}; - -/* - * The first four bytes of index formats later than version 1 should - * start with this signature, as all older git binaries would find this - * value illegal and abort reading the file. - * - * This is the case because the number of objects in a packfile - * cannot exceed 1,431,660,000 as every object would need at least - * 3 bytes of data and the overall packfile cannot exceed 4 GiB with - * version 1 of the index file due to the offsets limited to 32 bits. - * Clearly the signature exceeds this maximum. - * - * Very old git binaries will also compare the first 4 bytes to the - * next 4 bytes in the index and abort with a "non-monotonic index" - * error if the second 4 byte word is smaller than the first 4 - * byte word. This would be true in the proposed future index - * format as idx_signature would be greater than idx_version. - */ -#define PACK_IDX_SIGNATURE 0xff744f63 /* "\377tOc" */ - -struct pack_idx_header { - uint32_t idx_signature; - uint32_t idx_version; -}; - -struct pack_window { - struct pack_window *next; - git_map window_map; - off_t offset; - unsigned int last_used; - unsigned int inuse_cnt; -}; - -struct pack_file { - struct pack_window *windows; - off_t pack_size; - - git_map index_map; - - uint32_t num_objects; - uint32_t num_bad_objects; - git_oid *bad_object_sha1; /* array of git_oid */ - - int index_version; - git_time_t mtime; - int pack_fd; - unsigned pack_local:1, pack_keep:1; - git_oid sha1; - - /* something like ".git/objects/pack/xxxxx.pack" */ - char pack_name[GIT_FLEX_ARRAY]; /* more */ -}; - -struct pack_entry { - off_t offset; - git_oid sha1; - struct pack_file *p; -}; - struct pack_backend { git_odb_backend parent; git_vector packs; - struct pack_file *last_found; + struct git_pack_file *last_found; char *pack_folder; time_t pack_folder_mtime; - - size_t window_size; /* needs default value */ - - size_t mapped_limit; /* needs default value */ - size_t peak_mapped; - size_t mapped; - - size_t used_ctr; - - unsigned int peak_open_windows; - unsigned int open_windows; - - unsigned int mmap_calls; }; /** @@ -226,65 +140,21 @@ struct pack_backend { */ - - /*********************************************************** * * FORWARD DECLARATIONS * ***********************************************************/ -static void pack_window_free_all(struct pack_backend *backend, struct pack_file *p); -static int pack_window_contains(struct pack_window *win, off_t offset); - -static void pack_window_scan_lru(struct pack_file *p, struct pack_file **lru_p, - struct pack_window **lru_w, struct pack_window **lru_l); - -static int pack_window_close_lru( struct pack_backend *backend, - struct pack_file *current, git_file keep_fd); - -static void pack_window_close(struct pack_window **w_cursor); - -static unsigned char *pack_window_open( struct pack_backend *backend, - struct pack_file *p, struct pack_window **w_cursor, off_t offset, - unsigned int *left); +static void pack_window_free_all(struct pack_backend *backend, struct git_pack_file *p); +static int pack_window_contains(git_mwindow *win, off_t offset); static int packfile_sort__cb(const void *a_, const void *b_); -static void pack_index_free(struct pack_file *p); - -static int pack_index_check(const char *path, struct pack_file *p); -static int pack_index_open(struct pack_file *p); - -static struct pack_file *packfile_alloc(int extra); -static int packfile_open(struct pack_file *p); -static int packfile_check(struct pack_file **pack_out, const char *path); static int packfile_load__cb(void *_data, char *path); static int packfile_refresh_all(struct pack_backend *backend); -static off_t nth_packed_object_offset(const struct pack_file *p, uint32_t n); - -/* Can find the offset of an object given - * a prefix of an identifier. - * Throws GIT_EAMBIGUOUSOIDPREFIX if short oid - * is ambiguous within the pack. - * This method assumes that len is between - * GIT_OID_MINPREFIXLEN and GIT_OID_HEXSZ. - */ -static int pack_entry_find_offset( - off_t *offset_out, - git_oid *found_oid, - struct pack_file *p, - const git_oid *short_oid, - unsigned int len); - -static int pack_entry_find1( - struct pack_entry *e, - struct pack_file *p, - const git_oid *short_oid, - unsigned int len); - -static int pack_entry_find(struct pack_entry *e, +static int pack_entry_find(struct git_pack_entry *e, struct pack_backend *backend, const git_oid *oid); /* Can find the offset of an object given @@ -294,54 +164,11 @@ static int pack_entry_find(struct pack_entry *e, * This method assumes that len is between * GIT_OID_MINPREFIXLEN and GIT_OID_HEXSZ. */ -static int pack_entry_find_prefix(struct pack_entry *e, +static int pack_entry_find_prefix(struct git_pack_entry *e, struct pack_backend *backend, const git_oid *short_oid, unsigned int len); -static off_t get_delta_base(struct pack_backend *backend, - struct pack_file *p, struct pack_window **w_curs, - off_t *curpos, git_otype type, - off_t delta_obj_offset); - -static unsigned long packfile_unpack_header1( - size_t *sizep, - git_otype *type, - const unsigned char *buf, - unsigned long len); - -static int packfile_unpack_header( - size_t *size_p, - git_otype *type_p, - struct pack_backend *backend, - struct pack_file *p, - struct pack_window **w_curs, - off_t *curpos); - -static int packfile_unpack_compressed( - git_rawobj *obj, - struct pack_backend *backend, - struct pack_file *p, - struct pack_window **w_curs, - off_t curpos, - size_t size, - git_otype type); - -static int packfile_unpack_delta( - git_rawobj *obj, - struct pack_backend *backend, - struct pack_file *p, - struct pack_window **w_curs, - off_t curpos, - size_t delta_size, - git_otype delta_type, - off_t obj_offset); - -static int packfile_unpack(git_rawobj *obj, struct pack_backend *backend, - struct pack_file *p, off_t obj_offset); - - - /*********************************************************** @@ -350,23 +177,12 @@ static int packfile_unpack(git_rawobj *obj, struct pack_backend *backend, * ***********************************************************/ -void pack_window_free_all(struct pack_backend *backend, struct pack_file *p) +GIT_INLINE(void) pack_window_free_all(struct pack_backend *GIT_UNUSED(backend), struct git_pack_file *p) { - while (p->windows) { - struct pack_window *w = p->windows; - assert(w->inuse_cnt == 0); - - backend->mapped -= w->window_map.len; - backend->open_windows--; - - git_futils_mmap_free(&w->window_map); - - p->windows = w->next; - free(w); - } + git_mwindow_free_all(&p->mwf); } -GIT_INLINE(int) pack_window_contains(struct pack_window *win, off_t offset) +GIT_INLINE(int) pack_window_contains(git_mwindow *win, off_t offset) { /* We must promise at least 20 bytes (one hash) after the * offset is available from this window, otherwise the offset @@ -374,333 +190,13 @@ GIT_INLINE(int) pack_window_contains(struct pack_window *win, off_t offset) * has that one hash excess) must be used. This is to support * the object header and delta base parsing routines below. */ - off_t win_off = win->offset; - return win_off <= offset - && (offset + 20) <= (off_t)(win_off + win->window_map.len); + return git_mwindow_contains(win, offset + 20); } -static void pack_window_scan_lru( - struct pack_file *p, - struct pack_file **lru_p, - struct pack_window **lru_w, - struct pack_window **lru_l) -{ - struct pack_window *w, *w_l; - - for (w_l = NULL, w = p->windows; w; w = w->next) { - if (!w->inuse_cnt) { - if (!*lru_w || w->last_used < (*lru_w)->last_used) { - *lru_p = p; - *lru_w = w; - *lru_l = w_l; - } - } - w_l = w; - } -} - -static int pack_window_close_lru( - struct pack_backend *backend, - struct pack_file *current, - git_file keep_fd) -{ - struct pack_file *lru_p = NULL; - struct pack_window *lru_w = NULL, *lru_l = NULL; - size_t i; - - if (current) - pack_window_scan_lru(current, &lru_p, &lru_w, &lru_l); - - for (i = 0; i < backend->packs.length; ++i) - pack_window_scan_lru(git_vector_get(&backend->packs, i), &lru_p, &lru_w, &lru_l); - - if (lru_p) { - backend->mapped -= lru_w->window_map.len; - git_futils_mmap_free(&lru_w->window_map); - - if (lru_l) - lru_l->next = lru_w->next; - else { - lru_p->windows = lru_w->next; - if (!lru_p->windows && lru_p->pack_fd != keep_fd) { - p_close(lru_p->pack_fd); - lru_p->pack_fd = -1; - } - } - - free(lru_w); - backend->open_windows--; - return GIT_SUCCESS; - } - - return git__throw(GIT_ERROR, "Failed to close pack window"); -} - -static void pack_window_close(struct pack_window **w_cursor) -{ - struct pack_window *w = *w_cursor; - if (w) { - w->inuse_cnt--; - *w_cursor = NULL; - } -} - -static unsigned char *pack_window_open( - struct pack_backend *backend, - struct pack_file *p, - struct pack_window **w_cursor, - off_t offset, - unsigned int *left) -{ - struct pack_window *win = *w_cursor; - - if (p->pack_fd == -1 && packfile_open(p) < GIT_SUCCESS) - return NULL; - - /* Since packfiles end in a hash of their content and it's - * pointless to ask for an offset into the middle of that - * hash, and the pack_window_contains function above wouldn't match - * don't allow an offset too close to the end of the file. - */ - if (offset > (p->pack_size - 20)) - return NULL; - - if (!win || !pack_window_contains(win, offset)) { - - if (win) - win->inuse_cnt--; - - for (win = p->windows; win; win = win->next) { - if (pack_window_contains(win, offset)) - break; - } - - if (!win) { - size_t window_align = backend->window_size / 2; - size_t len; - - win = git__calloc(1, sizeof(*win)); - if (win == NULL) - return NULL; - - win->offset = (offset / window_align) * window_align; - - len = (size_t)(p->pack_size - win->offset); - if (len > backend->window_size) - len = backend->window_size; - - backend->mapped += len; - - while (backend->mapped_limit < backend->mapped && - pack_window_close_lru(backend, p, p->pack_fd) == GIT_SUCCESS) {} - - if (git_futils_mmap_ro(&win->window_map, p->pack_fd, - win->offset, len) < GIT_SUCCESS) { - free(win); - return NULL; - } - - backend->mmap_calls++; - backend->open_windows++; - - if (backend->mapped > backend->peak_mapped) - backend->peak_mapped = backend->mapped; - - if (backend->open_windows > backend->peak_open_windows) - backend->peak_open_windows = backend->open_windows; - - win->next = p->windows; - p->windows = win; - } - } - - if (win != *w_cursor) { - win->last_used = backend->used_ctr++; - win->inuse_cnt++; - *w_cursor = win; - } - - offset -= win->offset; - assert(git__is_sizet(offset)); - - if (left) - *left = win->window_map.len - (size_t)offset; - - return (unsigned char *)win->window_map.data + offset; -} - - - - - - - -/*********************************************************** - * - * PACK INDEX METHODS - * - ***********************************************************/ - -static void pack_index_free(struct pack_file *p) -{ - if (p->index_map.data) { - git_futils_mmap_free(&p->index_map); - p->index_map.data = NULL; - } -} - -static int pack_index_check(const char *path, struct pack_file *p) -{ - struct pack_idx_header *hdr; - uint32_t version, nr, i, *index; - - void *idx_map; - size_t idx_size; - - struct stat st; - - /* TODO: properly open the file without access time */ - git_file fd = p_open(path, O_RDONLY /*| O_NOATIME */); - - int error; - - if (fd < 0) - return git__throw(GIT_EOSERR, "Failed to check index. File missing or corrupted"); - - if (p_fstat(fd, &st) < GIT_SUCCESS) { - p_close(fd); - return git__throw(GIT_EOSERR, "Failed to check index. File appears to be corrupted"); - } - - if (!git__is_sizet(st.st_size)) - return GIT_ENOMEM; - - idx_size = (size_t)st.st_size; - - if (idx_size < 4 * 256 + 20 + 20) { - p_close(fd); - return git__throw(GIT_EOBJCORRUPTED, "Failed to check index. Object is corrupted"); - } - - error = git_futils_mmap_ro(&p->index_map, fd, 0, idx_size); - p_close(fd); - - if (error < GIT_SUCCESS) - return git__rethrow(error, "Failed to check index"); - - hdr = idx_map = p->index_map.data; - - if (hdr->idx_signature == htonl(PACK_IDX_SIGNATURE)) { - version = ntohl(hdr->idx_version); - - if (version < 2 || version > 2) { - git_futils_mmap_free(&p->index_map); - return git__throw(GIT_EOBJCORRUPTED, "Failed to check index. Unsupported index version"); - } - - } else - version = 1; - - nr = 0; - index = idx_map; - - if (version > 1) - index += 2; /* skip index header */ - - for (i = 0; i < 256; i++) { - uint32_t n = ntohl(index[i]); - if (n < nr) { - git_futils_mmap_free(&p->index_map); - return git__throw(GIT_EOBJCORRUPTED, "Failed to check index. Index is non-monotonic"); - } - nr = n; - } - - if (version == 1) { - /* - * Total size: - * - 256 index entries 4 bytes each - * - 24-byte entries * nr (20-byte sha1 + 4-byte offset) - * - 20-byte SHA1 of the packfile - * - 20-byte SHA1 file checksum - */ - if (idx_size != 4*256 + nr * 24 + 20 + 20) { - git_futils_mmap_free(&p->index_map); - return git__throw(GIT_EOBJCORRUPTED, "Failed to check index. Object is corrupted"); - } - } else if (version == 2) { - /* - * Minimum size: - * - 8 bytes of header - * - 256 index entries 4 bytes each - * - 20-byte sha1 entry * nr - * - 4-byte crc entry * nr - * - 4-byte offset entry * nr - * - 20-byte SHA1 of the packfile - * - 20-byte SHA1 file checksum - * And after the 4-byte offset table might be a - * variable sized table containing 8-byte entries - * for offsets larger than 2^31. - */ - unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20; - unsigned long max_size = min_size; - - if (nr) - max_size += (nr - 1)*8; - - if (idx_size < min_size || idx_size > max_size) { - git_futils_mmap_free(&p->index_map); - return git__throw(GIT_EOBJCORRUPTED, "Failed to check index. Wrong index size"); - } - - /* Make sure that off_t is big enough to access the whole pack... - * Is this an issue in libgit2? It shouldn't. */ - if (idx_size != min_size && (sizeof(off_t) <= 4)) { - git_futils_mmap_free(&p->index_map); - return git__throw(GIT_EOSERR, "Failed to check index. off_t not big enough to access the whole pack"); - } - } - - p->index_version = version; - p->num_objects = nr; - return GIT_SUCCESS; -} - -static int pack_index_open(struct pack_file *p) -{ - char *idx_name; - int error; - - if (p->index_map.data) - return GIT_SUCCESS; - - idx_name = git__strdup(p->pack_name); - strcpy(idx_name + strlen(idx_name) - STRLEN(".pack"), ".idx"); - - error = pack_index_check(idx_name, p); - free(idx_name); - - return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to open index"); -} - - - - - - - - - -/*********************************************************** - * - * PACKFILE METHODS - * - ***********************************************************/ - static int packfile_sort__cb(const void *a_, const void *b_) { - const struct pack_file *a = a_; - const struct pack_file *b = b_; + const struct git_pack_file *a = a_; + const struct git_pack_file *b = b_; int st; /* @@ -726,152 +222,12 @@ static int packfile_sort__cb(const void *a_, const void *b_) return -1; } -static struct pack_file *packfile_alloc(int extra) -{ - struct pack_file *p = git__malloc(sizeof(*p) + extra); - memset(p, 0, sizeof(*p)); - p->pack_fd = -1; - return p; -} -static void packfile_free(struct pack_backend *backend, struct pack_file *p) -{ - assert(p); - - /* clear_delta_base_cache(); */ - pack_window_free_all(backend, p); - - if (p->pack_fd != -1) - p_close(p->pack_fd); - - pack_index_free(p); - - free(p->bad_object_sha1); - free(p); -} - -static int packfile_open(struct pack_file *p) -{ - struct stat st; - struct pack_header hdr; - git_oid sha1; - unsigned char *idx_sha1; - - if (!p->index_map.data && pack_index_open(p) < GIT_SUCCESS) - return git__throw(GIT_ENOTFOUND, "Failed to open packfile. File not found"); - - /* TODO: open with noatime */ - p->pack_fd = p_open(p->pack_name, O_RDONLY); - if (p->pack_fd < 0 || p_fstat(p->pack_fd, &st) < GIT_SUCCESS) - return git__throw(GIT_EOSERR, "Failed to open packfile. File appears to be corrupted"); - - /* If we created the struct before we had the pack we lack size. */ - if (!p->pack_size) { - if (!S_ISREG(st.st_mode)) - goto cleanup; - p->pack_size = (off_t)st.st_size; - } else if (p->pack_size != st.st_size) - goto cleanup; - -#if 0 - /* We leave these file descriptors open with sliding mmap; - * there is no point keeping them open across exec(), though. - */ - fd_flag = fcntl(p->pack_fd, F_GETFD, 0); - if (fd_flag < 0) - return error("cannot determine file descriptor flags"); - - fd_flag |= FD_CLOEXEC; - if (fcntl(p->pack_fd, F_SETFD, fd_flag) == -1) - return GIT_EOSERR; -#endif - - /* Verify we recognize this pack file format. */ - if (p_read(p->pack_fd, &hdr, sizeof(hdr)) < GIT_SUCCESS) - goto cleanup; - - if (hdr.hdr_signature != htonl(PACK_SIGNATURE)) - goto cleanup; - - if (!pack_version_ok(hdr.hdr_version)) - goto cleanup; - - /* Verify the pack matches its index. */ - if (p->num_objects != ntohl(hdr.hdr_entries)) - goto cleanup; - - if (p_lseek(p->pack_fd, p->pack_size - GIT_OID_RAWSZ, SEEK_SET) == -1) - goto cleanup; - - if (p_read(p->pack_fd, sha1.id, GIT_OID_RAWSZ) < GIT_SUCCESS) - goto cleanup; - - idx_sha1 = ((unsigned char *)p->index_map.data) + p->index_map.len - 40; - - if (git_oid_cmp(&sha1, (git_oid *)idx_sha1) != 0) - goto cleanup; - - return GIT_SUCCESS; - -cleanup: - p_close(p->pack_fd); - p->pack_fd = -1; - return git__throw(GIT_EPACKCORRUPTED, "Failed to packfile. Pack is corrupted"); -} - -static int packfile_check(struct pack_file **pack_out, const char *path) -{ - struct stat st; - struct pack_file *p; - size_t path_len; - - *pack_out = NULL; - path_len = strlen(path); - p = packfile_alloc(path_len + 2); - - /* - * Make sure a corresponding .pack file exists and that - * the index looks sane. - */ - path_len -= STRLEN(".idx"); - if (path_len < 1) { - free(p); - return git__throw(GIT_ENOTFOUND, "Failed to check packfile. Wrong path name"); - } - - memcpy(p->pack_name, path, path_len); - - strcpy(p->pack_name + path_len, ".keep"); - if (git_futils_exists(p->pack_name) == GIT_SUCCESS) - p->pack_keep = 1; - - strcpy(p->pack_name + path_len, ".pack"); - if (p_stat(p->pack_name, &st) < GIT_SUCCESS || !S_ISREG(st.st_mode)) { - free(p); - return git__throw(GIT_ENOTFOUND, "Failed to check packfile. File not found"); - } - - /* ok, it looks sane as far as we can check without - * actually mapping the pack file. - */ - p->pack_size = (off_t)st.st_size; - p->pack_local = 1; - p->mtime = (git_time_t)st.st_mtime; - - /* see if we can parse the sha1 oid in the packfile name */ - if (path_len < 40 || - git_oid_fromstr(&p->sha1, path + path_len - GIT_OID_HEXSZ) < GIT_SUCCESS) - memset(&p->sha1, 0x0, GIT_OID_RAWSZ); - - *pack_out = p; - return GIT_SUCCESS; -} - static int packfile_load__cb(void *_data, char *path) { struct pack_backend *backend = (struct pack_backend *)_data; - struct pack_file *pack; + struct git_pack_file *pack; int error; size_t i; @@ -879,12 +235,12 @@ static int packfile_load__cb(void *_data, char *path) return GIT_SUCCESS; /* not an index */ for (i = 0; i < backend->packs.length; ++i) { - struct pack_file *p = git_vector_get(&backend->packs, i); + struct git_pack_file *p = git_vector_get(&backend->packs, i); if (memcmp(p->pack_name, path, strlen(path) - STRLEN(".idx")) == 0) return GIT_SUCCESS; } - error = packfile_check(&pack, path); + error = git_packfile_check(&pack, path); if (error < GIT_SUCCESS) return git__rethrow(error, "Failed to load packfile"); @@ -923,169 +279,7 @@ static int packfile_refresh_all(struct pack_backend *backend) return GIT_SUCCESS; } - - - - - - - -/*********************************************************** - * - * PACKFILE ENTRY SEARCH INTERNALS - * - ***********************************************************/ - -static off_t nth_packed_object_offset(const struct pack_file *p, uint32_t n) -{ - const unsigned char *index = p->index_map.data; - index += 4 * 256; - if (p->index_version == 1) { - return ntohl(*((const uint32_t *)(index + 24 * n))); - } else { - uint32_t off; - index += 8 + p->num_objects * (20 + 4); - off = ntohl(*((const uint32_t *)(index + 4 * n))); - if (!(off & 0x80000000)) - return off; - index += p->num_objects * 4 + (off & 0x7fffffff) * 8; - return (((uint64_t)ntohl(*((const uint32_t *)(index + 0)))) << 32) | - ntohl(*((const uint32_t *)(index + 4))); - } -} - -static int pack_entry_find_offset( - off_t *offset_out, - git_oid *found_oid, - struct pack_file *p, - const git_oid *short_oid, - unsigned int len) -{ - const uint32_t *level1_ofs = p->index_map.data; - const unsigned char *index = p->index_map.data; - unsigned hi, lo, stride; - int pos, found = 0; - const unsigned char *current = 0; - - *offset_out = 0; - - if (index == NULL) { - int error; - - if ((error = pack_index_open(p)) < GIT_SUCCESS) - return git__rethrow(error, "Failed to find offset for pack entry"); - - assert(p->index_map.data); - - index = p->index_map.data; - level1_ofs = p->index_map.data; - } - - if (p->index_version > 1) { - level1_ofs += 2; - index += 8; - } - - index += 4 * 256; - hi = ntohl(level1_ofs[(int)short_oid->id[0]]); - lo = ((short_oid->id[0] == 0x0) ? 0 : ntohl(level1_ofs[(int)short_oid->id[0] - 1])); - - if (p->index_version > 1) { - stride = 20; - } else { - stride = 24; - index += 4; - } - -#ifdef INDEX_DEBUG_LOOKUP - printf("%02x%02x%02x... lo %u hi %u nr %d\n", - short_oid->id[0], short_oid->id[1], short_oid->id[2], lo, hi, p->num_objects); -#endif - - /* Use git.git lookup code */ - pos = sha1_entry_pos(index, stride, 0, lo, hi, p->num_objects, short_oid->id); - - if (pos >= 0) { - /* An object matching exactly the oid was found */ - found = 1; - current = index + pos * stride; - } else { - /* No object was found */ - /* pos refers to the object with the "closest" oid to short_oid */ - pos = - 1 - pos; - if (pos < (int)p->num_objects) { - current = index + pos * stride; - - if (!git_oid_ncmp(short_oid, (const git_oid *)current, len)) { - found = 1; - } - } - } - - if (found && pos + 1 < (int)p->num_objects) { - /* Check for ambiguousity */ - const unsigned char *next = current + stride; - - if (!git_oid_ncmp(short_oid, (const git_oid *)next, len)) { - found = 2; - } - } - - if (!found) { - return git__throw(GIT_ENOTFOUND, "Failed to find offset for pack entry. Entry not found"); - } else if (found > 1) { - return git__throw(GIT_EAMBIGUOUSOIDPREFIX, "Failed to find offset for pack entry. Ambiguous sha1 prefix within pack"); - } else { - *offset_out = nth_packed_object_offset(p, pos); - git_oid_fromraw(found_oid, current); - -#ifdef INDEX_DEBUG_LOOKUP - unsigned char hex_sha1[GIT_OID_HEXSZ + 1]; - git_oid_fmt(hex_sha1, found_oid); - hex_sha1[GIT_OID_HEXSZ] = '\0'; - printf("found lo=%d %s\n", lo, hex_sha1); -#endif - return GIT_SUCCESS; - } -} - -static int pack_entry_find1( - struct pack_entry *e, - struct pack_file *p, - const git_oid *short_oid, - unsigned int len) -{ - off_t offset; - git_oid found_oid; - int error; - - assert(p); - - if (len == GIT_OID_HEXSZ && p->num_bad_objects) { - unsigned i; - for (i = 0; i < p->num_bad_objects; i++) - if (git_oid_cmp(short_oid, &p->bad_object_sha1[i]) == 0) - return git__throw(GIT_ERROR, "Failed to find pack entry. Bad object found"); - } - - error = pack_entry_find_offset(&offset, &found_oid, p, short_oid, len); - if (error < GIT_SUCCESS) - return git__rethrow(error, "Failed to find pack entry. Couldn't find offset"); - - /* we found a unique entry in the index; - * make sure the packfile backing the index - * still exists on disk */ - if (p->pack_fd == -1 && packfile_open(p) < GIT_SUCCESS) - return git__throw(GIT_EOSERR, "Failed to find pack entry. Packfile doesn't exist on disk"); - - e->offset = offset; - e->p = p; - - git_oid_cpy(&e->sha1, &found_oid); - return GIT_SUCCESS; -} - -static int pack_entry_find(struct pack_entry *e, struct pack_backend *backend, const git_oid *oid) +static int pack_entry_find(struct git_pack_entry *e, struct pack_backend *backend, const git_oid *oid) { int error; size_t i; @@ -1094,17 +288,17 @@ static int pack_entry_find(struct pack_entry *e, struct pack_backend *backend, c return git__rethrow(error, "Failed to find pack entry"); if (backend->last_found && - pack_entry_find1(e, backend->last_found, oid, GIT_OID_HEXSZ) == GIT_SUCCESS) + git_pack_entry_find(e, backend->last_found, oid, GIT_OID_HEXSZ) == GIT_SUCCESS) return GIT_SUCCESS; for (i = 0; i < backend->packs.length; ++i) { - struct pack_file *p; + struct git_pack_file *p; p = git_vector_get(&backend->packs, i); if (p == backend->last_found) continue; - if (pack_entry_find1(e, p, oid, GIT_OID_HEXSZ) == GIT_SUCCESS) { + if (git_pack_entry_find(e, p, oid, GIT_OID_HEXSZ) == GIT_SUCCESS) { backend->last_found = p; return GIT_SUCCESS; } @@ -1114,7 +308,7 @@ static int pack_entry_find(struct pack_entry *e, struct pack_backend *backend, c } static int pack_entry_find_prefix( - struct pack_entry *e, + struct git_pack_entry *e, struct pack_backend *backend, const git_oid *short_oid, unsigned int len) @@ -1127,7 +321,7 @@ static int pack_entry_find_prefix( return git__rethrow(error, "Failed to find pack entry"); if (backend->last_found) { - error = pack_entry_find1(e, backend->last_found, short_oid, len); + error = git_pack_entry_find(e, backend->last_found, short_oid, len); if (error == GIT_EAMBIGUOUSOIDPREFIX) { return git__rethrow(error, "Failed to find pack entry. Ambiguous sha1 prefix"); } else if (error == GIT_SUCCESS) { @@ -1136,13 +330,13 @@ static int pack_entry_find_prefix( } for (i = 0; i < backend->packs.length; ++i) { - struct pack_file *p; + struct git_pack_file *p; p = git_vector_get(&backend->packs, i); if (p == backend->last_found) continue; - error = pack_entry_find1(e, p, short_oid, len); + error = git_pack_entry_find(e, p, short_oid, len); if (error == GIT_EAMBIGUOUSOIDPREFIX) { return git__rethrow(error, "Failed to find pack entry. Ambiguous sha1 prefix"); } else if (error == GIT_SUCCESS) { @@ -1164,274 +358,6 @@ static int pack_entry_find_prefix( } - - - - - - - - - - -/*********************************************************** - * - * PACKFILE ENTRY UNPACK INTERNALS - * - ***********************************************************/ - -static unsigned long packfile_unpack_header1( - size_t *sizep, - git_otype *type, - const unsigned char *buf, - unsigned long len) -{ - unsigned shift; - unsigned long size, c; - unsigned long used = 0; - - c = buf[used++]; - *type = (c >> 4) & 7; - size = c & 15; - shift = 4; - while (c & 0x80) { - if (len <= used || bitsizeof(long) <= shift) - return 0; - - c = buf[used++]; - size += (c & 0x7f) << shift; - shift += 7; - } - - *sizep = (size_t)size; - return used; -} - -static int packfile_unpack_header( - size_t *size_p, - git_otype *type_p, - struct pack_backend *backend, - struct pack_file *p, - struct pack_window **w_curs, - off_t *curpos) -{ - unsigned char *base; - unsigned int left; - unsigned long used; - - /* pack_window_open() assures us we have [base, base + 20) available - * as a range that we can look at at. (Its actually the hash - * size that is assured.) With our object header encoding - * the maximum deflated object size is 2^137, which is just - * insane, so we know won't exceed what we have been given. - */ - base = pack_window_open(backend, p, w_curs, *curpos, &left); - if (base == NULL) - return GIT_ENOMEM; - - used = packfile_unpack_header1(size_p, type_p, base, left); - - if (used == 0) - return git__throw(GIT_EOBJCORRUPTED, "Header length is zero"); - - *curpos += used; - return GIT_SUCCESS; -} - -static int packfile_unpack_compressed( - git_rawobj *obj, - struct pack_backend *backend, - struct pack_file *p, - struct pack_window **w_curs, - off_t curpos, - size_t size, - git_otype type) -{ - int st; - z_stream stream; - unsigned char *buffer, *in; - - buffer = git__malloc(size + 1); - memset(buffer, 0x0, size + 1); - - memset(&stream, 0, sizeof(stream)); - stream.next_out = buffer; - stream.avail_out = size + 1; - - st = inflateInit(&stream); - if (st != Z_OK) { - free(buffer); - return git__throw(GIT_EZLIB, "Error in zlib"); - } - - do { - in = pack_window_open(backend, p, w_curs, curpos, &stream.avail_in); - stream.next_in = in; - st = inflate(&stream, Z_FINISH); - - if (!stream.avail_out) - break; /* the payload is larger than it should be */ - - curpos += stream.next_in - in; - } while (st == Z_OK || st == Z_BUF_ERROR); - - inflateEnd(&stream); - - if ((st != Z_STREAM_END) || stream.total_out != size) { - free(buffer); - return git__throw(GIT_EZLIB, "Error in zlib"); - } - - obj->type = type; - obj->len = size; - obj->data = buffer; - return GIT_SUCCESS; -} - -static off_t get_delta_base( - struct pack_backend *backend, - struct pack_file *p, - struct pack_window **w_curs, - off_t *curpos, - git_otype type, - off_t delta_obj_offset) -{ - unsigned char *base_info = pack_window_open(backend, p, w_curs, *curpos, NULL); - off_t base_offset; - git_oid unused; - - /* pack_window_open() assured us we have [base_info, base_info + 20) - * as a range that we can look at without walking off the - * end of the mapped window. Its actually the hash size - * that is assured. An OFS_DELTA longer than the hash size - * is stupid, as then a REF_DELTA would be smaller to store. - */ - if (type == GIT_OBJ_OFS_DELTA) { - unsigned used = 0; - unsigned char c = base_info[used++]; - base_offset = c & 127; - while (c & 128) { - base_offset += 1; - if (!base_offset || MSB(base_offset, 7)) - return 0; /* overflow */ - c = base_info[used++]; - base_offset = (base_offset << 7) + (c & 127); - } - base_offset = delta_obj_offset - base_offset; - if (base_offset <= 0 || base_offset >= delta_obj_offset) - return 0; /* out of bound */ - *curpos += used; - } else if (type == GIT_OBJ_REF_DELTA) { - /* The base entry _must_ be in the same pack */ - if (pack_entry_find_offset(&base_offset, &unused, p, (git_oid *)base_info, GIT_OID_HEXSZ) < GIT_SUCCESS) - return git__throw(GIT_EPACKCORRUPTED, "Base entry delta is not in the same pack"); - *curpos += 20; - } else - return 0; - - return base_offset; -} - -static int packfile_unpack_delta( - git_rawobj *obj, - struct pack_backend *backend, - struct pack_file *p, - struct pack_window **w_curs, - off_t curpos, - size_t delta_size, - git_otype delta_type, - off_t obj_offset) -{ - off_t base_offset; - git_rawobj base, delta; - int error; - - base_offset = get_delta_base(backend, p, w_curs, &curpos, delta_type, obj_offset); - if (base_offset == 0) - return git__throw(GIT_EOBJCORRUPTED, "Delta offset is zero"); - - pack_window_close(w_curs); - error = packfile_unpack(&base, backend, p, base_offset); - - /* TODO: git.git tries to load the base from other packfiles - * or loose objects */ - if (error < GIT_SUCCESS) - return git__rethrow(error, "Corrupted delta"); - - error = packfile_unpack_compressed(&delta, backend, p, w_curs, curpos, delta_size, delta_type); - if (error < GIT_SUCCESS) { - free(base.data); - return git__rethrow(error, "Corrupted delta"); - } - - obj->type = base.type; - error = git__delta_apply(obj, - base.data, base.len, - delta.data, delta.len); - - free(base.data); - free(delta.data); - - /* TODO: we might want to cache this shit. eventually */ - //add_delta_base_cache(p, base_offset, base, base_size, *type); - return error; /* error set by git__delta_apply */ -} - -static int packfile_unpack( - git_rawobj *obj, - struct pack_backend *backend, - struct pack_file *p, - off_t obj_offset) -{ - struct pack_window *w_curs = NULL; - off_t curpos = obj_offset; - int error; - - size_t size = 0; - git_otype type; - - /* - * TODO: optionally check the CRC on the packfile - */ - - obj->data = NULL; - obj->len = 0; - obj->type = GIT_OBJ_BAD; - - error = packfile_unpack_header(&size, &type, backend, p, &w_curs, &curpos); - if (error < GIT_SUCCESS) - return git__rethrow(error, "Failed to unpack packfile"); - - switch (type) { - case GIT_OBJ_OFS_DELTA: - case GIT_OBJ_REF_DELTA: - error = packfile_unpack_delta( - obj, backend, p, &w_curs, curpos, - size, type, obj_offset); - break; - - case GIT_OBJ_COMMIT: - case GIT_OBJ_TREE: - case GIT_OBJ_BLOB: - case GIT_OBJ_TAG: - error = packfile_unpack_compressed( - obj, backend, p, &w_curs, curpos, - size, type); - break; - - default: - error = GIT_EOBJCORRUPTED; - break; - } - - pack_window_close(&w_curs); - return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to unpack packfile"); -} - - - - - /*********************************************************** * * PACKED BACKEND PUBLIC API @@ -1456,14 +382,14 @@ int pack_backend__read_header(git_rawobj *obj, git_odb_backend *backend, const g int pack_backend__read(void **buffer_p, size_t *len_p, git_otype *type_p, git_odb_backend *backend, const git_oid *oid) { - struct pack_entry e; + struct git_pack_entry e; git_rawobj raw; int error; if ((error = pack_entry_find(&e, (struct pack_backend *)backend, oid)) < GIT_SUCCESS) return git__rethrow(error, "Failed to read pack backend"); - if ((error = packfile_unpack(&raw, (struct pack_backend *)backend, e.p, e.offset)) < GIT_SUCCESS) + if ((error = git_packfile_unpack(&raw, e.p, &e.offset)) < GIT_SUCCESS) return git__rethrow(error, "Failed to read pack backend"); *buffer_p = raw.data; @@ -1493,14 +419,14 @@ int pack_backend__read_prefix( return error; } else { - struct pack_entry e; + struct git_pack_entry e; git_rawobj raw; int error; if ((error = pack_entry_find_prefix(&e, (struct pack_backend *)backend, short_oid, len)) < GIT_SUCCESS) return git__rethrow(error, "Failed to read pack backend"); - if ((error = packfile_unpack(&raw, (struct pack_backend *)backend, e.p, e.offset)) < GIT_SUCCESS) + if ((error = git_packfile_unpack(&raw, e.p, &e.offset)) < GIT_SUCCESS) return git__rethrow(error, "Failed to read pack backend"); *buffer_p = raw.data; @@ -1514,7 +440,7 @@ int pack_backend__read_prefix( int pack_backend__exists(git_odb_backend *backend, const git_oid *oid) { - struct pack_entry e; + struct git_pack_entry e; return pack_entry_find(&e, (struct pack_backend *)backend, oid) == GIT_SUCCESS; } @@ -1528,8 +454,8 @@ void pack_backend__free(git_odb_backend *_backend) backend = (struct pack_backend *)_backend; for (i = 0; i < backend->packs.length; ++i) { - struct pack_file *p = git_vector_get(&backend->packs, i); - packfile_free(backend, p); + struct git_pack_file *p = git_vector_get(&backend->packs, i); + packfile_free(p); } git_vector_free(&backend->packs); @@ -1551,9 +477,6 @@ int git_odb_backend_pack(git_odb_backend **backend_out, const char *objects_dir) return GIT_ENOMEM; } - backend->window_size = DEFAULT_WINDOW_SIZE; - backend->mapped_limit = DEFAULT_MAPPED_LIMIT; - git_path_join(path, objects_dir, "pack"); if (git_futils_isdir(path) == GIT_SUCCESS) { backend->pack_folder = git__strdup(path); diff --git a/src/pack.c b/src/pack.c new file mode 100644 index 000000000..f7bad2f80 --- /dev/null +++ b/src/pack.c @@ -0,0 +1,790 @@ +/* + * This file is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, version 2, + * as published by the Free Software Foundation. + * + * In addition to the permissions in the GNU General Public License, + * the authors give you unlimited permission to link the compiled + * version of this file into combinations with other programs, + * and to distribute those combinations without any restriction + * coming from the use of this file. (The General Public License + * restrictions do apply in other respects; for example, they cover + * modification of the file, and distribution when not linked into + * a combined executable.) + * + * This file is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; see the file COPYING. If not, write to + * the Free Software Foundation, 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "mwindow.h" +#include "odb.h" +#include "pack.h" +#include "delta-apply.h" +#include "sha1_lookup.h" + +#include "git2/oid.h" +#include "git2/zlib.h" + +static int packfile_open(struct git_pack_file *p); +static off_t nth_packed_object_offset(const struct git_pack_file *p, uint32_t n); +int packfile_unpack_compressed( + git_rawobj *obj, + struct git_pack_file *p, + git_mwindow **w_curs, + off_t *curpos, + size_t size, + git_otype type); + +/* Can find the offset of an object given + * a prefix of an identifier. + * Throws GIT_EAMBIGUOUSOIDPREFIX if short oid + * is ambiguous within the pack. + * This method assumes that len is between + * GIT_OID_MINPREFIXLEN and GIT_OID_HEXSZ. + */ +static int pack_entry_find_offset( + off_t *offset_out, + git_oid *found_oid, + struct git_pack_file *p, + const git_oid *short_oid, + unsigned int len); + +/*********************************************************** + * + * PACK INDEX METHODS + * + ***********************************************************/ + +static void pack_index_free(struct git_pack_file *p) +{ + if (p->index_map.data) { + git_futils_mmap_free(&p->index_map); + p->index_map.data = NULL; + } +} + +static int pack_index_check(const char *path, struct git_pack_file *p) +{ + struct git_pack_idx_header *hdr; + uint32_t version, nr, i, *index; + + void *idx_map; + size_t idx_size; + + struct stat st; + + /* TODO: properly open the file without access time */ + git_file fd = p_open(path, O_RDONLY /*| O_NOATIME */); + + int error; + + if (fd < 0) + return git__throw(GIT_EOSERR, "Failed to check index. File missing or corrupted"); + + if (p_fstat(fd, &st) < GIT_SUCCESS) { + p_close(fd); + return git__throw(GIT_EOSERR, "Failed to check index. File appears to be corrupted"); + } + + if (!git__is_sizet(st.st_size)) + return GIT_ENOMEM; + + idx_size = (size_t)st.st_size; + + if (idx_size < 4 * 256 + 20 + 20) { + p_close(fd); + return git__throw(GIT_EOBJCORRUPTED, "Failed to check index. Object is corrupted"); + } + + error = git_futils_mmap_ro(&p->index_map, fd, 0, idx_size); + p_close(fd); + + if (error < GIT_SUCCESS) + return git__rethrow(error, "Failed to check index"); + + hdr = idx_map = p->index_map.data; + + if (hdr->idx_signature == htonl(PACK_IDX_SIGNATURE)) { + version = ntohl(hdr->idx_version); + + if (version < 2 || version > 2) { + git_futils_mmap_free(&p->index_map); + return git__throw(GIT_EOBJCORRUPTED, "Failed to check index. Unsupported index version"); + } + + } else + version = 1; + + nr = 0; + index = idx_map; + + if (version > 1) + index += 2; /* skip index header */ + + for (i = 0; i < 256; i++) { + uint32_t n = ntohl(index[i]); + if (n < nr) { + git_futils_mmap_free(&p->index_map); + return git__throw(GIT_EOBJCORRUPTED, "Failed to check index. Index is non-monotonic"); + } + nr = n; + } + + if (version == 1) { + /* + * Total size: + * - 256 index entries 4 bytes each + * - 24-byte entries * nr (20-byte sha1 + 4-byte offset) + * - 20-byte SHA1 of the packfile + * - 20-byte SHA1 file checksum + */ + if (idx_size != 4*256 + nr * 24 + 20 + 20) { + git_futils_mmap_free(&p->index_map); + return git__throw(GIT_EOBJCORRUPTED, "Failed to check index. Object is corrupted"); + } + } else if (version == 2) { + /* + * Minimum size: + * - 8 bytes of header + * - 256 index entries 4 bytes each + * - 20-byte sha1 entry * nr + * - 4-byte crc entry * nr + * - 4-byte offset entry * nr + * - 20-byte SHA1 of the packfile + * - 20-byte SHA1 file checksum + * And after the 4-byte offset table might be a + * variable sized table containing 8-byte entries + * for offsets larger than 2^31. + */ + unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20; + unsigned long max_size = min_size; + + if (nr) + max_size += (nr - 1)*8; + + if (idx_size < min_size || idx_size > max_size) { + git_futils_mmap_free(&p->index_map); + return git__throw(GIT_EOBJCORRUPTED, "Failed to check index. Wrong index size"); + } + + /* Make sure that off_t is big enough to access the whole pack... + * Is this an issue in libgit2? It shouldn't. */ + if (idx_size != min_size && (sizeof(off_t) <= 4)) { + git_futils_mmap_free(&p->index_map); + return git__throw(GIT_EOSERR, "Failed to check index. off_t not big enough to access the whole pack"); + } + } + + p->index_version = version; + p->num_objects = nr; + return GIT_SUCCESS; +} + +static int pack_index_open(struct git_pack_file *p) +{ + char *idx_name; + int error; + + if (p->index_map.data) + return GIT_SUCCESS; + + idx_name = git__strdup(p->pack_name); + strcpy(idx_name + strlen(idx_name) - STRLEN(".pack"), ".idx"); + + error = pack_index_check(idx_name, p); + free(idx_name); + + return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to open index"); +} + +static unsigned char *pack_window_open( + struct git_pack_file *p, + git_mwindow **w_cursor, + off_t offset, + unsigned int *left) +{ + if (p->mwf.fd == -1 && packfile_open(p) < GIT_SUCCESS) + return NULL; + + /* Since packfiles end in a hash of their content and it's + * pointless to ask for an offset into the middle of that + * hash, and the pack_window_contains function above wouldn't match + * don't allow an offset too close to the end of the file. + */ + if (offset > (p->mwf.size - 20)) + return NULL; + + return git_mwindow_open(&p->mwf, w_cursor, offset, 20, left); + } + +static unsigned long packfile_unpack_header1( + size_t *sizep, + git_otype *type, + const unsigned char *buf, + unsigned long len) +{ + unsigned shift; + unsigned long size, c; + unsigned long used = 0; + + c = buf[used++]; + *type = (c >> 4) & 7; + size = c & 15; + shift = 4; + while (c & 0x80) { + if (len <= used || bitsizeof(long) <= shift) + return 0; + + c = buf[used++]; + size += (c & 0x7f) << shift; + shift += 7; + } + + *sizep = (size_t)size; + return used; +} + +int git_packfile_unpack_header( + size_t *size_p, + git_otype *type_p, + git_mwindow_file *mwf, + git_mwindow **w_curs, + off_t *curpos) +{ + unsigned char *base; + unsigned int left; + unsigned long used; + + /* pack_window_open() assures us we have [base, base + 20) available + * as a range that we can look at at. (Its actually the hash + * size that is assured.) With our object header encoding + * the maximum deflated object size is 2^137, which is just + * insane, so we know won't exceed what we have been given. + */ +// base = pack_window_open(p, w_curs, *curpos, &left); + base = git_mwindow_open(mwf, w_curs, *curpos, 20, &left); + if (base == NULL) + return GIT_ENOMEM; + + used = packfile_unpack_header1(size_p, type_p, base, left); + + if (used == 0) + return git__throw(GIT_EOBJCORRUPTED, "Header length is zero"); + + *curpos += used; + return GIT_SUCCESS; +} + +static int packfile_unpack_delta( + git_rawobj *obj, + struct git_pack_file *p, + git_mwindow **w_curs, + off_t *curpos, + size_t delta_size, + git_otype delta_type, + off_t obj_offset) +{ + off_t base_offset; + git_rawobj base, delta; + int error; + + base_offset = get_delta_base(p, w_curs, curpos, delta_type, obj_offset); + if (base_offset == 0) + return git__throw(GIT_EOBJCORRUPTED, "Delta offset is zero"); + + git_mwindow_close(w_curs); + error = git_packfile_unpack(&base, p, &base_offset); + + /* + * TODO: git.git tries to load the base from other packfiles + * or loose objects. + * + * We'll need to do this in order to support thin packs. + */ + if (error < GIT_SUCCESS) + return git__rethrow(error, "Corrupted delta"); + + error = packfile_unpack_compressed(&delta, p, w_curs, curpos, delta_size, delta_type); + if (error < GIT_SUCCESS) { + free(base.data); + return git__rethrow(error, "Corrupted delta"); + } + + obj->type = base.type; + error = git__delta_apply(obj, + base.data, base.len, + delta.data, delta.len); + + free(base.data); + free(delta.data); + + /* TODO: we might want to cache this shit. eventually */ + //add_delta_base_cache(p, base_offset, base, base_size, *type); + return error; /* error set by git__delta_apply */ +} + +int git_packfile_unpack( + git_rawobj *obj, + struct git_pack_file *p, + off_t *obj_offset) +{ + git_mwindow *w_curs = NULL; + off_t curpos = *obj_offset; + int error; + + size_t size = 0; + git_otype type; + + /* + * TODO: optionally check the CRC on the packfile + */ + + obj->data = NULL; + obj->len = 0; + obj->type = GIT_OBJ_BAD; + + error = git_packfile_unpack_header(&size, &type, &p->mwf, &w_curs, &curpos); + if (error < GIT_SUCCESS) + return git__rethrow(error, "Failed to unpack packfile"); + + switch (type) { + case GIT_OBJ_OFS_DELTA: + case GIT_OBJ_REF_DELTA: + error = packfile_unpack_delta( + obj, p, &w_curs, &curpos, + size, type, *obj_offset); + break; + + case GIT_OBJ_COMMIT: + case GIT_OBJ_TREE: + case GIT_OBJ_BLOB: + case GIT_OBJ_TAG: + error = packfile_unpack_compressed( + obj, p, &w_curs, &curpos, + size, type); + break; + + default: + error = GIT_EOBJCORRUPTED; + break; + } + + git_mwindow_close(&w_curs); + + if (error < GIT_SUCCESS) + return git__rethrow(error, "Failed to unpack object"); + + *obj_offset = curpos; + return GIT_SUCCESS; +} + +int packfile_unpack_compressed( + git_rawobj *obj, + struct git_pack_file *p, + git_mwindow **w_curs, + off_t *curpos, + size_t size, + git_otype type) +{ + int st; + z_stream stream; + unsigned char *buffer, *in; + + buffer = git__malloc(size + 1); + memset(buffer, 0x0, size + 1); + + memset(&stream, 0, sizeof(stream)); + stream.next_out = buffer; + stream.avail_out = size + 1; + + st = inflateInit(&stream); + if (st != Z_OK) { + free(buffer); + return git__throw(GIT_EZLIB, "Error in zlib"); + } + + do { + in = pack_window_open(p, w_curs, *curpos, &stream.avail_in); + stream.next_in = in; + st = inflate(&stream, Z_FINISH); + + if (!stream.avail_out) + break; /* the payload is larger than it should be */ + + *curpos += stream.next_in - in; + } while (st == Z_OK || st == Z_BUF_ERROR); + + inflateEnd(&stream); + + if ((st != Z_STREAM_END) || stream.total_out != size) { + free(buffer); + return git__throw(GIT_EZLIB, "Error in zlib"); + } + + obj->type = type; + obj->len = size; + obj->data = buffer; + return GIT_SUCCESS; +} + +/* + * curpos is where the data starts, delta_obj_offset is the where the + * header starts + */ +off_t get_delta_base( + struct git_pack_file *p, + git_mwindow **w_curs, + off_t *curpos, + git_otype type, + off_t delta_obj_offset) +{ + unsigned char *base_info = pack_window_open(p, w_curs, *curpos, NULL); + off_t base_offset; + git_oid unused; + + /* pack_window_open() assured us we have [base_info, base_info + 20) + * as a range that we can look at without walking off the + * end of the mapped window. Its actually the hash size + * that is assured. An OFS_DELTA longer than the hash size + * is stupid, as then a REF_DELTA would be smaller to store. + */ + if (type == GIT_OBJ_OFS_DELTA) { + unsigned used = 0; + unsigned char c = base_info[used++]; + base_offset = c & 127; + while (c & 128) { + base_offset += 1; + if (!base_offset || MSB(base_offset, 7)) + return 0; /* overflow */ + c = base_info[used++]; + base_offset = (base_offset << 7) + (c & 127); + } + base_offset = delta_obj_offset - base_offset; + if (base_offset <= 0 || base_offset >= delta_obj_offset) + return 0; /* out of bound */ + *curpos += used; + } else if (type == GIT_OBJ_REF_DELTA) { + /* The base entry _must_ be in the same pack */ + if (pack_entry_find_offset(&base_offset, &unused, p, (git_oid *)base_info, GIT_OID_HEXSZ) < GIT_SUCCESS) + return git__throw(GIT_EPACKCORRUPTED, "Base entry delta is not in the same pack"); + *curpos += 20; + } else + return 0; + + return base_offset; +} + +/*********************************************************** + * + * PACKFILE METHODS + * + ***********************************************************/ + +static struct git_pack_file *packfile_alloc(int extra) +{ + struct git_pack_file *p = git__malloc(sizeof(*p) + extra); + memset(p, 0, sizeof(*p)); + p->mwf.fd = -1; + return p; +} + + +void packfile_free(struct git_pack_file *p) +{ + assert(p); + + /* clear_delta_base_cache(); */ + git_mwindow_free_all(&p->mwf); + + if (p->mwf.fd != -1) + p_close(p->mwf.fd); + + pack_index_free(p); + + free(p->bad_object_sha1); + free(p); +} + +static int packfile_open(struct git_pack_file *p) +{ + struct stat st; + struct git_pack_header hdr; + git_oid sha1; + unsigned char *idx_sha1; + + if (!p->index_map.data && pack_index_open(p) < GIT_SUCCESS) + return git__throw(GIT_ENOTFOUND, "Failed to open packfile. File not found"); + + /* TODO: open with noatime */ + p->mwf.fd = p_open(p->pack_name, O_RDONLY); + if (p->mwf.fd < 0 || p_fstat(p->mwf.fd, &st) < GIT_SUCCESS) + return git__throw(GIT_EOSERR, "Failed to open packfile. File appears to be corrupted"); + + if (git_mwindow_file_register(&p->mwf) < GIT_SUCCESS) { + p_close(p->mwf.fd); + return git__throw(GIT_ERROR, "Failed to register packfile windows"); + } + + /* If we created the struct before we had the pack we lack size. */ + if (!p->mwf.size) { + if (!S_ISREG(st.st_mode)) + goto cleanup; + p->mwf.size = (off_t)st.st_size; + } else if (p->mwf.size != st.st_size) + goto cleanup; + +#if 0 + /* We leave these file descriptors open with sliding mmap; + * there is no point keeping them open across exec(), though. + */ + fd_flag = fcntl(p->mwf.fd, F_GETFD, 0); + if (fd_flag < 0) + return error("cannot determine file descriptor flags"); + + fd_flag |= FD_CLOEXEC; + if (fcntl(p->pack_fd, F_SETFD, fd_flag) == -1) + return GIT_EOSERR; +#endif + + /* Verify we recognize this pack file format. */ + if (p_read(p->mwf.fd, &hdr, sizeof(hdr)) < GIT_SUCCESS) + goto cleanup; + + if (hdr.hdr_signature != htonl(PACK_SIGNATURE)) + goto cleanup; + + if (!pack_version_ok(hdr.hdr_version)) + goto cleanup; + + /* Verify the pack matches its index. */ + if (p->num_objects != ntohl(hdr.hdr_entries)) + goto cleanup; + + if (p_lseek(p->mwf.fd, p->mwf.size - GIT_OID_RAWSZ, SEEK_SET) == -1) + goto cleanup; + + if (p_read(p->mwf.fd, sha1.id, GIT_OID_RAWSZ) < GIT_SUCCESS) + goto cleanup; + + idx_sha1 = ((unsigned char *)p->index_map.data) + p->index_map.len - 40; + + if (git_oid_cmp(&sha1, (git_oid *)idx_sha1) != 0) + goto cleanup; + + return GIT_SUCCESS; + +cleanup: + p_close(p->mwf.fd); + p->mwf.fd = -1; + return git__throw(GIT_EPACKCORRUPTED, "Failed to open packfile. Pack is corrupted"); +} + +int git_packfile_check(struct git_pack_file **pack_out, const char *path) +{ + struct stat st; + struct git_pack_file *p; + size_t path_len; + + *pack_out = NULL; + path_len = strlen(path); + p = packfile_alloc(path_len + 2); + + /* + * Make sure a corresponding .pack file exists and that + * the index looks sane. + */ + path_len -= STRLEN(".idx"); + if (path_len < 1) { + free(p); + return git__throw(GIT_ENOTFOUND, "Failed to check packfile. Wrong path name"); + } + + memcpy(p->pack_name, path, path_len); + + strcpy(p->pack_name + path_len, ".keep"); + if (git_futils_exists(p->pack_name) == GIT_SUCCESS) + p->pack_keep = 1; + + strcpy(p->pack_name + path_len, ".pack"); + if (p_stat(p->pack_name, &st) < GIT_SUCCESS || !S_ISREG(st.st_mode)) { + free(p); + return git__throw(GIT_ENOTFOUND, "Failed to check packfile. File not found"); + } + + /* ok, it looks sane as far as we can check without + * actually mapping the pack file. + */ + p->mwf.size = (off_t)st.st_size; + p->pack_local = 1; + p->mtime = (git_time_t)st.st_mtime; + + /* see if we can parse the sha1 oid in the packfile name */ + if (path_len < 40 || + git_oid_fromstr(&p->sha1, path + path_len - GIT_OID_HEXSZ) < GIT_SUCCESS) + memset(&p->sha1, 0x0, GIT_OID_RAWSZ); + + *pack_out = p; + return GIT_SUCCESS; +} + +/*********************************************************** + * + * PACKFILE ENTRY SEARCH INTERNALS + * + ***********************************************************/ + +static off_t nth_packed_object_offset(const struct git_pack_file *p, uint32_t n) +{ + const unsigned char *index = p->index_map.data; + index += 4 * 256; + if (p->index_version == 1) { + return ntohl(*((uint32_t *)(index + 24 * n))); + } else { + uint32_t off; + index += 8 + p->num_objects * (20 + 4); + off = ntohl(*((uint32_t *)(index + 4 * n))); + if (!(off & 0x80000000)) + return off; + index += p->num_objects * 4 + (off & 0x7fffffff) * 8; + return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) | + ntohl(*((uint32_t *)(index + 4))); + } +} + +static int pack_entry_find_offset( + off_t *offset_out, + git_oid *found_oid, + struct git_pack_file *p, + const git_oid *short_oid, + unsigned int len) +{ + const uint32_t *level1_ofs = p->index_map.data; + const unsigned char *index = p->index_map.data; + unsigned hi, lo, stride; + int pos, found = 0; + const unsigned char *current = 0; + + *offset_out = 0; + + if (index == NULL) { + int error; + + if ((error = pack_index_open(p)) < GIT_SUCCESS) + return git__rethrow(error, "Failed to find offset for pack entry"); + + assert(p->index_map.data); + + index = p->index_map.data; + level1_ofs = p->index_map.data; + } + + if (p->index_version > 1) { + level1_ofs += 2; + index += 8; + } + + index += 4 * 256; + hi = ntohl(level1_ofs[(int)short_oid->id[0]]); + lo = ((short_oid->id[0] == 0x0) ? 0 : ntohl(level1_ofs[(int)short_oid->id[0] - 1])); + + if (p->index_version > 1) { + stride = 20; + } else { + stride = 24; + index += 4; + } + +#ifdef INDEX_DEBUG_LOOKUP + printf("%02x%02x%02x... lo %u hi %u nr %d\n", + short_oid->id[0], short_oid->id[1], short_oid->id[2], lo, hi, p->num_objects); +#endif + + /* Use git.git lookup code */ + pos = sha1_entry_pos(index, stride, 0, lo, hi, p->num_objects, short_oid->id); + + if (pos >= 0) { + /* An object matching exactly the oid was found */ + found = 1; + current = index + pos * stride; + } else { + /* No object was found */ + /* pos refers to the object with the "closest" oid to short_oid */ + pos = - 1 - pos; + if (pos < (int)p->num_objects) { + current = index + pos * stride; + + if (!git_oid_ncmp(short_oid, (const git_oid *)current, len)) { + found = 1; + } + } + } + + if (found && pos + 1 < (int)p->num_objects) { + /* Check for ambiguousity */ + const unsigned char *next = current + stride; + + if (!git_oid_ncmp(short_oid, (const git_oid *)next, len)) { + found = 2; + } + } + + if (!found) { + return git__throw(GIT_ENOTFOUND, "Failed to find offset for pack entry. Entry not found"); + } else if (found > 1) { + return git__throw(GIT_EAMBIGUOUSOIDPREFIX, "Failed to find offset for pack entry. Ambiguous sha1 prefix within pack"); + } else { + *offset_out = nth_packed_object_offset(p, pos); + git_oid_fromraw(found_oid, current); + +#ifdef INDEX_DEBUG_LOOKUP + unsigned char hex_sha1[GIT_OID_HEXSZ + 1]; + git_oid_fmt(hex_sha1, found_oid); + hex_sha1[GIT_OID_HEXSZ] = '\0'; + printf("found lo=%d %s\n", lo, hex_sha1); +#endif + return GIT_SUCCESS; + } +} + +int git_pack_entry_find( + struct git_pack_entry *e, + struct git_pack_file *p, + const git_oid *short_oid, + unsigned int len) +{ + off_t offset; + git_oid found_oid; + int error; + + assert(p); + + if (len == GIT_OID_HEXSZ && p->num_bad_objects) { + unsigned i; + for (i = 0; i < p->num_bad_objects; i++) + if (git_oid_cmp(short_oid, &p->bad_object_sha1[i]) == 0) + return git__throw(GIT_ERROR, "Failed to find pack entry. Bad object found"); + } + + error = pack_entry_find_offset(&offset, &found_oid, p, short_oid, len); + if (error < GIT_SUCCESS) + return git__rethrow(error, "Failed to find pack entry. Couldn't find offset"); + + /* we found a unique entry in the index; + * make sure the packfile backing the index + * still exists on disk */ + if (p->mwf.fd == -1 && packfile_open(p) < GIT_SUCCESS) + return git__throw(GIT_EOSERR, "Failed to find pack entry. Packfile doesn't exist on disk"); + + e->offset = offset; + e->p = p; + + git_oid_cpy(&e->sha1, &found_oid); + return GIT_SUCCESS; +} diff --git a/src/pack.h b/src/pack.h new file mode 100644 index 000000000..a7112a6aa --- /dev/null +++ b/src/pack.h @@ -0,0 +1,114 @@ +/* + * This file is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, version 2, + * as published by the Free Software Foundation. + * + * In addition to the permissions in the GNU General Public License, + * the authors give you unlimited permission to link the compiled + * version of this file into combinations with other programs, + * and to distribute those combinations without any restriction + * coming from the use of this file. (The General Public License + * restrictions do apply in other respects; for example, they cover + * modification of the file, and distribution when not linked into + * a combined executable.) + * + * This file is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; see the file COPYING. If not, write to + * the Free Software Foundation, 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef INCLUDE_pack_h__ +#define INCLUDE_pack_h__ + +#include "git2/oid.h" + +#include "common.h" +#include "map.h" +#include "mwindow.h" +#include "odb.h" + +#define PACK_SIGNATURE 0x5041434b /* "PACK" */ +#define PACK_VERSION 2 +#define pack_version_ok(v) ((v) == htonl(2) || (v) == htonl(3)) +struct git_pack_header { + uint32_t hdr_signature; + uint32_t hdr_version; + uint32_t hdr_entries; +}; + +/* + * The first four bytes of index formats later than version 1 should + * start with this signature, as all older git binaries would find this + * value illegal and abort reading the file. + * + * This is the case because the number of objects in a packfile + * cannot exceed 1,431,660,000 as every object would need at least + * 3 bytes of data and the overall packfile cannot exceed 4 GiB with + * version 1 of the index file due to the offsets limited to 32 bits. + * Clearly the signature exceeds this maximum. + * + * Very old git binaries will also compare the first 4 bytes to the + * next 4 bytes in the index and abort with a "non-monotonic index" + * error if the second 4 byte word is smaller than the first 4 + * byte word. This would be true in the proposed future index + * format as idx_signature would be greater than idx_version. + */ + +#define PACK_IDX_SIGNATURE 0xff744f63 /* "\377tOc" */ + +struct git_pack_idx_header { + uint32_t idx_signature; + uint32_t idx_version; +}; + +struct git_pack_file { + git_mwindow_file mwf; + git_map index_map; + + uint32_t num_objects; + uint32_t num_bad_objects; + git_oid *bad_object_sha1; /* array of git_oid */ + + int index_version; + git_time_t mtime; + unsigned pack_local:1, pack_keep:1; + git_oid sha1; + + /* something like ".git/objects/pack/xxxxx.pack" */ + char pack_name[GIT_FLEX_ARRAY]; /* more */ +}; + +struct git_pack_entry { + off_t offset; + git_oid sha1; + struct git_pack_file *p; +}; + +int git_packfile_unpack_header( + size_t *size_p, + git_otype *type_p, + git_mwindow_file *mwf, + git_mwindow **w_curs, + off_t *curpos); + +int git_packfile_unpack(git_rawobj *obj, struct git_pack_file *p, off_t *obj_offset); + +off_t get_delta_base(struct git_pack_file *p, git_mwindow **w_curs, + off_t *curpos, git_otype type, + off_t delta_obj_offset); + +void packfile_free(struct git_pack_file *p); +int git_packfile_check(struct git_pack_file **pack_out, const char *path); +int git_pack_entry_find( + struct git_pack_entry *e, + struct git_pack_file *p, + const git_oid *short_oid, + unsigned int len); + +#endif diff --git a/src/vector.h b/src/vector.h index 76778ba4e..c43a7ce07 100644 --- a/src/vector.h +++ b/src/vector.h @@ -30,6 +30,9 @@ GIT_INLINE(void *) git_vector_get(git_vector *v, unsigned int position) return (position < v->length) ? v->contents[position] : NULL; } +#define git_vector_foreach(v, iter, elem) \ + for ((iter) = 0; (iter) < (v)->length && ((elem) = (v)->contents[(iter)], 1); (iter)++ ) + int git_vector_insert(git_vector *v, void *element); int git_vector_remove(git_vector *v, unsigned int idx); void git_vector_uniq(git_vector *v);