diff --git a/src/odb.c b/src/odb.c index 627513e78..be896cc57 100644 --- a/src/odb.c +++ b/src/odb.c @@ -29,6 +29,7 @@ #include "fileops.h" #include "hash.h" #include "odb.h" +#include "delta-apply.h" #define GIT_PACK_NAME_MAX (5 + 40 + 1) @@ -74,6 +75,9 @@ struct git_pack { /** File descriptor for the .pack file. */ git_file pack_fd; + /** Memory map of the pack's contents */ + git_map pack_map; + /** The size of the .pack file. */ off_t pack_size; @@ -437,7 +441,7 @@ static int inflate_buffer(void *in, size_t inlen, void *out, size_t outlen) inflateEnd(&zs); - if ((status != Z_STREAM_END) || (zs.avail_in != 0)) + if ((status != Z_STREAM_END) /*|| (zs.avail_in != 0) */) return GIT_ERROR; if (zs.total_out != outlen) @@ -1160,23 +1164,26 @@ static int open_pack(git_pack *p) if (pack_openidx(p)) return GIT_ERROR; - if ((p->pack_fd = gitfo_open(pb, O_RDONLY)) < 0) { - p->pack_fd = -1; - pack_decidx(p); - return GIT_ERROR; - } + if ((p->pack_fd = gitfo_open(pb, O_RDONLY)) < 0) + goto error_cleanup; if (gitfo_fstat(p->pack_fd, &sb) || !S_ISREG(sb.st_mode) || p->pack_size != sb.st_size - || check_pack_hdr(p) || check_pack_sha1(p)) { - gitfo_close(p->pack_fd); - p->pack_fd = -1; - pack_decidx(p); - return GIT_ERROR; - } + || check_pack_hdr(p) || check_pack_sha1(p)) + goto error_cleanup; + + if (!git__is_sizet(p->pack_size) || + gitfo_map_ro(&p->pack_map, p->pack_fd, 0, (size_t)p->pack_size) < 0) + goto error_cleanup; pack_decidx(p); return GIT_SUCCESS; + +error_cleanup: + gitfo_close(p->pack_fd); + p->pack_fd = -1; + pack_decidx(p); + return GIT_ERROR; } static void pack_dec(git_pack *p) @@ -1194,8 +1201,10 @@ static void pack_dec(git_pack *p) free(p->im_fanout); free(p->im_off_idx); free(p->im_off_next); - if (p->pack_fd != -1) + if (p->pack_fd != -1) { gitfo_close(p->pack_fd); + gitfo_free_map(&p->pack_map); + } } gitlck_free(&p->lock); @@ -1475,14 +1484,141 @@ int git_odb__read_loose(git_obj *out, git_odb *db, const git_oid *id) return GIT_SUCCESS; } +static int unpack_object(git_obj *out, git_pack *p, index_entry *e); + +static int unpack_object_delta(git_obj *out, git_pack *p, + index_entry *base_entry, + uint8_t *delta_buffer, + size_t delta_deflated_size, + size_t delta_inflated_size) +{ + int res = 0; + uint8_t *delta = NULL; + git_obj base_obj; + + base_obj.data = NULL; + base_obj.type = GIT_OBJ_BAD; + base_obj.len = 0; + + if ((res = unpack_object(&base_obj, p, base_entry)) < 0) + goto cleanup; + + delta = git__malloc(delta_inflated_size + 1); + + if ((res = inflate_buffer(delta_buffer, delta_deflated_size, + delta, delta_inflated_size)) < 0) + goto cleanup; + + res = git__delta_apply(out, base_obj.data, base_obj.len, delta, delta_inflated_size); + + out->type = base_obj.type; + +cleanup: + free(delta); + git_obj_close(&base_obj); + return res; +} + static int unpack_object(git_obj *out, git_pack *p, index_entry *e) { - assert(out && p && e); + git_otype object_type; + size_t inflated_size, deflated_size, shift; + uint8_t *buffer, byte; + + assert(out && p && e && git__is_sizet(e->size)); if (open_pack(p)) return GIT_ERROR; - /* TODO - actually unpack the data! */ + buffer = (uint8_t *)p->pack_map.data + e->offset; + deflated_size = (size_t)e->size; + + if (deflated_size == 0) + deflated_size = p->pack_size - e->offset; + + byte = *buffer++ & 0xFF; + deflated_size--; + object_type = (byte >> 4) & 0x7; + inflated_size = byte & 0xF; + shift = 4; + + while (byte & 0x80) { + byte = *buffer++ & 0xFF; + deflated_size--; + inflated_size += (byte & 0x7F) << shift; + shift += 7; + } + + switch (object_type) { + case GIT_OBJ_COMMIT: + case GIT_OBJ_TREE: + case GIT_OBJ_BLOB: + case GIT_OBJ_TAG: { + + /* Handle a normal zlib stream */ + out->len = inflated_size; + out->type = object_type; + out->data = git__malloc(inflated_size + 1); + + if (inflate_buffer(buffer, deflated_size, out->data, out->len) < 0) { + free(out->data); + out->data = NULL; + return GIT_ERROR; + } + + return GIT_SUCCESS; + } + + case GIT_OBJ_OFS_DELTA: { + + off_t delta_offset; + index_entry entry; + + byte = *buffer++ & 0xFF; + delta_offset = byte & 0x7F; + + while (byte & 0x80) { + delta_offset += 1; + byte = *buffer++ & 0xFF; + delta_offset <<= 7; + delta_offset += (byte & 0x7F); + } + + entry.n = 0; + entry.oid = NULL; + entry.offset = e->offset - delta_offset; + entry.size = 0; + + if (unpack_object_delta(out, p, &entry, + buffer, deflated_size, inflated_size) < 0) + return GIT_ERROR; + + return GIT_SUCCESS; + } + + case GIT_OBJ_REF_DELTA: { + + git_oid base_id; + uint32_t n; + index_entry entry; + int res = GIT_ERROR; + + git_oid_mkraw(&base_id, buffer); + + if (!p->idx_search(&n, p, &base_id) && + !p->idx_get(&entry, p, n)) { + + /* FIXME: deflated_size - 20 ? */ + res = unpack_object_delta(out, p, &entry, + buffer + GIT_OID_RAWSZ, deflated_size, inflated_size); + } + + return res; + } + + default: + return GIT_EOBJCORRUPTED; + } return GIT_SUCCESS; } @@ -1497,15 +1633,15 @@ static int read_packed(git_obj *out, git_pack *p, const git_oid *id) if (pack_openidx(p)) return GIT_ERROR; + res = p->idx_search(&n, p, id); if (!res) res = p->idx_get(&e, p, n); - pack_decidx(p); - if (!res) { - /* TODO unpack object */ + if (!res) res = unpack_object(out, p, &e); - } + + pack_decidx(p); return res; } diff --git a/tests/resources/pack-odb/info/packs b/tests/resources/pack-odb/info/packs new file mode 100644 index 000000000..895b1c3b3 Binary files /dev/null and b/tests/resources/pack-odb/info/packs differ diff --git a/tests/resources/pack-odb/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx b/tests/resources/pack-odb/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx new file mode 100644 index 000000000..5068f2818 Binary files /dev/null and b/tests/resources/pack-odb/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx differ diff --git a/tests/resources/pack-odb/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.pack b/tests/resources/pack-odb/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.pack new file mode 100644 index 000000000..a6a1f3020 Binary files /dev/null and b/tests/resources/pack-odb/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.pack differ diff --git a/tests/resources/pack-odb/pack/pack-d85f5d483273108c9d8dd0e4728ccf0b2982423a.idx b/tests/resources/pack-odb/pack/pack-d85f5d483273108c9d8dd0e4728ccf0b2982423a.idx new file mode 100644 index 000000000..555cfa977 Binary files /dev/null and b/tests/resources/pack-odb/pack/pack-d85f5d483273108c9d8dd0e4728ccf0b2982423a.idx differ diff --git a/tests/resources/pack-odb/pack/pack-d85f5d483273108c9d8dd0e4728ccf0b2982423a.pack b/tests/resources/pack-odb/pack/pack-d85f5d483273108c9d8dd0e4728ccf0b2982423a.pack new file mode 100644 index 000000000..4d539ed0a Binary files /dev/null and b/tests/resources/pack-odb/pack/pack-d85f5d483273108c9d8dd0e4728ccf0b2982423a.pack differ diff --git a/tests/t0204-readpack.c b/tests/t0204-readpack.c new file mode 100644 index 000000000..49ae9af1b --- /dev/null +++ b/tests/t0204-readpack.c @@ -0,0 +1,154 @@ +#include "test_lib.h" +#include "test_helpers.h" +#include + +#define ODB_FOLDER "../resources/pack-odb" + +static const char *packed_objects[] = { + "0266163a49e280c4f5ed1e08facd36a2bd716bcf", + "53fc32d17276939fc79ed05badaef2db09990016", + "6336846bd5c88d32f93ae57d846683e61ab5c530", + "6dcf9bf7541ee10456529833502442f385010c3d", + "bed08a0b30b72a9d4aed7f1af8c8ca124e8d64b9", + "e90810b8df3e80c413d903f631643c716887138d", + "fc3c3a2083e9f6f89e6bd53e9420e70d1e357c9b", + "fc58168adf502d0c0ef614c3111a7038fc8c09c8", + "fd0ec0333948dfe23265ac46be0205a436a8c3a5", + "fd8430bc864cfcd5f10e5590f8a447e01b942bfe", + "fd899f45951c15c1c5f7c34b1c864e91bd6556c6", + "fda23b974899e7e1f938619099280bfda13bdca9", + "fdbec189efb657c8325962b494875987881a356b", + "fe1ca6bd22b5d8353ce6c2f3aba80805c438a7a5", + "fe3a6a42c87ff1239370c741a265f3997add87c1", + "deb106bfd2d36ecf9f0079224c12022201a39ad1", + "dec93efc79e60f2680de3e666755d335967eec30", + "def425bf8568b9c1e20879bf5be6f9c52b7361c4", + "df48000ac4f48570054e3a71a81916357997b680", + "dfae6ed8f6dd8acc3b40a31811ea316239223559", + "dff79e27d3d2cdc09790ded80fe2ea8ff5d61034", + "e00e46abe4c542e17c8bc83d72cf5be8018d7b0e", + "e01b107b4f77f8f98645adac0206a504f2d29d7c", + "e032d863f512c47b479bd984f8b6c8061f66b7d4", + "e044baa468a1c74f9f9da36805445f6888358b49", + "e04529998989ba8ae3419538dd57969af819b241", + "e0637ddfbea67c8d7f557c709e095af8906e9176", + "e0743ad4031231e71700abdc6fdbe94f189d20e5", + "cf33ac7a3d8b2b8f6bb266518aadbf59de397608", + "cf5f7235b9c9689b133f6ea12015720b411329bd", + "cf6cccf1297284833a9a03138a1f5738fa1c6c94", + "cf7992bde17ce7a79cab5f0c1fcbe8a0108721ed", + "cfe3a027ab12506d4144ee8a35669ae8fc4b7ab1", + "cfe96f31dfad7bab49977aa1df7302f7fafcb025", + "cff54d138945ef4de384e9d2759291d0c13ea90a", + "d01f7573ac34c2f502bd1cf18cde73480c741151", + "d03f567593f346a1ca96a57f8191def098d126e3", + "d047b47aadf88501238f36f5c17dd0a50dc62087", + "d0a0d63086fae3b0682af7261df21f7d0f7f066d", + "d0a44bd6ed0be21b725a96c0891bbc79bc1a540c", + "d0d7e736e536a41bcb885005f8bf258c61cad682", + "d0e7959d4b95ffec6198df6f5a7ae259b23a5f50", + "bf2fe2acca17d13356ce802ba9dc8343f710dfb7", + "bf55f407d6d9418e51f42ea7a3a6aadf17388349", + "bf92206f8b633b88a66dca4a911777630b06fbac", + "bfaf8c42eb8842abe206179fee864cfba87e3ca9", + "bfe05675d4e8f6b59d50932add8790f1a06b10ee", + "bff8618112330763327cfa6ce6e914db84f51ddf", + "bff873e9853ed99fed52c25f7ad29f78b27dcec2", + "c01c3fae7251098d7af1b459bcd0786e81d4616d", + "c0220fca67f48b8a5d4163d53b1486224be3a198", + "c02d0b160b82ee72469c269f13de4c26a7ea09cb", + "c059510ad1b45ab58390e042d7dee1ac46703854", + "c07204a1897aeeaa3c248d29dbfa9b033baf9755", + "c073337a4dd7276931b4b3fdbc3f0040e9441793", + "0fd7e4bfba5b3a82be88d1057757ca8b2c5e6d26", + "100746511cc45c9f1ad6721c4ef5be49222fee4d", + "1088490171d9b984d68b8b9be9ca003f4eafff59", + "1093c8ff4cb78fcf5f79dbbeedcb6e824bd4e253", + "10aa3fa72afab7ee31e116ae06442fe0f7b79df2", + "10b759e734e8299aa0dca08be935d95d886127b6", + "111d5ccf0bb010c4e8d7af3eedfa12ef4c5e265b", + "11261fbff21758444d426356ff6327ee01e90752", + "112998d425717bb922ce74e8f6f0f831d8dc4510", + "2ef4e5d838b6507bd61d457cf6466662b791c5c0", + "2ef4faa0f82efa00eeac6cae9e8b2abccc8566ee", + "2f06098183b0d7be350acbe39cdbaccff2df0c4a", + "2f1c5d509ac5bffb3c62f710a1c2c542e126dfd1", + "2f205b20fc16423c42b3ba51b2ea78d7b9ff3578", + "2f9b6b6e3d9250ba09360734aa47973a993b59d1", + "30c62a2d5a8d644f1311d4f7fe3f6a788e4c8188", + "31438e245492d85fd6da4d1406eba0fbde8332a4", + "3184a3abdfea231992254929ff4e275898e5bbf6", + "3188ffdbb3a3d52e0f78f30c484533899224436e", + "32581d0093429770d044a60eb0e9cc0462bedb13", + "32679a9544d83e5403202c4d5efb61ad02492847", + "4e7e9f60b7e2049b7f5697daf133161a18ef688f", + "4e8cda27ddc8be7db875ceb0f360c37734724c6d", + "4ea481c61c59ab55169b7cbaae536ad50b49d6f0", + "4f0adcd0e61eabe06fe32be66b16559537124b7a", + "4f1355c91100d12f9e7202f91b245df0c110867c", + "4f6eadeb08b9d0d1e8b1b3eac8a34940adf29a2d", + "4f9339df943c53117a5fc8e86e2f38716ff3a668", + "4fc3874b118752e40de556b1c3e7b4a9f1737d00", + "4ff1dd0992dd6baafdb5e166be6f9f23b59bdf87", + "5018a35e0b7e2eec7ce5050baf9c7343f3f74164", + "50298f44a45eda3a29dae82dbe911b5aa176ac07", + "502acd164fb115768d723144da2e7bb5a24891bb", + "50330c02bd4fd95c9db1fcf2f97f4218e42b7226", + "5052bf355d9f8c52446561a39733a8767bf31e37", + "6f2cd729ae42988c1dd43588d3a6661ba48ad7a0", + "6f4e2c42d9138bfbf3e0f908f1308828cc6f2178", + "6f6a17db05a83620cef4572761831c20a70ba9b9", + "6faad60901e36538634f0d8b8ff3f21f83503c71", + "6fc72e46de3df0c3842dab302bbacf697a63abab", + "6fdccd49f442a7204399ca9b418f017322dbded8", + "6fe7568fc3861c334cb008fd85d57d9647249ef5", + "700f55d91d7b55665594676a4bada1f1457a0598", + "702bd70595a7b19afc48a1f784a6505be68469d4", + "7033f9ee0e52b08cb5679cd49b7b7999eaf9eaf8", + "70957110ce446c4e250f865760fb3da513cdcc92", + "8ec696a4734f16479d091bc70574d23dd9fe7443", + "8ed341c55ed4d6f4cdc8bf4f0ca18a08c93f6962", + "8edc2805f1f11b63e44bf81f4557f8b473612b69", + "8ef9060a954118a698fc10e20acdc430566a100f", + "8f0c4b543f4bb6eb1518ecfc3d4699e43108d393", + "8fac94df3035405c2e60b3799153ce7c428af6b9", + "904c0ac12b23548de524adae712241b423d765a3", + "90bbaa9a809c3a768d873a9cc7d52b4f3bf3d1b9", + "90d4d2f0fc362beabbbf76b4ffda0828229c198d", + "90f9ff6755330b685feff6c3d81782ee3592ab04", + "91822c50ebe4f9bf5bbb8308ecf9f6557062775c", + "91d973263a55708fa8255867b3202d81ef9c2868", + "af292c99c6148d772af3315a1c74e83330e7ead7", + "af3b99d5be330dbbce0b9250c3a5fb05911908cc", + "af55d0cdeb280af2db8697e5afa506e081012719", + "af795e498d411142ddb073e8ca2c5447c3295a4c", + "afadc73a392f8cc8e2cc77dd62a7433dd3bafa8c", + "affd84ed8ec7ce67612fe3c12a80f8164b101f6a", + "b0941f9c70ffe67f0387a827b338e64ecf3190f0", + "b0a3077f9ef6e093f8d9869bdb0c07095bd722cb", + "b0a8568a7614806378a54db5706ee3b06ae58693", + "b0fb7372f242233d1d35ce7d8e74d3990cbc5841", + "b10489944b9ead17427551759d180d10203e06ba", + "b196a807b323f2748ffc6b1d42cd0812d04c9a40", + "b1bb1d888f0c5e19278536d49fa77db035fac7ae" +}; + +BEGIN_TEST(readpacked_test) + unsigned int i; + git_odb *db; + + must_pass(git_odb_open(&db, ODB_FOLDER)); + + for (i = 0; i < ARRAY_SIZE(packed_objects); ++i) { + git_oid id; + git_obj obj; + + must_pass(git_oid_mkstr(&id, packed_objects[i])); + must_be_true(git_odb_exists(db, &id) == 1); + must_pass(git_odb__read_packed(&obj, db, &id)); + + git_obj_close(&obj); + } + + git_odb_close(db); +END_TEST