proxmox-mirrors/libgit2
1
0
Fork 0
mirror of https://git.proxmox.com/git/libgit2 synced 2025-05-28 14:02:57 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

remote: use the same code to control git and http

Browse Source 
This allows us to add capabilitites to both at the same time, keeps
them in sync and removes a lot of code.

gitno_buffer now uses a callback to fill its buffer, allowing us to
use the same interface for git and http (which uses callbacks).
This commit is contained in:
Carlos Martín Nieto 2012-07-24 19:03:22 +02:00
parent 114dc6e14c
commit b49c8f71ae
10 changed files with 208 additions and 332 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
src/fetch.c
View File

@ -79,7 +79,7 @@ static int recv_pkt(git_pkt **out, gitno_buffer *buf)
{
const char *ptr = buf->data, *line_end = ptr;
git_pkt *pkt;
int pkt_type, error = 0;
int pkt_type, error = 0, ret;
do {
if (buf->offset > 0)
@ -106,7 +106,7 @@ static int recv_pkt(git_pkt **out, gitno_buffer *buf)
return GIT_PKT_NAK;
}
if ((error = gitno_recv(buf)) < 0)
if ((ret = gitno_recv(buf)) < 0)
return -1;
} while (error);
@ -122,7 +122,6 @@ static int recv_pkt(git_pkt **out, gitno_buffer *buf)
static int store_common(git_transport *t)
{
int done = 0;
git_pkt *pkt = NULL;
gitno_buffer *buf = &t->buffer;
@ -219,12 +218,42 @@ int git_fetch_negotiate(git_remote *remote)
if (t->common.length > 0)
break;
if (i % 20 == 0 && t->rpc) {
git_pkt_ack *pkt;
unsigned int i;
if (git_pkt_buffer_wants(&remote->refs, &t->caps, &data) < 0)
goto on_error;
git_vector_foreach(&t->common, i, pkt) {
git_pkt_buffer_have(&pkt->oid, &data);
}
if (git_buf_oom(&data))
goto on_error;
}
}
if (error < 0 && error != GIT_REVWALKOVER)
goto on_error;
/* Tell the other end that we're done negotiating */
if (t->rpc && t->common.length > 0) {
git_pkt_ack *pkt;
unsigned int i;
if (git_pkt_buffer_wants(&remote->refs, &t->caps, &data) < 0)
goto on_error;
git_vector_foreach(&t->common, i, pkt) {
git_pkt_buffer_have(&pkt->oid, &data);
}
if (git_buf_oom(&data))
goto on_error;
}
git_pkt_buffer_done(&data);
if (t->negotiation_step(t, data.ptr, data.size) < 0)
goto on_error;
@ -233,10 +262,26 @@ int git_fetch_negotiate(git_remote *remote)
git_revwalk_free(walk);
/* Now let's eat up whatever the server gives us */
pkt_type = recv_pkt(NULL, buf);
if (pkt_type != GIT_PKT_ACK && pkt_type != GIT_PKT_NAK) {
giterr_set(GITERR_NET, "Unexpected pkt type");
return -1;
if (!t->caps.multi_ack) {
pkt_type = recv_pkt(NULL, buf);
if (pkt_type != GIT_PKT_ACK && pkt_type != GIT_PKT_NAK) {
giterr_set(GITERR_NET, "Unexpected pkt type");
return -1;
}
} else {
git_pkt_ack *pkt;
do {
if (recv_pkt((git_pkt **)&pkt, buf) < 0)
return -1;
if (pkt->type == GIT_PKT_NAK ||
(pkt->type == GIT_PKT_ACK && pkt->status != GIT_ACK_CONTINUE)) {
git__free(pkt);
break;
}
git__free(pkt);
} while (1);
}
return 0;
@ -257,50 +302,39 @@ int git_fetch_download_pack(git_remote *remote, git_off_t *bytes, git_indexer_st
if (t->own_logic)
return t->download_pack(t, remote->repo, bytes, stats);
return git_fetch__download_pack(NULL, 0, t, remote->repo, bytes, stats);
return git_fetch__download_pack(t, remote->repo, bytes, stats);
}
/* Receiving data from a socket and storing it is pretty much the same for git and HTTP */
int git_fetch__download_pack(
const char *buffered,
size_t buffered_size,
git_transport *t,
git_repository *repo,
git_off_t *bytes,
git_indexer_stats *stats)
{
int recvd;
char buff[1024];
gitno_buffer buf;
git_buf path = GIT_BUF_INIT;
gitno_buffer *buf = &t->buffer;
git_indexer_stream *idx = NULL;
gitno_buffer_setup(t, &buf, buff, sizeof(buff));
if (buffered && memcmp(buffered, "PACK", strlen("PACK"))) {
giterr_set(GITERR_NET, "The pack doesn't start with the signature");
return -1;
}
if (git_buf_joinpath(&path, git_repository_path(repo), "objects/pack") < 0)
return -1;
if (git_indexer_stream_new(&idx, git_buf_cstr(&path)) < 0)
goto on_error;
git_buf_free(&path);
memset(stats, 0, sizeof(git_indexer_stats));
if (buffered && git_indexer_stream_add(idx, buffered, buffered_size, stats) < 0)
goto on_error;
*bytes = buffered_size;
*bytes = 0;
do {
if (git_indexer_stream_add(idx, buf.data, buf.offset, stats) < 0)
if (git_indexer_stream_add(idx, buf->data, buf->offset, stats) < 0)
goto on_error;
gitno_consume_n(&buf, buf.offset);
if ((recvd = gitno_recv(&buf)) < 0)
gitno_consume_n(buf, buf->offset);
if ((recvd = gitno_recv(buf)) < 0)
goto on_error;
*bytes += recvd;

3
src/fetch.h
View File

@ -12,8 +12,7 @@
int git_fetch_negotiate(git_remote *remote);
int git_fetch_download_pack(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats);
int git_fetch__download_pack(const char *buffered, size_t buffered_size, git_transport *t,
git_repository *repo, git_off_t *bytes, git_indexer_stats *stats);
int git_fetch__download_pack(git_transport *t, git_repository *repo, git_off_t *bytes, git_indexer_stats *stats);
int git_fetch_setup_walk(git_revwalk **out, git_repository *repo);
#endif

17
src/netops.c
View File

@ -61,7 +61,7 @@ static int ssl_set_error(gitno_ssl *ssl, int error)
}
#endif
void gitno_buffer_setup(git_transport *t, gitno_buffer *buf, char *data, unsigned int len)
void gitno_buffer_setup_callback(git_transport *t, gitno_buffer *buf, char *data, unsigned int len, int (*recv)(gitno_buffer *buf), void *cb_data)
{
memset(buf, 0x0, sizeof(gitno_buffer));
memset(data, 0x0, len);
@ -73,6 +73,19 @@ void gitno_buffer_setup(git_transport *t, gitno_buffer *buf, char *data, unsigne
if (t->encrypt)
buf->ssl = &t->ssl;
#endif
buf->recv = recv;
buf->cb_data = cb_data;
}
void gitno_buffer_setup(git_transport *t, gitno_buffer *buf, char *data, unsigned int len)
{
gitno_buffer_setup_callback(t, buf, data, len, gitno__recv, NULL);
}
int gitno_recv(gitno_buffer *buf)
{
return buf->recv(buf);
}
#ifdef GIT_SSL
@ -91,7 +104,7 @@ static int ssl_recv(gitno_ssl *ssl, void *data, size_t len)
}
#endif
int gitno_recv(gitno_buffer *buf)
int gitno__recv(gitno_buffer *buf)
{
int ret;

4
src/netops.h
View File

@ -18,10 +18,14 @@ struct gitno_buffer {
#ifdef GIT_SSL
struct gitno_ssl *ssl;
#endif
int (*recv)(gitno_buffer *buffer);
void *cb_data;
};
void gitno_buffer_setup(git_transport *t, gitno_buffer *buf, char *data, unsigned int len);
void gitno_buffer_setup_callback(git_transport *t, gitno_buffer *buf, char *data, unsigned int len, int (*recv)(gitno_buffer *buf), void *cb_data);
int gitno_recv(gitno_buffer *buf);
int gitno__recv(gitno_buffer *buf);
void gitno_consume(gitno_buffer *buf, const char *ptr);
void gitno_consume_n(gitno_buffer *buf, size_t cons);

20
src/pkt.c
View File

@ -42,15 +42,29 @@ static int flush_pkt(git_pkt **out)
/* the rest of the line will be useful for multi_ack */
static int ack_pkt(git_pkt **out, const char *line, size_t len)
{
git_pkt *pkt;
git_pkt_ack *pkt;
GIT_UNUSED(line);
GIT_UNUSED(len);
pkt = git__malloc(sizeof(git_pkt));
pkt = git__calloc(1, sizeof(git_pkt_ack));
GITERR_CHECK_ALLOC(pkt);
pkt->type = GIT_PKT_ACK;
*out = pkt;
line += 3;
len -= 3;
if (len >= GIT_OID_HEXSZ) {
git_oid_fromstr(&pkt->oid, line + 1);
line += GIT_OID_HEXSZ + 1;
len -= GIT_OID_HEXSZ + 1;
}
if (len >= 7) {
if (!git__prefixcmp(line + 1, "continue"))
pkt->status = GIT_ACK_CONTINUE;
}
*out = (git_pkt *) pkt;
return 0;
}

32
src/protocol.c
View File

@ -56,3 +56,35 @@ int git_protocol_store_refs(git_protocol *p, const char *data, size_t len)
return 0;
}
int git_protocol_detect_caps(git_pkt_ref *pkt, git_transport_caps *caps)
{
const char *ptr;
/* No refs or capabilites, odd but not a problem */
if (pkt == NULL || pkt->capabilities == NULL)
return 0;
ptr = pkt->capabilities;
while (ptr != NULL && *ptr != '\0') {
if (*ptr == ' ')
ptr++;
if(!git__prefixcmp(ptr, GIT_CAP_OFS_DELTA)) {
caps->common = caps->ofs_delta = 1;
ptr += strlen(GIT_CAP_OFS_DELTA);
continue;
}
if(!git__prefixcmp(ptr, GIT_CAP_MULTI_ACK)) {
caps->common = caps->multi_ack = 1;
ptr += strlen(GIT_CAP_MULTI_ACK);
continue;
}
/* We don't know this capability, so skip it */
ptr = strchr(ptr, ' ');
}
return 0;
}

2
src/protocol.h
View File

@ -9,6 +9,7 @@
#include "transport.h"
#include "buffer.h"
#include "pkt.h"
typedef struct {
git_transport *transport;
@ -19,5 +20,6 @@ typedef struct {
} git_protocol;
int git_protocol_store_refs(git_protocol *p, const char *data, size_t len);
int git_protocol_detect_caps(git_pkt_ref *pkt, git_transport_caps *caps);
#endif

3
src/transport.h
View File

@ -74,7 +74,8 @@ struct git_transport {
connected : 1,
check_cert: 1,
encrypt : 1,
own_logic: 1; /* transitional */
own_logic: 1, /* transitional */
rpc: 1; /* git-speak for the HTTP transport */
#ifdef GIT_SSL
struct gitno_ssl ssl;
#endif

38
src/transports/git.c
View File

@ -156,42 +156,6 @@ static int store_refs(transport_git *t)
}
}
static int detect_caps(transport_git *t)
{
git_vector *refs = &t->refs;
git_pkt_ref *pkt;
git_transport_caps *caps = &t->parent.caps;
const char *ptr;
pkt = git_vector_get(refs, 0);
/* No refs or capabilites, odd but not a problem */
if (pkt == NULL || pkt->capabilities == NULL)
return 0;
ptr = pkt->capabilities;
while (ptr != NULL && *ptr != '\0') {
if (*ptr == ' ')
ptr++;
if(!git__prefixcmp(ptr, GIT_CAP_OFS_DELTA)) {
caps->common = caps->ofs_delta = 1;
ptr += strlen(GIT_CAP_OFS_DELTA);
continue;
}
if(!git__prefixcmp(ptr, GIT_CAP_MULTI_ACK)) {
caps->common = caps->multi_ack = 1;
ptr += strlen(GIT_CAP_MULTI_ACK);
continue;
}
/* We don't know this capability, so skip it */
ptr = strchr(ptr, ' ');
}
return 0;
}
/*
* Since this is a network connection, we need to parse and store the
* pkt-lines at this stage and keep them there.
@ -219,7 +183,7 @@ static int git_connect(git_transport *transport, int direction)
if (store_refs(t) < 0)
goto cleanup;
if (detect_caps(t) < 0)
if (git_protocol_detect_caps(git_vector_get(&t->refs, 0), &transport->caps) < 0)
goto cleanup;
return 0;

335
src/transports/http.c
View File

@ -31,7 +31,7 @@ typedef struct {
git_transport parent;
git_protocol proto;
git_vector refs;
git_vector common;
http_parser_settings settings;
git_buf buf;
git_remote_head **heads;
int error;
@ -46,7 +46,7 @@ typedef struct {
char *host;
char *port;
char *service;
git_transport_caps caps;
char buffer[4096];
#ifdef GIT_WIN32
WSADATA wsd;
#endif
@ -210,6 +210,7 @@ static int on_message_complete(http_parser *parser)
static int store_refs(transport_http *t)
{
git_transport *transport = (git_transport *) t;
http_parser_settings settings;
char buffer[1024];
gitno_buffer buf;
@ -241,7 +242,7 @@ static int store_refs(transport_http *t)
gitno_consume_n(&buf, parsed);
if (ret == 0 || t->transfer_finished)
return 0;
break;
}
pkt = git_vector_get(&t->refs, 0);
@ -249,9 +250,14 @@ static int store_refs(transport_http *t)
giterr_set(GITERR_NET, "Invalid HTTP response");
return t->error = -1;
} else {
/* Remove the comment and flush pkts */
git_vector_remove(&t->refs, 0);
git_vector_remove(&t->refs, 0);
}
if (git_protocol_detect_caps(git_vector_get(&t->refs, 0), &transport->caps) < 0)
return t->error = -1;
return 0;
}
@ -333,278 +339,86 @@ static int http_ls(git_transport *transport, git_headlist_cb list_cb, void *opaq
return 0;
}
static int on_body_parse_response(http_parser *parser, const char *str, size_t len)
static int on_body_fill_buffer(http_parser *parser, const char *str, size_t len)
{
git_transport *transport = (git_transport *) parser->data;
transport_http *t = (transport_http *) parser->data;
git_buf *buf = &t->buf;
git_vector *common = &t->common;
int error;
const char *line_end, *ptr;
gitno_buffer *buf = &transport->buffer;
if (len == 0) { /* EOF */
if (git_buf_len(buf) != 0) {
giterr_set(GITERR_NET, "Unexpected EOF");
return t->error = -1;
} else {
return 0;
}
if (buf->len - buf->offset < len) {
giterr_set(GITERR_NET, "Can't fit data in the buffer");
return t->error = -1;
}
git_buf_put(buf, str, len);
ptr = buf->ptr;
while (1) {
git_pkt *pkt;
if (git_buf_len(buf) == 0)
return 0;
error = git_pkt_parse_line(&pkt, ptr, &line_end, git_buf_len(buf));
if (error == GIT_EBUFS) {
return 0; /* Ask for more */
}
if (error < 0)
return t->error = -1;
git_buf_consume(buf, line_end);
if (pkt->type == GIT_PKT_PACK) {
git__free(pkt);
t->pack_ready = 1;
return 0;
}
if (pkt->type == GIT_PKT_NAK) {
git__free(pkt);
return 0;
}
if (pkt->type != GIT_PKT_ACK) {
git__free(pkt);
continue;
}
if (git_vector_insert(common, pkt) < 0)
return -1;
}
return error;
memcpy(buf->data + buf->offset, str, len);
buf->offset += len;
return 0;
}
static int parse_response(transport_http *t)
static int http_recv_cb(gitno_buffer *buf)
{
int ret = 0;
http_parser_settings settings;
char buffer[1024];
gitno_buffer buf;
git_transport *transport = (git_transport *) buf->cb_data;
transport_http *t = (transport_http *) transport;
size_t parsed, old_len;
gitno_buffer inner;
char buffer[2048];
int error;
if (t->transfer_finished)
return 0;
gitno_buffer_setup(transport, &inner, buffer, sizeof(buffer));
if ((error = gitno_recv(&inner)) < 0)
return -1;
old_len = buf->offset;
parsed = http_parser_execute(&t->parser, &t->settings, inner.data, inner.offset);
if (t->error < 0)
return t->error;
return buf->offset - old_len;
}
static int http_negotiation_step(struct git_transport *transport, void *data, size_t len)
{
transport_http *t = (transport_http *) transport;
git_buf request = GIT_BUF_INIT;
int ret;
/* First, send the data as a HTTP POST request */
if ((ret = do_connect(t, t->host, t->port)) < 0)
return -1;
if ((ret = gen_request(&request, t->path, t->host, "POST", "upload-pack", len, 0)) < 0)
goto on_error;
if ((ret = gitno_send(transport, request.ptr, request.size, 0)) < 0)
goto on_error;
if ((ret = gitno_send(transport, data, len, 0)) < 0)
goto on_error;
git_buf_free(&request);
/* Then we need to set up the buffer to grab data from the HTTP response */
http_parser_init(&t->parser, HTTP_RESPONSE);
t->parser.data = t;
t->transfer_finished = 0;
memset(&settings, 0x0, sizeof(http_parser_settings));
settings.on_header_field = on_header_field;
settings.on_header_value = on_header_value;
settings.on_headers_complete = on_headers_complete;
settings.on_body = on_body_parse_response;
settings.on_message_complete = on_message_complete;
memset(&t->settings, 0x0, sizeof(http_parser_settings));
t->settings.on_header_field = on_header_field;
t->settings.on_header_value = on_header_value;
t->settings.on_headers_complete = on_headers_complete;
t->settings.on_body = on_body_fill_buffer;
t->settings.on_message_complete = on_message_complete;
gitno_buffer_setup((git_transport *)t, &buf, buffer, sizeof(buffer));
gitno_buffer_setup_callback(transport, &transport->buffer, t->buffer, sizeof(t->buffer), http_recv_cb, t);
while(1) {
size_t parsed;
if ((ret = gitno_recv(&buf)) < 0)
return -1;
parsed = http_parser_execute(&t->parser, &settings, buf.data, buf.offset);
/* Both should happen at the same time */
if (parsed != buf.offset || t->error < 0)
return t->error;
gitno_consume_n(&buf, parsed);
if (ret == 0 || t->transfer_finished || t->pack_ready) {
return 0;
}
}
return ret;
}
static int http_negotiate_fetch(git_transport *transport, git_repository *repo, const git_vector *wants)
{
transport_http *t = (transport_http *) transport;
int ret;
unsigned int i;
char buff[128];
gitno_buffer buf;
git_revwalk *walk = NULL;
git_oid oid;
git_pkt_ack *pkt;
git_vector *common = &t->common;
git_buf request = GIT_BUF_INIT, data = GIT_BUF_INIT;
gitno_buffer_setup(transport, &buf, buff, sizeof(buff));
if (git_vector_init(common, 16, NULL) < 0)
return -1;
if (git_fetch_setup_walk(&walk, repo) < 0)
return -1;
do {
if ((ret = do_connect(t, t->host, t->port)) < 0)
goto cleanup;
if ((ret = git_pkt_buffer_wants(wants, &t->caps, &data)) < 0)
goto cleanup;
/* We need to send these on each connection */
git_vector_foreach (common, i, pkt) {
if ((ret = git_pkt_buffer_have(&pkt->oid, &data)) < 0)
goto cleanup;
}
i = 0;
while ((i < 20) && ((ret = git_revwalk_next(&oid, walk)) == 0)) {
if ((ret = git_pkt_buffer_have(&oid, &data)) < 0)
goto cleanup;
i++;
}
git_pkt_buffer_done(&data);
if ((ret = gen_request(&request, t->path, t->host, "POST", "upload-pack", data.size, 0)) < 0)
goto cleanup;
if ((ret = gitno_send(transport, request.ptr, request.size, 0)) < 0)
goto cleanup;
if ((ret = gitno_send(transport, data.ptr, data.size, 0)) < 0)
goto cleanup;
git_buf_clear(&request);
git_buf_clear(&data);
if (ret < 0 || i >= 256)
break;
if ((ret = parse_response(t)) < 0)
goto cleanup;
if (t->pack_ready) {
ret = 0;
goto cleanup;
}
} while(1);
cleanup:
git_buf_free(&request);
git_buf_free(&data);
git_revwalk_free(walk);
return ret;
}
typedef struct {
git_indexer_stream *idx;
git_indexer_stats *stats;
transport_http *transport;
} download_pack_cbdata;
static int on_message_complete_download_pack(http_parser *parser)
{
download_pack_cbdata *data = (download_pack_cbdata *) parser->data;
data->transport->transfer_finished = 1;
return 0;
}
static int on_body_download_pack(http_parser *parser, const char *str, size_t len)
{
download_pack_cbdata *data = (download_pack_cbdata *) parser->data;
transport_http *t = data->transport;
git_indexer_stream *idx = data->idx;
git_indexer_stats *stats = data->stats;
return t->error = git_indexer_stream_add(idx, str, len, stats);
}
/*
* As the server is probably using Transfer-Encoding: chunked, we have
* to use the HTTP parser to download the pack instead of giving it to
* the simple downloader. Furthermore, we're using keep-alive
* connections, so the simple downloader would just hang.
*/
static int http_download_pack(git_transport *transport, git_repository *repo, git_off_t *bytes, git_indexer_stats *stats)
{
transport_http *t = (transport_http *) transport;
git_buf *oldbuf = &t->buf;
int recvd;
http_parser_settings settings;
char buffer[1024];
gitno_buffer buf;
git_buf path = GIT_BUF_INIT;
git_indexer_stream *idx = NULL;
download_pack_cbdata data;
gitno_buffer_setup(transport, &buf, buffer, sizeof(buffer));
if (memcmp(oldbuf->ptr, "PACK", strlen("PACK"))) {
giterr_set(GITERR_NET, "The pack doesn't start with a pack signature");
return -1;
}
if (git_buf_joinpath(&path, git_repository_path(repo), "objects/pack") < 0)
return -1;
if (git_indexer_stream_new(&idx, git_buf_cstr(&path)) < 0)
return -1;
/*
* This is part of the previous response, so we don't want to
* re-init the parser, just set these two callbacks.
*/
memset(stats, 0, sizeof(git_indexer_stats));
data.stats = stats;
data.idx = idx;
data.transport = t;
t->parser.data = &data;
t->transfer_finished = 0;
memset(&settings, 0x0, sizeof(settings));
settings.on_message_complete = on_message_complete_download_pack;
settings.on_body = on_body_download_pack;
*bytes = git_buf_len(oldbuf);
if (git_indexer_stream_add(idx, git_buf_cstr(oldbuf), git_buf_len(oldbuf), stats) < 0)
goto on_error;
gitno_buffer_setup(transport, &buf, buffer, sizeof(buffer));
do {
size_t parsed;
if ((recvd = gitno_recv(&buf)) < 0)
goto on_error;
parsed = http_parser_execute(&t->parser, &settings, buf.data, buf.offset);
if (parsed != buf.offset || t->error < 0)
goto on_error;
*bytes += recvd;
gitno_consume_n(&buf, parsed);
} while (recvd > 0 && !t->transfer_finished);
if (git_indexer_stream_finalize(idx, stats) < 0)
goto on_error;
git_indexer_stream_free(idx);
return 0;
on_error:
git_indexer_stream_free(idx);
git_buf_free(&path);
git_buf_free(&request);
return -1;
}
@ -628,7 +442,7 @@ static void http_free(git_transport *transport)
{
transport_http *t = (transport_http *) transport;
git_vector *refs = &t->refs;
git_vector *common = &t->common;
git_vector *common = &transport->common;
unsigned int i;
git_pkt *p;
@ -668,13 +482,12 @@ int git_transport_http(git_transport **out)
memset(t, 0x0, sizeof(transport_http));
t->parent.own_logic = 1;
t->parent.connect = http_connect;
t->parent.ls = http_ls;
t->parent.negotiate_fetch = http_negotiate_fetch;
t->parent.download_pack = http_download_pack;
t->parent.negotiation_step = http_negotiation_step;
t->parent.close = http_close;
t->parent.free = http_free;
t->parent.rpc = 1;
t->proto.refs = &t->refs;
t->proto.transport = (git_transport *) t;