static int fix_thin_pack(git_indexer *idx, git_transfer_progress *stats)
{
int error, found_ref_delta = 0;
unsigned int i;
struct delta_info *delta;
size_t size;
git_otype type;
git_mwindow *w = NULL;
git_off_t curpos = 0;
unsigned char *base_info;
unsigned int left = 0;
git_oid base;
assert(git_vector_length(&idx->deltas) > 0);
if (idx->odb == NULL) {
giterr_set(GITERR_INDEXER, "cannot fix a thin pack without an ODB");
return -1;
}
/* Loop until we find the first REF delta */
git_vector_foreach(&idx->deltas, i, delta, delta_info*) {
if (!delta)
continue;
curpos = delta->delta_off;
error = git_packfile_unpack_header(&size, &type, &idx->pack->mwf, &w, &curpos);
git_mwindow_close(&w);
if (error < 0)
return error;
if (type == GIT_OBJ_REF_DELTA) {
found_ref_delta = 1;
break;
}
}
if (!found_ref_delta) {
giterr_set(GITERR_INDEXER, "no REF_DELTA found, cannot inject object");
return -1;
}
/* curpos now points to the base information, which is an OID */
base_info = git_mwindow_open(&idx->pack->mwf, &w, curpos, GIT_OID_RAWSZ, &left);
if (base_info == NULL) {
giterr_set(GITERR_INDEXER, "failed to map delta information");
return -1;
}
git_oid_fromraw(&base, base_info);
git_mwindow_close(&w);
if (inject_object(idx, &base) < 0)
return -1;
stats->local_objects++;
return 0;
}
static int read_unmerged(git_index *index, const char *buffer, size_t size)
{
const char *endptr;
size_t len;
int i;
git_vector_init(&index->unmerged, 16, unmerged_cmp);
while (size) {
git_index_entry_unmerged *lost;
len = strlen(buffer) + 1;
if (size <= len)
return git__throw(GIT_ERROR, "Failed to read unmerged entries");
if ((lost = git__malloc(sizeof(git_index_entry_unmerged))) == NULL)
return GIT_ENOMEM;
if (git_vector_insert(&index->unmerged, lost) < GIT_SUCCESS)
return git__throw(GIT_ERROR, "Failed to read unmerged entries");
lost->path = git__strdup(buffer);
if (!lost->path)
return GIT_ENOMEM;
size -= len;
buffer += len;
for (i = 0; i < 3; i++) {
long tmp;
if (git__strtol32(&tmp, buffer, &endptr, 8) < GIT_SUCCESS ||
!endptr || endptr == buffer || *endptr || tmp > UINT_MAX)
return GIT_ERROR;
lost->mode[i] = tmp;
len = (endptr + 1) - buffer;
if (size <= len)
return git__throw(GIT_ERROR, "Failed to read unmerged entries");
size -= len;
buffer += len;
}
for (i = 0; i < 3; i++) {
if (!lost->mode[i])
continue;
if (size < 20)
return git__throw(GIT_ERROR, "Failed to read unmerged entries");
git_oid_fromraw(&lost->oid[i], (unsigned char *) buffer);
size -= 20;
buffer += 20;
}
}
return GIT_SUCCESS;
}
static int tree_parse_buffer(git_tree *tree, const char *buffer, const char *buffer_end)
{
int error = GIT_SUCCESS;
if (git_vector_init(&tree->entries, DEFAULT_TREE_SIZE, entry_sort_cmp) < GIT_SUCCESS)
return GIT_ENOMEM;
while (buffer < buffer_end) {
git_tree_entry *entry;
int tmp;
entry = git__calloc(1, sizeof(git_tree_entry));
if (entry == NULL) {
error = GIT_ENOMEM;
break;
}
if (git_vector_insert(&tree->entries, entry) < GIT_SUCCESS)
return GIT_ENOMEM;
if (git__strtol32(&tmp, buffer, &buffer, 8) < GIT_SUCCESS ||
!buffer || !valid_attributes(tmp))
return git__throw(GIT_EOBJCORRUPTED, "Failed to parse tree. Can't parse attributes");
entry->attr = tmp;
if (*buffer++ != ' ') {
error = git__throw(GIT_EOBJCORRUPTED, "Failed to parse tree. Object it corrupted");
break;
}
if (memchr(buffer, 0, buffer_end - buffer) == NULL) {
error = git__throw(GIT_EOBJCORRUPTED, "Failed to parse tree. Object it corrupted");
break;
}
entry->filename = git__strdup(buffer);
entry->filename_len = strlen(buffer);
while (buffer < buffer_end && *buffer != 0)
buffer++;
buffer++;
git_oid_fromraw(&entry->oid, (const unsigned char *)buffer);
buffer += GIT_OID_RAWSZ;
}
return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to parse buffer");
}
/*
* call-seq:
* Rugged.raw_to_hex(buffer) -> hex_oid
*
* Turn a buffer of 20 bytes (representing a SHA1 OID) into its
* readable hexadecimal representation.
*
* Rugged.raw_to_hex("\330xk\374\227H^\215{\031\262\037\270\214\216\361\361\231\374?")
* #=> "d8786bfc97485e8d7b19b21fb88c8ef1f199fc3f"
*/
static VALUE rb_git_raw_to_hex(VALUE self, VALUE raw)
{
git_oid oid;
char out[40];
Check_Type(raw, T_STRING);
if (RSTRING_LEN(raw) != GIT_OID_RAWSZ)
rb_raise(rb_eTypeError, "Invalid buffer size for an OID");
git_oid_fromraw(&oid, (const unsigned char *)RSTRING_PTR(raw));
git_oid_fmt(out, &oid);
return rb_str_new(out, 40);
}
int
py_str_to_git_oid(PyObject *py_str, git_oid *oid)
{
PyObject *py_hex;
char *hex_or_bin;
int err;
Py_ssize_t len;
/* Case 1: raw sha */
if (PyString_Check(py_str)) {
hex_or_bin = PyString_AsString(py_str);
if (hex_or_bin == NULL)
return -1;
git_oid_fromraw(oid, (const unsigned char*)hex_or_bin);
return GIT_OID_HEXSZ;
}
/* Case 2: hex sha */
if (PyUnicode_Check(py_str)) {
py_hex = PyUnicode_AsASCIIString(py_str);
if (py_hex == NULL)
return -1;
err = PyString_AsStringAndSize(py_hex, &hex_or_bin, &len);
if (err) {
Py_DECREF(py_hex);
return -1;
}
err = git_oid_fromstrn(oid, hex_or_bin, len);
Py_DECREF(py_hex);
if (err < 0) {
PyErr_SetObject(Error_type(err), py_str);
return -1;
}
return len;
}
/* Type error */
PyErr_Format(PyExc_TypeError,
"Git object id must be byte or a text string, not: %.200s",
Py_TYPE(py_str)->tp_name);
return -1;
}
static int read_tree_internal(git_tree_cache **out,
const char **buffer_in, const char *buffer_end, git_tree_cache *parent)
{
git_tree_cache *tree = NULL;
const char *name_start, *buffer;
int count;
size_t name_len;
buffer = name_start = *buffer_in;
if ((buffer = memchr(buffer, '\0', buffer_end - buffer)) == NULL)
goto corrupted;
if (++buffer >= buffer_end)
goto corrupted;
name_len = strlen(name_start);
tree = git__malloc(sizeof(git_tree_cache) + name_len + 1);
GITERR_CHECK_ALLOC(tree);
memset(tree, 0x0, sizeof(git_tree_cache));
tree->parent = parent;
/* NUL-terminated tree name */
memcpy(tree->name, name_start, name_len);
tree->name[name_len] = '\0';
/* Blank-terminated ASCII decimal number of entries in this tree */
if (git__strtol32(&count, buffer, &buffer, 10) < 0)
goto corrupted;
tree->entries = count;
if (*buffer != ' ' || ++buffer >= buffer_end)
goto corrupted;
/* Number of children of the tree, newline-terminated */
if (git__strtol32(&count, buffer, &buffer, 10) < 0 || count < 0)
goto corrupted;
tree->children_count = count;
if (*buffer != '\n' || ++buffer > buffer_end)
goto corrupted;
/* The SHA1 is only there if it's not invalidated */
if (tree->entries >= 0) {
/* 160-bit SHA-1 for this tree and it's children */
if (buffer + GIT_OID_RAWSZ > buffer_end)
goto corrupted;
git_oid_fromraw(&tree->oid, (const unsigned char *)buffer);
buffer += GIT_OID_RAWSZ;
}
/* Parse children: */
if (tree->children_count > 0) {
unsigned int i;
tree->children = git__malloc(tree->children_count * sizeof(git_tree_cache *));
GITERR_CHECK_ALLOC(tree->children);
memset(tree->children, 0x0, tree->children_count * sizeof(git_tree_cache *));
for (i = 0; i < tree->children_count; ++i) {
if (read_tree_internal(&tree->children[i], &buffer, buffer_end, tree) < 0)
goto corrupted;
}
}
*buffer_in = buffer;
*out = tree;
return 0;
corrupted:
git_tree_cache_free(tree);
giterr_set(GITERR_INDEX, "Corrupted TREE extension in index");
return -1;
}
static VALUE
backend_read_tree(VALUE rb_obj, VALUE rb_oid)
{
git_oid tree_oid, blob_oid;
git_odb *odb;
git_odb_object *g_obj;
git_otype type;
int error, size, attr, len;
char *head, *cur, *tmp, *tail, str_oid[40];
VALUE rb_hash, rb_key, rb_value;
odb = DATA_PTR(rb_obj);
error = git_oid_fromstr(&tree_oid, RSTRING_PTR(rb_oid));
backend_exception_check(error);
error = git_odb_read(&g_obj, odb, &tree_oid);
backend_exception_check(error);
head = (char *)git_odb_object_data(g_obj);
size = git_odb_object_size(g_obj);
type = git_odb_object_type(g_obj);
if (type != GIT_OBJ_TREE) {
git_odb_object_free(g_obj);
rb_raise(rb_eArgError, "oid does not belong to a tree");
}
rb_hash = rb_hash_new();
cur = head;
tail = head + size;
while (cur < tail) {
attr = strtol(cur, &tmp, 8);
if (tmp - cur != 3) {
git_odb_object_free(g_obj);
rb_raise(rb_eRuntimeError, "tree is invalid, bad attributes");
}
if (*tmp != ' ') {
git_odb_object_free(g_obj);
rb_raise(rb_eRuntimeError, "tree is invalid, no space after attributes");
}
cur = tmp + 1;
/* Grab the filename */
len = strlen(cur);
rb_key = backend_str_new(cur, len, NULL);
cur += len + 1;
/* Grab the oid */
git_oid_fromraw(&blob_oid, (unsigned char *)cur);
git_oid_fmt(str_oid, &blob_oid);
rb_value = backend_str_new(str_oid, 40, NULL);
cur += GIT_OID_RAWSZ;
/* Set the hash entry */
rb_hash_aset(rb_hash, rb_key, rb_value);
}
git_odb_object_free(g_obj);
return rb_hash;
}
int git_indexer_commit(git_indexer *idx, git_transfer_progress *stats)
{
git_mwindow *w = NULL;
unsigned int i, long_offsets = 0, left;
int error;
struct git_pack_idx_header hdr;
git_buf filename = GIT_BUF_INIT;
struct entry *entry;
git_oid trailer_hash, file_hash;
git_hash_ctx ctx;
git_filebuf index_file = {0};
void *packfile_trailer;
if (git_hash_ctx_init(&ctx) < 0)
return -1;
/* Test for this before resolve_deltas(), as it plays with idx->off */
if (idx->off < idx->pack->mwf.size - 20) {
giterr_set(GITERR_INDEXER, "Unexpected data at the end of the pack");
return -1;
}
packfile_trailer = git_mwindow_open(&idx->pack->mwf, &w, idx->pack->mwf.size - GIT_OID_RAWSZ, GIT_OID_RAWSZ, &left);
if (packfile_trailer == NULL) {
git_mwindow_close(&w);
goto on_error;
}
/* Compare the packfile trailer as it was sent to us and what we calculated */
git_oid_fromraw(&file_hash, (unsigned char*) packfile_trailer);
git_mwindow_close(&w);
git_hash_final(&trailer_hash, &idx->trailer);
if (git_oid_cmp(&file_hash, &trailer_hash)) {
giterr_set(GITERR_INDEXER, "packfile trailer mismatch");
return -1;
}
/* Freeze the number of deltas */
stats->total_deltas = stats->total_objects - stats->indexed_objects;
if ((error = resolve_deltas(idx, stats)) < 0)
return error;
if (stats->indexed_objects != stats->total_objects) {
giterr_set(GITERR_INDEXER, "early EOF");
return -1;
}
if (stats->local_objects > 0) {
if (update_header_and_rehash(idx, stats) < 0)
return -1;
git_hash_final(&trailer_hash, &idx->trailer);
write_at(idx, &trailer_hash, idx->pack->mwf.size - GIT_OID_RAWSZ, GIT_OID_RAWSZ);
}
git_vector_sort(&idx->objects);
git_buf_sets(&filename, idx->pack->pack_name);
git_buf_shorten(&filename, strlen("pack"));
git_buf_puts(&filename, "idx");
if (git_buf_oom(&filename))
return -1;
if (git_filebuf_open(&index_file, filename.ptr,
GIT_FILEBUF_HASH_CONTENTS, idx->mode) < 0)
goto on_error;
/* Write out the header */
hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
hdr.idx_version = htonl(2);
git_filebuf_write(&index_file, &hdr, sizeof(hdr));
/* Write out the fanout table */
for (i = 0; i < 256; ++i) {
uint32_t n = htonl(idx->fanout[i]);
git_filebuf_write(&index_file, &n, sizeof(n));
}
/* Write out the object names (SHA-1 hashes) */
git_vector_foreach(&idx->objects, i, entry, struct entry*) {
git_filebuf_write(&index_file, &entry->oid, sizeof(git_oid));
git_hash_update(&ctx, &entry->oid, GIT_OID_RAWSZ);
}
git_hash_final(&idx->hash, &ctx);
/* Write out the CRC32 values */
git_vector_foreach(&idx->objects, i, entry, struct entry*) {
git_filebuf_write(&index_file, &entry->crc, sizeof(uint32_t));
}
/* Write out the offsets */
git_vector_foreach(&idx->objects, i, entry, struct entry*) {
uint32_t n;
if (entry->offset == UINT32_MAX)
n = htonl(0x80000000 | long_offsets++);
else
n = htonl(entry->offset);
git_filebuf_write(&index_file, &n, sizeof(uint32_t));
}
/* Write out the long offsets */
git_vector_foreach(&idx->objects, i, entry, struct entry*) {
uint32_t split[2];
if (entry->offset != UINT32_MAX)
continue;
split[0] = htonl(entry->offset_long >> 32);
split[1] = htonl(entry->offset_long & 0xffffffff);
git_filebuf_write(&index_file, &split, sizeof(uint32_t) * 2);
}
/* Write out the packfile trailer to the index */
if (git_filebuf_write(&index_file, &trailer_hash, GIT_OID_RAWSZ) < 0)
goto on_error;
/* Write out the hash of the idx */
if (git_filebuf_hash(&trailer_hash, &index_file) < 0)
goto on_error;
git_filebuf_write(&index_file, &trailer_hash, sizeof(git_oid));
/* Figure out what the final name should be */
if (index_path(&filename, idx, ".idx") < 0)
goto on_error;
/* Commit file */
if (git_filebuf_commit_at(&index_file, filename.ptr) < 0)
goto on_error;
git_mwindow_free_all(&idx->pack->mwf);
/* We need to close the descriptor here so Windows doesn't choke on commit_at */
if (p_close(idx->pack->mwf.fd) < 0) {
giterr_set(GITERR_OS, "failed to close packfile");
goto on_error;
}
idx->pack->mwf.fd = -1;
if (index_path(&filename, idx, ".pack") < 0)
goto on_error;
/* And don't forget to rename the packfile to its new place. */
p_rename(idx->pack->pack_name, git_buf_cstr(&filename));
git_buf_free(&filename);
git_hash_ctx_cleanup(&ctx);
return 0;
on_error:
git_mwindow_free_all(&idx->pack->mwf);
git_filebuf_cleanup(&index_file);
git_buf_free(&filename);
git_hash_ctx_cleanup(&ctx);
return -1;
}
ERL_NIF_TERM
geef_index_add(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[])
{
geef_index *index;
const ERL_NIF_TERM *eentry;
int arity;
unsigned int tmp;
ErlNifBinary path, id;
git_index_entry entry;
if (!enif_get_resource(env, argv[0], geef_index_type, (void **) &index))
return enif_make_badarg(env);
if (!enif_get_tuple(env, argv[1], &arity, &eentry))
return enif_make_badarg(env);
memset(&entry, 0, sizeof(entry));
if (enif_compare(eentry[1], atoms.undefined) &&
!enif_get_int64(env, eentry[1], &entry.ctime.seconds))
return enif_make_badarg(env);
if (enif_compare(eentry[2], atoms.undefined) &&
!enif_get_int64(env, eentry[2], &entry.mtime.seconds))
return enif_make_badarg(env);
if (enif_compare(eentry[3], atoms.undefined) &&
!enif_get_uint(env, eentry[3], &entry.dev))
return enif_make_badarg(env);
if (enif_compare(eentry[4], atoms.undefined) &&
!enif_get_uint(env, eentry[4], &entry.ino))
return enif_make_badarg(env);
if (!enif_get_uint(env, eentry[5], &entry.mode))
return enif_make_badarg(env);
if (enif_compare(eentry[6], atoms.undefined) &&
!enif_get_uint(env, eentry[6], &entry.uid))
return enif_make_badarg(env);
if (enif_compare(eentry[7], atoms.undefined) &&
!enif_get_uint(env, eentry[7], &entry.gid))
return enif_make_badarg(env);
if (!enif_get_int64(env, eentry[8], &entry.file_size))
return enif_make_badarg(env);
/* [9] comes later */
tmp = 0;
if (enif_compare(eentry[10], atoms.undefined) &&
!enif_get_uint(env, eentry[10], &tmp))
return enif_make_badarg(env);
entry.flags = tmp;
tmp = 0;
if (enif_compare(eentry[11], atoms.undefined) &&
!enif_get_uint(env, eentry[11], &tmp))
return enif_make_badarg(env);
entry.flags_extended = tmp;
if (!enif_inspect_iolist_as_binary(env, eentry[12], &path))
return enif_make_badarg(env);
if (!geef_terminate_binary(&path))
return geef_oom(env);
entry.path = (char *) path.data;
if (!enif_inspect_binary(env, eentry[9], &id))
return enif_make_badarg(env);
git_oid_fromraw(&entry.id, id.data);
if (git_index_add(index->index, &entry) < 0)
return geef_error(env);
return atoms.ok;
}
static int parse_index(git_index *index, const char *buffer, size_t buffer_size)
{
unsigned int i;
struct index_header header;
git_oid checksum_calculated, checksum_expected;
#define seek_forward(_increase) { \
if (_increase >= buffer_size) \
return git__throw(GIT_EOBJCORRUPTED, "Failed to seek forward. Buffer size exceeded"); \
buffer += _increase; \
buffer_size -= _increase;\
}
if (buffer_size < INDEX_HEADER_SIZE + INDEX_FOOTER_SIZE)
return git__throw(GIT_EOBJCORRUPTED, "Failed to parse index. Buffer too small");
/* Precalculate the SHA1 of the files's contents -- we'll match it to
* the provided SHA1 in the footer */
git_hash_buf(&checksum_calculated, buffer, buffer_size - INDEX_FOOTER_SIZE);
/* Parse header */
if (read_header(&header, buffer) < GIT_SUCCESS)
return git__throw(GIT_EOBJCORRUPTED, "Failed to parse index. Header is corrupted");
seek_forward(INDEX_HEADER_SIZE);
git_vector_clear(&index->entries);
/* Parse all the entries */
for (i = 0; i < header.entry_count && buffer_size > INDEX_FOOTER_SIZE; ++i) {
size_t entry_size;
git_index_entry *entry;
entry = git__malloc(sizeof(git_index_entry));
if (entry == NULL)
return GIT_ENOMEM;
entry_size = read_entry(entry, buffer, buffer_size);
/* 0 bytes read means an object corruption */
if (entry_size == 0)
return git__throw(GIT_EOBJCORRUPTED, "Failed to parse index. Entry size is zero");
if (git_vector_insert(&index->entries, entry) < GIT_SUCCESS)
return GIT_ENOMEM;
seek_forward(entry_size);
}
if (i != header.entry_count)
return git__throw(GIT_EOBJCORRUPTED, "Failed to parse index. Header entries changed while parsing");
/* There's still space for some extensions! */
while (buffer_size > INDEX_FOOTER_SIZE) {
size_t extension_size;
extension_size = read_extension(index, buffer, buffer_size);
/* see if we have read any bytes from the extension */
if (extension_size == 0)
return git__throw(GIT_EOBJCORRUPTED, "Failed to parse index. Extension size is zero");
seek_forward(extension_size);
}
if (buffer_size != INDEX_FOOTER_SIZE)
return git__throw(GIT_EOBJCORRUPTED, "Failed to parse index. Buffer size does not match index footer size");
/* 160-bit SHA-1 over the content of the index file before this checksum. */
git_oid_fromraw(&checksum_expected, (const unsigned char *)buffer);
if (git_oid_cmp(&checksum_calculated, &checksum_expected) != 0)
return git__throw(GIT_EOBJCORRUPTED, "Failed to parse index. Calculated checksum does not match expected checksum");
#undef seek_forward
/* force sorting in the vector: the entries are
* assured to be sorted on the index */
index->entries.sorted = 1;
return GIT_SUCCESS;
}
static int read_tree_internal(git_index_tree **out,
const char **buffer_in, const char *buffer_end, git_index_tree *parent)
{
git_index_tree *tree;
const char *name_start, *buffer;
long count;
int error = GIT_SUCCESS;
if ((tree = git__malloc(sizeof(git_index_tree))) == NULL)
return GIT_ENOMEM;
memset(tree, 0x0, sizeof(git_index_tree));
tree->parent = parent;
buffer = name_start = *buffer_in;
if ((buffer = memchr(buffer, '\0', buffer_end - buffer)) == NULL) {
error = GIT_EOBJCORRUPTED;
goto cleanup;
}
/* NUL-terminated tree name */
tree->name = git__strdup(name_start);
if (tree->name == NULL) {
error = GIT_ENOMEM;
goto cleanup;
}
if (++buffer >= buffer_end) {
error = GIT_EOBJCORRUPTED;
goto cleanup;
}
/* Blank-terminated ASCII decimal number of entries in this tree */
if (git__strtol32(&count, buffer, &buffer, 10) < GIT_SUCCESS || count < -1) {
error = GIT_EOBJCORRUPTED;
goto cleanup;
}
/* Invalidated TREE. Free the tree but report success */
if (count == -1) {
/* FIXME: return buffer_end or the end position for
* this single tree entry */
*buffer_in = buffer_end;
*out = NULL;
free_tree(tree); /* Needs to be done manually */
return GIT_SUCCESS;
}
tree->entries = count;
if (*buffer != ' ' || ++buffer >= buffer_end) {
error = GIT_EOBJCORRUPTED;
goto cleanup;
}
/* Number of children of the tree, newline-terminated */
if (git__strtol32(&count, buffer, &buffer, 10) < GIT_SUCCESS ||
count < 0) {
error = GIT_EOBJCORRUPTED;
goto cleanup;
}
tree->children_count = count;
if (*buffer != '\n' || ++buffer >= buffer_end) {
error = GIT_EOBJCORRUPTED;
goto cleanup;
}
/* 160-bit SHA-1 for this tree and it's children */
if (buffer + GIT_OID_RAWSZ > buffer_end) {
error = GIT_EOBJCORRUPTED;
goto cleanup;
}
git_oid_fromraw(&tree->oid, (const unsigned char *)buffer);
buffer += GIT_OID_RAWSZ;
/* Parse children: */
if (tree->children_count > 0) {
unsigned int i;
int err;
tree->children = git__malloc(tree->children_count * sizeof(git_index_tree *));
if (tree->children == NULL)
goto cleanup;
for (i = 0; i < tree->children_count; ++i) {
err = read_tree_internal(&tree->children[i], &buffer, buffer_end, tree);
if (err < GIT_SUCCESS)
goto cleanup;
}
}
*buffer_in = buffer;
*out = tree;
return GIT_SUCCESS;
cleanup:
free_tree(tree);
return error;
}
