static int fix_thin_pack(git_indexer *idx, git_transfer_progress *stats)
{
int error, found_ref_delta = 0;
unsigned int i;
struct delta_info *delta;
size_t size;
git_otype type;
git_mwindow *w = NULL;
git_off_t curpos = 0;
unsigned char *base_info;
unsigned int left = 0;
git_oid base;
assert(git_vector_length(&idx->deltas) > 0);
if (idx->odb == NULL) {
giterr_set(GITERR_INDEXER, "cannot fix a thin pack without an ODB");
return -1;
}
/* Loop until we find the first REF delta */
git_vector_foreach(&idx->deltas, i, delta, delta_info*) {
if (!delta)
continue;
curpos = delta->delta_off;
error = git_packfile_unpack_header(&size, &type, &idx->pack->mwf, &w, &curpos);
git_mwindow_close(&w);
if (error < 0)
return error;
if (type == GIT_OBJ_REF_DELTA) {
found_ref_delta = 1;
break;
}
}
if (!found_ref_delta) {
giterr_set(GITERR_INDEXER, "no REF_DELTA found, cannot inject object");
return -1;
}
/* curpos now points to the base information, which is an OID */
base_info = git_mwindow_open(&idx->pack->mwf, &w, curpos, GIT_OID_RAWSZ, &left);
if (base_info == NULL) {
giterr_set(GITERR_INDEXER, "failed to map delta information");
return -1;
}
git_oid_fromraw(&base, base_info);
git_mwindow_close(&w);
if (inject_object(idx, &base) < 0)
return -1;
stats->local_objects++;
return 0;
}
/*
* Close the least recently used window. You should check to see if
* the file descriptors need closing from time to time.
*/
static int git_mwindow_close_lru(git_mwindow_file *mwf)
{
unsigned int i;
git_mwindow *lru_w = NULL, *lru_l = NULL;
/* FIMXE: Does this give us any advantage? */
if(mwf->windows)
git_mwindow_scan_lru(mwf, &lru_w, &lru_l);
for (i = 0; i < ctl.windowfiles.length; ++i) {
git_mwindow_scan_lru(git_vector_get(&ctl.windowfiles, i), &lru_w, &lru_l);
}
if (lru_w) {
git_mwindow_close(&lru_w);
ctl.mapped -= lru_w->window_map.len;
git_futils_mmap_free(&lru_w->window_map);
if (lru_l)
lru_l->next = lru_w->next;
else
mwf->windows = lru_w->next;
free(lru_w);
ctl.open_windows--;
return GIT_SUCCESS;
}
return git__throw(GIT_ERROR, "Failed to close memory window. Couln't find LRU");
}
/* In order to create the packfile stream, we need to skip over the delta base description */
static int advance_delta_offset(git_indexer *idx, git_otype type)
{
git_mwindow *w = NULL;
assert(type == GIT_OBJ_REF_DELTA || type == GIT_OBJ_OFS_DELTA);
if (type == GIT_OBJ_REF_DELTA) {
idx->off += GIT_OID_RAWSZ;
} else {
git_off_t base_off = get_delta_base(idx->pack, &w, &idx->off, type, idx->entry_start);
git_mwindow_close(&w);
if (base_off < 0)
return (int)base_off;
}
return 0;
}
static int update_header_and_rehash(git_indexer *idx, git_transfer_progress *stats)
{
void *ptr;
size_t chunk = 1024*1024;
git_off_t hashed = 0;
git_mwindow *w = NULL;
git_mwindow_file *mwf;
unsigned int left;
mwf = &idx->pack->mwf;
git_hash_init(&idx->trailer);
/* Update the header to include the numer of local objects we injected */
idx->hdr.hdr_entries = htonl(stats->total_objects + stats->local_objects);
if (write_at(idx, &idx->hdr, 0, sizeof(struct git_pack_header)) < 0)
return -1;
/*
* We now use the same technique as before to determine the
* hash. We keep reading up to the end and let
* hash_partially() keep the existing trailer out of the
* calculation.
*/
git_mwindow_free_all(mwf);
idx->inbuf_len = 0;
while (hashed < mwf->size) {
ptr = git_mwindow_open(mwf, &w, hashed, chunk, &left);
if (ptr == NULL)
return -1;
hash_partially(idx, (unsigned char*) ptr, left);
hashed += left;
git_mwindow_close(&w);
}
return 0;
}
static int crc_object(uint32_t *crc_out, git_mwindow_file *mwf, git_off_t start, git_off_t size)
{
void *ptr;
uint32_t crc;
unsigned int left, len;
git_mwindow *w = NULL;
crc = crc32(0L, Z_NULL, 0);
while (size) {
ptr = git_mwindow_open(mwf, &w, start, (size_t)size, &left);
if (ptr == NULL)
return -1;
len = min(left, (unsigned int)size);
crc = crc32(crc, (unsigned char*) ptr, len);
size -= len;
start += len;
git_mwindow_close(&w);
}
*crc_out = htonl(crc);
return 0;
}
int git_indexer_commit(git_indexer *idx, git_transfer_progress *stats)
{
git_mwindow *w = NULL;
unsigned int i, long_offsets = 0, left;
int error;
struct git_pack_idx_header hdr;
git_buf filename = GIT_BUF_INIT;
struct entry *entry;
git_oid trailer_hash, file_hash;
git_hash_ctx ctx;
git_filebuf index_file = {0};
void *packfile_trailer;
if (git_hash_ctx_init(&ctx) < 0)
return -1;
/* Test for this before resolve_deltas(), as it plays with idx->off */
if (idx->off < idx->pack->mwf.size - 20) {
giterr_set(GITERR_INDEXER, "Unexpected data at the end of the pack");
return -1;
}
packfile_trailer = git_mwindow_open(&idx->pack->mwf, &w, idx->pack->mwf.size - GIT_OID_RAWSZ, GIT_OID_RAWSZ, &left);
if (packfile_trailer == NULL) {
git_mwindow_close(&w);
goto on_error;
}
/* Compare the packfile trailer as it was sent to us and what we calculated */
git_oid_fromraw(&file_hash, (unsigned char*) packfile_trailer);
git_mwindow_close(&w);
git_hash_final(&trailer_hash, &idx->trailer);
if (git_oid_cmp(&file_hash, &trailer_hash)) {
giterr_set(GITERR_INDEXER, "packfile trailer mismatch");
return -1;
}
/* Freeze the number of deltas */
stats->total_deltas = stats->total_objects - stats->indexed_objects;
if ((error = resolve_deltas(idx, stats)) < 0)
return error;
if (stats->indexed_objects != stats->total_objects) {
giterr_set(GITERR_INDEXER, "early EOF");
return -1;
}
if (stats->local_objects > 0) {
if (update_header_and_rehash(idx, stats) < 0)
return -1;
git_hash_final(&trailer_hash, &idx->trailer);
write_at(idx, &trailer_hash, idx->pack->mwf.size - GIT_OID_RAWSZ, GIT_OID_RAWSZ);
}
git_vector_sort(&idx->objects);
git_buf_sets(&filename, idx->pack->pack_name);
git_buf_shorten(&filename, strlen("pack"));
git_buf_puts(&filename, "idx");
if (git_buf_oom(&filename))
return -1;
if (git_filebuf_open(&index_file, filename.ptr,
GIT_FILEBUF_HASH_CONTENTS, idx->mode) < 0)
goto on_error;
/* Write out the header */
hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
hdr.idx_version = htonl(2);
git_filebuf_write(&index_file, &hdr, sizeof(hdr));
/* Write out the fanout table */
for (i = 0; i < 256; ++i) {
uint32_t n = htonl(idx->fanout[i]);
git_filebuf_write(&index_file, &n, sizeof(n));
}
/* Write out the object names (SHA-1 hashes) */
git_vector_foreach(&idx->objects, i, entry, struct entry*) {
git_filebuf_write(&index_file, &entry->oid, sizeof(git_oid));
git_hash_update(&ctx, &entry->oid, GIT_OID_RAWSZ);
}
git_hash_final(&idx->hash, &ctx);
/* Write out the CRC32 values */
git_vector_foreach(&idx->objects, i, entry, struct entry*) {
git_filebuf_write(&index_file, &entry->crc, sizeof(uint32_t));
}
/* Write out the offsets */
git_vector_foreach(&idx->objects, i, entry, struct entry*) {
uint32_t n;
if (entry->offset == UINT32_MAX)
n = htonl(0x80000000 | long_offsets++);
else
n = htonl(entry->offset);
git_filebuf_write(&index_file, &n, sizeof(uint32_t));
}
/* Write out the long offsets */
git_vector_foreach(&idx->objects, i, entry, struct entry*) {
uint32_t split[2];
if (entry->offset != UINT32_MAX)
continue;
split[0] = htonl(entry->offset_long >> 32);
split[1] = htonl(entry->offset_long & 0xffffffff);
git_filebuf_write(&index_file, &split, sizeof(uint32_t) * 2);
}
/* Write out the packfile trailer to the index */
if (git_filebuf_write(&index_file, &trailer_hash, GIT_OID_RAWSZ) < 0)
goto on_error;
/* Write out the hash of the idx */
if (git_filebuf_hash(&trailer_hash, &index_file) < 0)
goto on_error;
git_filebuf_write(&index_file, &trailer_hash, sizeof(git_oid));
/* Figure out what the final name should be */
if (index_path(&filename, idx, ".idx") < 0)
goto on_error;
/* Commit file */
if (git_filebuf_commit_at(&index_file, filename.ptr) < 0)
goto on_error;
git_mwindow_free_all(&idx->pack->mwf);
/* We need to close the descriptor here so Windows doesn't choke on commit_at */
if (p_close(idx->pack->mwf.fd) < 0) {
giterr_set(GITERR_OS, "failed to close packfile");
goto on_error;
}
idx->pack->mwf.fd = -1;
if (index_path(&filename, idx, ".pack") < 0)
goto on_error;
/* And don't forget to rename the packfile to its new place. */
p_rename(idx->pack->pack_name, git_buf_cstr(&filename));
git_buf_free(&filename);
git_hash_ctx_cleanup(&ctx);
return 0;
on_error:
git_mwindow_free_all(&idx->pack->mwf);
git_filebuf_cleanup(&index_file);
git_buf_free(&filename);
git_hash_ctx_cleanup(&ctx);
return -1;
}
int git_indexer_append(git_indexer *idx, const void *data, size_t size, git_transfer_progress *stats)
{
int error = -1;
size_t processed;
struct git_pack_header *hdr = &idx->hdr;
git_mwindow_file *mwf = &idx->pack->mwf;
assert(idx && data && stats);
processed = stats->indexed_objects;
if ((error = append_to_pack(idx, data, size)) < 0)
return error;
hash_partially(idx, (unsigned char*) data, (int)size);
/* Make sure we set the new size of the pack */
idx->pack->mwf.size += size;
if (!idx->parsed_header) {
unsigned int total_objects;
if ((unsigned)idx->pack->mwf.size < sizeof(struct git_pack_header))
return 0;
if ((error = parse_header(&idx->hdr, idx->pack)) < 0)
return error;
idx->parsed_header = 1;
idx->nr_objects = ntohl(hdr->hdr_entries);
idx->off = sizeof(struct git_pack_header);
/* for now, limit to 2^32 objects */
assert(idx->nr_objects == (size_t)((unsigned int)idx->nr_objects));
if (idx->nr_objects == (size_t)((unsigned int)idx->nr_objects))
total_objects = (unsigned int)idx->nr_objects;
else
total_objects = UINT_MAX;
idx->pack->idx_cache = git_oidmap_alloc();
GITERR_CHECK_ALLOC(idx->pack->idx_cache);
idx->pack->has_cache = 1;
if (git_vector_init(&idx->objects, total_objects, objects_cmp) < 0)
return -1;
if (git_vector_init(&idx->deltas, total_objects / 2, NULL) < 0)
return -1;
stats->received_objects = 0;
stats->local_objects = 0;
stats->total_deltas = 0;
stats->indexed_deltas = 0;
processed = stats->indexed_objects = 0;
stats->total_objects = total_objects;
if ((error = do_progress_callback(idx, stats)) != 0)
return error;
}
/* Now that we have data in the pack, let's try to parse it */
/* As the file grows any windows we try to use will be out of date */
git_mwindow_free_all(mwf);
while (processed < idx->nr_objects) {
git_packfile_stream *stream = &idx->stream;
git_off_t entry_start = idx->off;
size_t entry_size;
git_otype type;
git_mwindow *w = NULL;
if (idx->pack->mwf.size <= idx->off + 20)
return 0;
if (!idx->have_stream) {
error = git_packfile_unpack_header(&entry_size, &type, mwf, &w, &idx->off);
if (error == GIT_EBUFS) {
idx->off = entry_start;
return 0;
}
if (error < 0)
goto on_error;
git_mwindow_close(&w);
idx->entry_start = entry_start;
git_hash_init(&idx->hash_ctx);
if (type == GIT_OBJ_REF_DELTA || type == GIT_OBJ_OFS_DELTA) {
error = advance_delta_offset(idx, type);
if (error == GIT_EBUFS) {
idx->off = entry_start;
return 0;
}
if (error < 0)
goto on_error;
idx->have_delta = 1;
} else {
idx->have_delta = 0;
hash_header(&idx->hash_ctx, entry_size, type);
}
idx->have_stream = 1;
error = git_packfile_stream_open(stream, idx->pack, idx->off);
if (error < 0)
goto on_error;
}
if (idx->have_delta) {
error = read_object_stream(idx, stream);
} else {
error = hash_object_stream(idx, stream);
}
idx->off = stream->curpos;
if (error == GIT_EBUFS)
return 0;
/* We want to free the stream reasorces no matter what here */
idx->have_stream = 0;
git_packfile_stream_free(stream);
if (error < 0)
goto on_error;
if (idx->have_delta) {
error = store_delta(idx);
} else {
error = store_object(idx);
}
if (error < 0)
goto on_error;
if (!idx->have_delta) {
stats->indexed_objects = (unsigned int)++processed;
}
stats->received_objects++;
if ((error = do_progress_callback(idx, stats)) != 0)
goto on_error;
}
return 0;
on_error:
git_mwindow_free_all(mwf);
return error;
}
