static int store_object(git_indexer *idx)
{
int i, error;
khiter_t k;
git_oid oid;
struct entry *entry;
git_off_t entry_size;
struct git_pack_entry *pentry;
git_off_t entry_start = idx->entry_start;
entry = (struct entry*) git__calloc(1, sizeof(*entry));
GITERR_CHECK_ALLOC(entry);
pentry = (git_pack_entry*) git__calloc(1, sizeof(struct git_pack_entry));
GITERR_CHECK_ALLOC(pentry);
git_hash_final(&oid, &idx->hash_ctx);
entry_size = idx->off - entry_start;
if (entry_start > UINT31_MAX) {
entry->offset = UINT32_MAX;
entry->offset_long = entry_start;
} else {
entry->offset = (uint32_t)entry_start;
}
git_oid_cpy(&pentry->sha1, &oid);
pentry->offset = entry_start;
k = kh_put(oid, idx->pack->idx_cache, &pentry->sha1, &error);
if (!error) {
git__free(pentry);
giterr_set(GITERR_INDEXER, "cannot handle duplicate objects in pack");
goto on_error;
}
kh_value(idx->pack->idx_cache, k) = pentry;
git_oid_cpy(&entry->oid, &oid);
if (crc_object(&entry->crc, &idx->pack->mwf, entry_start, entry_size) < 0)
goto on_error;
/* Add the object to the list */
if (git_vector_insert(&idx->objects, entry) < 0)
goto on_error;
for (i = oid.id[0]; i < 256; ++i) {
idx->fanout[i]++;
}
return 0;
on_error:
git__free(entry);
return -1;
}
void test_object_raw_hash__hash_by_blocks(void)
{
git_hash_ctx *ctx;
git_oid id1, id2;
cl_assert((ctx = git_hash_new_ctx()) != NULL);
/* should already be init'd */
git_hash_update(ctx, hello_text, strlen(hello_text));
git_hash_final(&id2, ctx);
cl_git_pass(git_oid_fromstr(&id1, hello_id));
cl_assert(git_oid_cmp(&id1, &id2) == 0);
/* reinit should permit reuse */
git_hash_init(ctx);
git_hash_update(ctx, bye_text, strlen(bye_text));
git_hash_final(&id2, ctx);
cl_git_pass(git_oid_fromstr(&id1, bye_id));
cl_assert(git_oid_cmp(&id1, &id2) == 0);
git_hash_free_ctx(ctx);
}
void test_object_raw_hash__hash_by_blocks(void)
{
git_hash_ctx ctx;
git_oid id1, id2;
cl_git_pass(git_hash_ctx_init(&ctx));
/* should already be init'd */
cl_git_pass(git_hash_update(&ctx, hello_text, strlen(hello_text)));
cl_git_pass(git_hash_final(&id2, &ctx));
cl_git_pass(git_oid_fromstr(&id1, hello_id));
cl_assert(git_oid_cmp(&id1, &id2) == 0);
/* reinit should permit reuse */
cl_git_pass(git_hash_init(&ctx));
cl_git_pass(git_hash_update(&ctx, bye_text, strlen(bye_text)));
cl_git_pass(git_hash_final(&id2, &ctx));
cl_git_pass(git_oid_fromstr(&id1, bye_id));
cl_assert(git_oid_cmp(&id1, &id2) == 0);
git_hash_ctx_cleanup(&ctx);
}
int git_odb_stream_finalize_write(git_oid *out, git_odb_stream *stream)
{
if (stream->received_bytes != stream->declared_size)
return git_odb_stream__invalid_length(stream,
"stream_finalize_write()");
git_hash_final(out, (git_hash_ctx*) stream->hash_ctx);
if (git_odb_exists(stream->backend->odb, out))
return 0;
return stream->finalize_write(stream, out);
}
int git_filebuf_hash(git_oid *oid, git_filebuf *file)
{
assert(oid && file && file->digest);
flush_buffer(file);
if (verify_last_error(file) < 0)
return -1;
git_hash_final(oid, file->digest);
git_hash_free_ctx(file->digest);
file->digest = NULL;
return 0;
}
int git_filebuf_hash(git_oid *oid, git_filebuf *file)
{
int error;
assert(oid && file && file->digest);
if ((error = flush_buffer(file)) < GIT_SUCCESS)
return git__rethrow(error, "Failed to get hash for file");
git_hash_final(oid, file->digest);
git_hash_free_ctx(file->digest);
file->digest = NULL;
return GIT_SUCCESS;
}
int git_hash_buf(git_oid *out, const void *data, size_t len)
{
git_hash_ctx ctx;
int error = 0;
if (git_hash_ctx_init(&ctx) < 0)
return -1;
if ((error = git_hash_update(&ctx, data, len)) >= 0)
error = git_hash_final(out, &ctx);
git_hash_ctx_cleanup(&ctx);
return error;
}
int git_filebuf_hash(git_oid *oid, git_filebuf *file)
{
int error;
if (file->digest == NULL)
return GIT_ERROR;
if ((error = flush_buffer(file)) < GIT_SUCCESS)
return error;
git_hash_final(oid, file->digest);
git_hash_free_ctx(file->digest);
file->digest = NULL;
return GIT_SUCCESS;
}
void test_odb_largefiles__streamread(void)
{
git_oid oid, read_oid;
git_odb_stream *stream;
char buf[10240];
char hdr[64];
size_t len, hdr_len, total = 0;
git_hash_ctx hash;
git_otype type;
int ret;
#ifndef GIT_ARCH_64
cl_skip();
#endif
if (!cl_is_env_set("GITTEST_INVASIVE_FS_SIZE") ||
!cl_is_env_set("GITTEST_SLOW"))
cl_skip();
writefile(&oid);
cl_git_pass(git_odb_open_rstream(&stream, &len, &type, odb, &oid));
cl_assert_equal_sz(LARGEFILE_SIZE, len);
cl_assert_equal_i(GIT_OBJ_BLOB, type);
cl_git_pass(git_hash_ctx_init(&hash));
cl_git_pass(git_odb__format_object_header(&hdr_len, hdr, sizeof(hdr), len, type));
cl_git_pass(git_hash_update(&hash, hdr, hdr_len));
while ((ret = git_odb_stream_read(stream, buf, 10240)) > 0) {
cl_git_pass(git_hash_update(&hash, buf, ret));
total += ret;
}
cl_assert_equal_sz(LARGEFILE_SIZE, total);
git_hash_final(&read_oid, &hash);
cl_assert_equal_oid(&oid, &read_oid);
git_hash_ctx_cleanup(&hash);
git_odb_stream_free(stream);
}
int git_odb__hashfd(git_oid *out, git_file fd, size_t size, git_otype type)
{
int hdr_len;
char hdr[64], buffer[2048];
git_hash_ctx ctx;
ssize_t read_len = 0;
int error = 0;
if (!git_object_typeisloose(type)) {
giterr_set(GITERR_INVALID, "Invalid object type for hash");
return -1;
}
if ((error = git_hash_ctx_init(&ctx)) < 0)
return -1;
hdr_len = git_odb__format_object_header(hdr, sizeof(hdr), size, type);
if ((error = git_hash_update(&ctx, hdr, hdr_len)) < 0)
goto done;
while (size > 0 && (read_len = p_read(fd, buffer, sizeof(buffer))) > 0) {
if ((error = git_hash_update(&ctx, buffer, read_len)) < 0)
goto done;
size -= read_len;
}
/* If p_read returned an error code, the read obviously failed.
* If size is not zero, the file was truncated after we originally
* stat'd it, so we consider this a read failure too */
if (read_len < 0 || size > 0) {
giterr_set(GITERR_OS, "Error reading file for hashing");
error = -1;
goto done;
}
error = git_hash_final(out, &ctx);
done:
git_hash_ctx_cleanup(&ctx);
return error;
}
int git_hash_vec(git_oid *out, git_buf_vec *vec, size_t n)
{
git_hash_ctx ctx;
size_t i;
int error = 0;
if (git_hash_ctx_init(&ctx) < 0)
return -1;
for (i = 0; i < n; i++) {
if ((error = git_hash_update(&ctx, vec[i].data, vec[i].len)) < 0)
goto done;
}
error = git_hash_final(out, &ctx);
done:
git_hash_ctx_cleanup(&ctx);
return error;
}
int git_index__write(git_index *index, git_filelock *file)
{
static const char NULL_BYTES[] = {0, 0, 0, 0, 0, 0, 0, 0};
int error = 0;
unsigned int i;
git_hash_ctx *digest;
git_oid hash_final;
assert(index && file && file->is_locked);
if ((digest = git_hash_new_ctx()) == NULL)
return GIT_ENOMEM;
#define WRITE_WORD(_word) {\
uint32_t network_word = htonl((_word));\
git_filelock_write(file, &network_word, 4);\
git_hash_update(digest, &network_word, 4);\
}
#define WRITE_SHORT(_shrt) {\
uint16_t network_shrt = htons((_shrt));\
git_filelock_write(file, &network_shrt, 2);\
git_hash_update(digest, &network_shrt, 2);\
}
#define WRITE_BYTES(_bytes, _n) {\
git_filelock_write(file, _bytes, _n);\
git_hash_update(digest, _bytes, _n);\
}
WRITE_BYTES(INDEX_HEADER_SIG, 4);
WRITE_WORD(INDEX_VERSION_NUMBER);
WRITE_WORD(index->entry_count);
for (i = 0; i < index->entry_count; ++i) {
git_index_entry *entry;
size_t path_length, padding;
entry = &index->entries[i];
path_length = strlen(entry->path);
WRITE_WORD(entry->ctime.seconds);
WRITE_WORD(entry->ctime.nanoseconds);
WRITE_WORD(entry->mtime.seconds);
WRITE_WORD(entry->mtime.nanoseconds);
WRITE_WORD(entry->dev);
WRITE_WORD(entry->ino);
WRITE_WORD(entry->mode);
WRITE_WORD(entry->uid);
WRITE_WORD(entry->gid);
WRITE_WORD(entry->file_size);
WRITE_BYTES(entry->oid.id, GIT_OID_RAWSZ);
WRITE_SHORT(entry->flags);
if (entry->flags & GIT_IDXENTRY_EXTENDED) {
WRITE_SHORT(entry->flags_extended);
padding = long_entry_padding(path_length);
} else
padding = short_entry_padding(path_length);
WRITE_BYTES(entry->path, path_length);
WRITE_BYTES(NULL_BYTES, padding);
}
#undef WRITE_WORD
#undef WRITE_BYTES
#undef WRITE_SHORT
#undef WRITE_FLAGS
/* TODO: write extensions (tree cache) */
git_hash_final(&hash_final, digest);
git_hash_free_ctx(digest);
git_filelock_write(file, hash_final.id, GIT_OID_RAWSZ);
return error;
}
int git_indexer_commit(git_indexer *idx, git_transfer_progress *stats)
{
git_mwindow *w = NULL;
unsigned int i, long_offsets = 0, left;
int error;
struct git_pack_idx_header hdr;
git_buf filename = GIT_BUF_INIT;
struct entry *entry;
git_oid trailer_hash, file_hash;
git_hash_ctx ctx;
git_filebuf index_file = {0};
void *packfile_trailer;
if (git_hash_ctx_init(&ctx) < 0)
return -1;
/* Test for this before resolve_deltas(), as it plays with idx->off */
if (idx->off < idx->pack->mwf.size - 20) {
giterr_set(GITERR_INDEXER, "Unexpected data at the end of the pack");
return -1;
}
packfile_trailer = git_mwindow_open(&idx->pack->mwf, &w, idx->pack->mwf.size - GIT_OID_RAWSZ, GIT_OID_RAWSZ, &left);
if (packfile_trailer == NULL) {
git_mwindow_close(&w);
goto on_error;
}
/* Compare the packfile trailer as it was sent to us and what we calculated */
git_oid_fromraw(&file_hash, (unsigned char*) packfile_trailer);
git_mwindow_close(&w);
git_hash_final(&trailer_hash, &idx->trailer);
if (git_oid_cmp(&file_hash, &trailer_hash)) {
giterr_set(GITERR_INDEXER, "packfile trailer mismatch");
return -1;
}
/* Freeze the number of deltas */
stats->total_deltas = stats->total_objects - stats->indexed_objects;
if ((error = resolve_deltas(idx, stats)) < 0)
return error;
if (stats->indexed_objects != stats->total_objects) {
giterr_set(GITERR_INDEXER, "early EOF");
return -1;
}
if (stats->local_objects > 0) {
if (update_header_and_rehash(idx, stats) < 0)
return -1;
git_hash_final(&trailer_hash, &idx->trailer);
write_at(idx, &trailer_hash, idx->pack->mwf.size - GIT_OID_RAWSZ, GIT_OID_RAWSZ);
}
git_vector_sort(&idx->objects);
git_buf_sets(&filename, idx->pack->pack_name);
git_buf_shorten(&filename, strlen("pack"));
git_buf_puts(&filename, "idx");
if (git_buf_oom(&filename))
return -1;
if (git_filebuf_open(&index_file, filename.ptr,
GIT_FILEBUF_HASH_CONTENTS, idx->mode) < 0)
goto on_error;
/* Write out the header */
hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
hdr.idx_version = htonl(2);
git_filebuf_write(&index_file, &hdr, sizeof(hdr));
/* Write out the fanout table */
for (i = 0; i < 256; ++i) {
uint32_t n = htonl(idx->fanout[i]);
git_filebuf_write(&index_file, &n, sizeof(n));
}
/* Write out the object names (SHA-1 hashes) */
git_vector_foreach(&idx->objects, i, entry, struct entry*) {
git_filebuf_write(&index_file, &entry->oid, sizeof(git_oid));
git_hash_update(&ctx, &entry->oid, GIT_OID_RAWSZ);
}
git_hash_final(&idx->hash, &ctx);
/* Write out the CRC32 values */
git_vector_foreach(&idx->objects, i, entry, struct entry*) {
git_filebuf_write(&index_file, &entry->crc, sizeof(uint32_t));
}
/* Write out the offsets */
git_vector_foreach(&idx->objects, i, entry, struct entry*) {
uint32_t n;
if (entry->offset == UINT32_MAX)
n = htonl(0x80000000 | long_offsets++);
else
n = htonl(entry->offset);
git_filebuf_write(&index_file, &n, sizeof(uint32_t));
}
/* Write out the long offsets */
git_vector_foreach(&idx->objects, i, entry, struct entry*) {
uint32_t split[2];
if (entry->offset != UINT32_MAX)
continue;
split[0] = htonl(entry->offset_long >> 32);
split[1] = htonl(entry->offset_long & 0xffffffff);
git_filebuf_write(&index_file, &split, sizeof(uint32_t) * 2);
}
/* Write out the packfile trailer to the index */
if (git_filebuf_write(&index_file, &trailer_hash, GIT_OID_RAWSZ) < 0)
goto on_error;
/* Write out the hash of the idx */
if (git_filebuf_hash(&trailer_hash, &index_file) < 0)
goto on_error;
git_filebuf_write(&index_file, &trailer_hash, sizeof(git_oid));
/* Figure out what the final name should be */
if (index_path(&filename, idx, ".idx") < 0)
goto on_error;
/* Commit file */
if (git_filebuf_commit_at(&index_file, filename.ptr) < 0)
goto on_error;
git_mwindow_free_all(&idx->pack->mwf);
/* We need to close the descriptor here so Windows doesn't choke on commit_at */
if (p_close(idx->pack->mwf.fd) < 0) {
giterr_set(GITERR_OS, "failed to close packfile");
goto on_error;
}
idx->pack->mwf.fd = -1;
if (index_path(&filename, idx, ".pack") < 0)
goto on_error;
/* And don't forget to rename the packfile to its new place. */
p_rename(idx->pack->pack_name, git_buf_cstr(&filename));
git_buf_free(&filename);
git_hash_ctx_cleanup(&ctx);
return 0;
on_error:
git_mwindow_free_all(&idx->pack->mwf);
git_filebuf_cleanup(&index_file);
git_buf_free(&filename);
git_hash_ctx_cleanup(&ctx);
return -1;
}
