int git_indexer_new(
git_indexer **out,
const char *prefix,
unsigned int mode,
git_odb *odb,
git_transfer_progress_cb progress_cb,
void *progress_payload)
{
git_indexer *idx;
git_buf path = GIT_BUF_INIT, tmp_path = GIT_BUF_INIT;
static const char suff[] = "/pack";
int error, fd = -1;
idx = (git_indexer *) git__calloc(1, sizeof(git_indexer));
GITERR_CHECK_ALLOC(idx);
idx->odb = odb;
idx->progress_cb = progress_cb;
idx->progress_payload = progress_payload;
idx->mode = mode ? mode : GIT_PACK_FILE_MODE;
git_hash_ctx_init(&idx->hash_ctx);
git_hash_ctx_init(&idx->trailer);
error = git_buf_joinpath(&path, prefix, suff);
if (error < 0)
goto cleanup;
fd = git_futils_mktmp(&tmp_path, git_buf_cstr(&path), idx->mode);
git_buf_free(&path);
if (fd < 0)
goto cleanup;
error = git_packfile_alloc(&idx->pack, git_buf_cstr(&tmp_path));
git_buf_free(&tmp_path);
if (error < 0)
goto cleanup;
idx->pack->mwf.fd = fd;
if ((error = git_mwindow_file_register(&idx->pack->mwf)) < 0)
goto cleanup;
*out = idx;
return 0;
cleanup:
if (fd != -1)
p_close(fd);
git_buf_free(&path);
git_buf_free(&tmp_path);
git__free(idx);
return -1;
}
int git_odb_open_wstream(
git_odb_stream **stream, git_odb *db, size_t size, git_otype type)
{
size_t i, writes = 0;
int error = GIT_ERROR;
git_hash_ctx *ctx = NULL;
assert(stream && db);
for (i = 0; i < db->backends.length && error < 0; ++i) {
backend_internal *internal = (backend_internal *) git_vector_get(&db->backends, i);
git_odb_backend *b = internal->backend;
/* we don't write in alternates! */
if (internal->is_alternate)
continue;
if (b->writestream != NULL) {
++writes;
error = b->writestream(stream, b, size, type);
} else if (b->write != NULL) {
++writes;
error = init_fake_wstream(stream, b, size, type);
}
}
if (error < 0) {
if (error == GIT_PASSTHROUGH)
error = 0;
else if (!writes)
error = git_odb__error_unsupported_in_backend("write object");
goto done;
}
ctx = (git_hash_ctx*) git__malloc(sizeof(git_hash_ctx));
GITERR_CHECK_ALLOC(ctx);
if ((error = git_hash_ctx_init(ctx)) < 0)
goto done;
hash_header(ctx, size, type);
(*stream)->hash_ctx = ctx;
(*stream)->declared_size = size;
(*stream)->received_bytes = 0;
done:
return error;
}
int git_hash_buf(git_oid *out, const void *data, size_t len)
{
git_hash_ctx ctx;
int error = 0;
if (git_hash_ctx_init(&ctx) < 0)
return -1;
if ((error = git_hash_update(&ctx, data, len)) >= 0)
error = git_hash_final(out, &ctx);
git_hash_ctx_cleanup(&ctx);
return error;
}
void test_odb_largefiles__streamread(void)
{
git_oid oid, read_oid;
git_odb_stream *stream;
char buf[10240];
char hdr[64];
size_t len, hdr_len, total = 0;
git_hash_ctx hash;
git_otype type;
int ret;
#ifndef GIT_ARCH_64
cl_skip();
#endif
if (!cl_is_env_set("GITTEST_INVASIVE_FS_SIZE") ||
!cl_is_env_set("GITTEST_SLOW"))
cl_skip();
writefile(&oid);
cl_git_pass(git_odb_open_rstream(&stream, &len, &type, odb, &oid));
cl_assert_equal_sz(LARGEFILE_SIZE, len);
cl_assert_equal_i(GIT_OBJ_BLOB, type);
cl_git_pass(git_hash_ctx_init(&hash));
cl_git_pass(git_odb__format_object_header(&hdr_len, hdr, sizeof(hdr), len, type));
cl_git_pass(git_hash_update(&hash, hdr, hdr_len));
while ((ret = git_odb_stream_read(stream, buf, 10240)) > 0) {
cl_git_pass(git_hash_update(&hash, buf, ret));
total += ret;
}
cl_assert_equal_sz(LARGEFILE_SIZE, total);
git_hash_final(&read_oid, &hash);
cl_assert_equal_oid(&oid, &read_oid);
git_hash_ctx_cleanup(&hash);
git_odb_stream_free(stream);
}
int git_odb__hashfd(git_oid *out, git_file fd, size_t size, git_otype type)
{
int hdr_len;
char hdr[64], buffer[2048];
git_hash_ctx ctx;
ssize_t read_len = 0;
int error = 0;
if (!git_object_typeisloose(type)) {
giterr_set(GITERR_INVALID, "Invalid object type for hash");
return -1;
}
if ((error = git_hash_ctx_init(&ctx)) < 0)
return -1;
hdr_len = git_odb__format_object_header(hdr, sizeof(hdr), size, type);
if ((error = git_hash_update(&ctx, hdr, hdr_len)) < 0)
goto done;
while (size > 0 && (read_len = p_read(fd, buffer, sizeof(buffer))) > 0) {
if ((error = git_hash_update(&ctx, buffer, read_len)) < 0)
goto done;
size -= read_len;
}
/* If p_read returned an error code, the read obviously failed.
* If size is not zero, the file was truncated after we originally
* stat'd it, so we consider this a read failure too */
if (read_len < 0 || size > 0) {
giterr_set(GITERR_OS, "Error reading file for hashing");
error = -1;
goto done;
}
error = git_hash_final(out, &ctx);
done:
git_hash_ctx_cleanup(&ctx);
return error;
}
int git_hash_vec(git_oid *out, git_buf_vec *vec, size_t n)
{
git_hash_ctx ctx;
size_t i;
int error = 0;
if (git_hash_ctx_init(&ctx) < 0)
return -1;
for (i = 0; i < n; i++) {
if ((error = git_hash_update(&ctx, vec[i].data, vec[i].len)) < 0)
goto done;
}
error = git_hash_final(out, &ctx);
done:
git_hash_ctx_cleanup(&ctx);
return error;
}
void test_object_raw_hash__hash_by_blocks(void)
{
git_hash_ctx ctx;
git_oid id1, id2;
cl_git_pass(git_hash_ctx_init(&ctx));
/* should already be init'd */
cl_git_pass(git_hash_update(&ctx, hello_text, strlen(hello_text)));
cl_git_pass(git_hash_final(&id2, &ctx));
cl_git_pass(git_oid_fromstr(&id1, hello_id));
cl_assert(git_oid_cmp(&id1, &id2) == 0);
/* reinit should permit reuse */
cl_git_pass(git_hash_init(&ctx));
cl_git_pass(git_hash_update(&ctx, bye_text, strlen(bye_text)));
cl_git_pass(git_hash_final(&id2, &ctx));
cl_git_pass(git_oid_fromstr(&id1, bye_id));
cl_assert(git_oid_cmp(&id1, &id2) == 0);
git_hash_ctx_cleanup(&ctx);
}
static int loose_backend__readstream(
git_odb_stream **stream_out,
size_t *len_out,
git_otype *type_out,
git_odb_backend *_backend,
const git_oid *oid)
{
loose_backend *backend;
loose_readstream *stream = NULL;
git_hash_ctx *hash_ctx = NULL;
git_buf object_path = GIT_BUF_INIT;
obj_hdr hdr;
int error = 0;
assert(stream_out && len_out && type_out && _backend && oid);
backend = (loose_backend *)_backend;
*stream_out = NULL;
*len_out = 0;
*type_out = GIT_OBJ_BAD;
if (locate_object(&object_path, backend, oid) < 0) {
error = git_odb__error_notfound("no matching loose object",
oid, GIT_OID_HEXSZ);
goto done;
}
stream = git__calloc(1, sizeof(loose_readstream));
GITERR_CHECK_ALLOC(stream);
hash_ctx = git__malloc(sizeof(git_hash_ctx));
GITERR_CHECK_ALLOC(hash_ctx);
if ((error = git_hash_ctx_init(hash_ctx)) < 0 ||
(error = git_futils_mmap_ro_file(&stream->map, object_path.ptr)) < 0 ||
(error = git_zstream_init(&stream->zstream, GIT_ZSTREAM_INFLATE)) < 0)
goto done;
/* check for a packlike loose object */
if (!is_zlib_compressed_data(stream->map.data, stream->map.len))
error = loose_backend__readstream_packlike(&hdr, stream);
else
error = loose_backend__readstream_standard(&hdr, stream);
if (error < 0)
goto done;
stream->stream.backend = _backend;
stream->stream.hash_ctx = hash_ctx;
stream->stream.read = &loose_backend__readstream_read;
stream->stream.free = &loose_backend__readstream_free;
*stream_out = (git_odb_stream *)stream;
*len_out = hdr.size;
*type_out = hdr.type;
done:
if (error < 0) {
git_futils_mmap_free(&stream->map);
git_zstream_free(&stream->zstream);
git_hash_ctx_cleanup(hash_ctx);
git__free(hash_ctx);
git__free(stream);
}
git_buf_dispose(&object_path);
return error;
}
int git_indexer_commit(git_indexer *idx, git_transfer_progress *stats)
{
git_mwindow *w = NULL;
unsigned int i, long_offsets = 0, left;
int error;
struct git_pack_idx_header hdr;
git_buf filename = GIT_BUF_INIT;
struct entry *entry;
git_oid trailer_hash, file_hash;
git_hash_ctx ctx;
git_filebuf index_file = {0};
void *packfile_trailer;
if (git_hash_ctx_init(&ctx) < 0)
return -1;
/* Test for this before resolve_deltas(), as it plays with idx->off */
if (idx->off < idx->pack->mwf.size - 20) {
giterr_set(GITERR_INDEXER, "Unexpected data at the end of the pack");
return -1;
}
packfile_trailer = git_mwindow_open(&idx->pack->mwf, &w, idx->pack->mwf.size - GIT_OID_RAWSZ, GIT_OID_RAWSZ, &left);
if (packfile_trailer == NULL) {
git_mwindow_close(&w);
goto on_error;
}
/* Compare the packfile trailer as it was sent to us and what we calculated */
git_oid_fromraw(&file_hash, (unsigned char*) packfile_trailer);
git_mwindow_close(&w);
git_hash_final(&trailer_hash, &idx->trailer);
if (git_oid_cmp(&file_hash, &trailer_hash)) {
giterr_set(GITERR_INDEXER, "packfile trailer mismatch");
return -1;
}
/* Freeze the number of deltas */
stats->total_deltas = stats->total_objects - stats->indexed_objects;
if ((error = resolve_deltas(idx, stats)) < 0)
return error;
if (stats->indexed_objects != stats->total_objects) {
giterr_set(GITERR_INDEXER, "early EOF");
return -1;
}
if (stats->local_objects > 0) {
if (update_header_and_rehash(idx, stats) < 0)
return -1;
git_hash_final(&trailer_hash, &idx->trailer);
write_at(idx, &trailer_hash, idx->pack->mwf.size - GIT_OID_RAWSZ, GIT_OID_RAWSZ);
}
git_vector_sort(&idx->objects);
git_buf_sets(&filename, idx->pack->pack_name);
git_buf_shorten(&filename, strlen("pack"));
git_buf_puts(&filename, "idx");
if (git_buf_oom(&filename))
return -1;
if (git_filebuf_open(&index_file, filename.ptr,
GIT_FILEBUF_HASH_CONTENTS, idx->mode) < 0)
goto on_error;
/* Write out the header */
hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
hdr.idx_version = htonl(2);
git_filebuf_write(&index_file, &hdr, sizeof(hdr));
/* Write out the fanout table */
for (i = 0; i < 256; ++i) {
uint32_t n = htonl(idx->fanout[i]);
git_filebuf_write(&index_file, &n, sizeof(n));
}
/* Write out the object names (SHA-1 hashes) */
git_vector_foreach(&idx->objects, i, entry, struct entry*) {
git_filebuf_write(&index_file, &entry->oid, sizeof(git_oid));
git_hash_update(&ctx, &entry->oid, GIT_OID_RAWSZ);
}
git_hash_final(&idx->hash, &ctx);
/* Write out the CRC32 values */
git_vector_foreach(&idx->objects, i, entry, struct entry*) {
git_filebuf_write(&index_file, &entry->crc, sizeof(uint32_t));
}
/* Write out the offsets */
git_vector_foreach(&idx->objects, i, entry, struct entry*) {
uint32_t n;
if (entry->offset == UINT32_MAX)
n = htonl(0x80000000 | long_offsets++);
else
n = htonl(entry->offset);
git_filebuf_write(&index_file, &n, sizeof(uint32_t));
}
/* Write out the long offsets */
git_vector_foreach(&idx->objects, i, entry, struct entry*) {
uint32_t split[2];
if (entry->offset != UINT32_MAX)
continue;
split[0] = htonl(entry->offset_long >> 32);
split[1] = htonl(entry->offset_long & 0xffffffff);
git_filebuf_write(&index_file, &split, sizeof(uint32_t) * 2);
}
/* Write out the packfile trailer to the index */
if (git_filebuf_write(&index_file, &trailer_hash, GIT_OID_RAWSZ) < 0)
goto on_error;
/* Write out the hash of the idx */
if (git_filebuf_hash(&trailer_hash, &index_file) < 0)
goto on_error;
git_filebuf_write(&index_file, &trailer_hash, sizeof(git_oid));
/* Figure out what the final name should be */
if (index_path(&filename, idx, ".idx") < 0)
goto on_error;
/* Commit file */
if (git_filebuf_commit_at(&index_file, filename.ptr) < 0)
goto on_error;
git_mwindow_free_all(&idx->pack->mwf);
/* We need to close the descriptor here so Windows doesn't choke on commit_at */
if (p_close(idx->pack->mwf.fd) < 0) {
giterr_set(GITERR_OS, "failed to close packfile");
goto on_error;
}
idx->pack->mwf.fd = -1;
if (index_path(&filename, idx, ".pack") < 0)
goto on_error;
/* And don't forget to rename the packfile to its new place. */
p_rename(idx->pack->pack_name, git_buf_cstr(&filename));
git_buf_free(&filename);
git_hash_ctx_cleanup(&ctx);
return 0;
on_error:
git_mwindow_free_all(&idx->pack->mwf);
git_filebuf_cleanup(&index_file);
git_buf_free(&filename);
git_hash_ctx_cleanup(&ctx);
return -1;
}
