/* Hash everything but the last 20B of input */
static void hash_partially(git_indexer *idx, const uint8_t *data, size_t size)
{
size_t to_expell, to_keep;
if (size == 0)
return;
/* Easy case, dump the buffer and the data minus the last 20 bytes */
if (size >= GIT_OID_RAWSZ) {
git_hash_update(&idx->trailer, idx->inbuf, idx->inbuf_len);
git_hash_update(&idx->trailer, data, size - GIT_OID_RAWSZ);
data += size - GIT_OID_RAWSZ;
memcpy(idx->inbuf, data, GIT_OID_RAWSZ);
idx->inbuf_len = GIT_OID_RAWSZ;
return;
}
/* We can just append */
if (idx->inbuf_len + size <= GIT_OID_RAWSZ) {
memcpy(idx->inbuf + idx->inbuf_len, data, size);
idx->inbuf_len += size;
return;
}
/* We need to partially drain the buffer and then append */
to_keep = GIT_OID_RAWSZ - size;
to_expell = idx->inbuf_len - to_keep;
git_hash_update(&idx->trailer, idx->inbuf, to_expell);
memmove(idx->inbuf, idx->inbuf + to_expell, to_keep);
memcpy(idx->inbuf + to_keep, data, size);
idx->inbuf_len += size - to_expell;
}
static int write_deflate(git_filebuf *file, const void *source, size_t len)
{
int result = Z_OK;
z_stream *zs = &file->zs;
if (len > 0 || file->flush_mode == Z_FINISH) {
zs->next_in = (void *)source;
zs->avail_in = len;
do {
int have;
zs->next_out = file->z_buf;
zs->avail_out = file->buf_size;
result = deflate(zs, file->flush_mode);
assert(result != Z_STREAM_ERROR);
have = file->buf_size - zs->avail_out;
if (gitfo_write(file->fd, file->z_buf, have) < GIT_SUCCESS)
return git__throw(GIT_EOSERR, "Failed to write to file");
} while (zs->avail_out == 0);
assert(zs->avail_in == 0);
if (file->digest)
git_hash_update(file->digest, source, len);
}
return GIT_SUCCESS;
}
int git_filebuf_write(git_filebuf *file, void *buff, size_t len)
{
int error;
unsigned char *buf = buff;
for (;;) {
size_t space_left = file->buf_size - file->buf_pos;
/* cache if it's small */
if (space_left > len) {
add_to_cache(file, buf, len);
return GIT_SUCCESS;
}
/* flush the cache if it doesn't fit */
if (file->buf_pos > 0) {
add_to_cache(file, buf, space_left);
if ((error = flush_buffer(file)) < GIT_SUCCESS)
return error;
len -= space_left;
buf += space_left;
}
/* write too-large chunks immediately */
if (len > file->buf_size) {
error = gitfo_write(file->fd, buf, len);
if (file->digest)
git_hash_update(file->digest, buf, len);
}
}
}
static void hash_header(git_hash_ctx *ctx, size_t size, git_otype type)
{
char header[64];
int hdrlen;
hdrlen = git_odb__format_object_header(header, sizeof(header), size, type);
git_hash_update(ctx, header, hdrlen);
}
static void hash_header(git_hash_ctx *ctx, git_off_t len, git_otype type)
{
char buffer[64];
size_t hdrlen;
hdrlen = git_odb__format_object_header(buffer, sizeof(buffer), (size_t)len, type);
git_hash_update(ctx, buffer, hdrlen);
}
void test_odb_largefiles__streamread(void)
{
git_oid oid, read_oid;
git_odb_stream *stream;
char buf[10240];
char hdr[64];
size_t len, hdr_len, total = 0;
git_hash_ctx hash;
git_otype type;
int ret;
#ifndef GIT_ARCH_64
cl_skip();
#endif
if (!cl_is_env_set("GITTEST_INVASIVE_FS_SIZE") ||
!cl_is_env_set("GITTEST_SLOW"))
cl_skip();
writefile(&oid);
cl_git_pass(git_odb_open_rstream(&stream, &len, &type, odb, &oid));
cl_assert_equal_sz(LARGEFILE_SIZE, len);
cl_assert_equal_i(GIT_OBJ_BLOB, type);
cl_git_pass(git_hash_ctx_init(&hash));
cl_git_pass(git_odb__format_object_header(&hdr_len, hdr, sizeof(hdr), len, type));
cl_git_pass(git_hash_update(&hash, hdr, hdr_len));
while ((ret = git_odb_stream_read(stream, buf, 10240)) > 0) {
cl_git_pass(git_hash_update(&hash, buf, ret));
total += ret;
}
cl_assert_equal_sz(LARGEFILE_SIZE, total);
git_hash_final(&read_oid, &hash);
cl_assert_equal_oid(&oid, &read_oid);
git_hash_ctx_cleanup(&hash);
git_odb_stream_free(stream);
}
int git_odb__hashfd(git_oid *out, git_file fd, size_t size, git_otype type)
{
int hdr_len;
char hdr[64], buffer[2048];
git_hash_ctx ctx;
ssize_t read_len = 0;
int error = 0;
if (!git_object_typeisloose(type)) {
giterr_set(GITERR_INVALID, "Invalid object type for hash");
return -1;
}
if ((error = git_hash_ctx_init(&ctx)) < 0)
return -1;
hdr_len = git_odb__format_object_header(hdr, sizeof(hdr), size, type);
if ((error = git_hash_update(&ctx, hdr, hdr_len)) < 0)
goto done;
while (size > 0 && (read_len = p_read(fd, buffer, sizeof(buffer))) > 0) {
if ((error = git_hash_update(&ctx, buffer, read_len)) < 0)
goto done;
size -= read_len;
}
/* If p_read returned an error code, the read obviously failed.
* If size is not zero, the file was truncated after we originally
* stat'd it, so we consider this a read failure too */
if (read_len < 0 || size > 0) {
giterr_set(GITERR_OS, "Error reading file for hashing");
error = -1;
goto done;
}
error = git_hash_final(out, &ctx);
done:
git_hash_ctx_cleanup(&ctx);
return error;
}
static int lock_file(git_filebuf *file, int flags)
{
if (git_path_exists(file->path_lock) == true) {
if (flags & GIT_FILEBUF_FORCE)
p_unlink(file->path_lock);
else {
giterr_clear(); /* actual OS error code just confuses */
giterr_set(GITERR_OS,
"Failed to lock file '%s' for writing", file->path_lock);
return -1;
}
}
/* create path to the file buffer is required */
if (flags & GIT_FILEBUF_FORCE) {
/* XXX: Should dirmode here be configurable? Or is 0777 always fine? */
file->fd = git_futils_creat_locked_withpath(file->path_lock, 0777, GIT_LOCK_FILE_MODE);
} else {
file->fd = git_futils_creat_locked(file->path_lock, GIT_LOCK_FILE_MODE);
}
if (file->fd < 0)
return -1;
file->fd_is_open = true;
if ((flags & GIT_FILEBUF_APPEND) && git_path_exists(file->path_original) == true) {
git_file source;
char buffer[2048];
ssize_t read_bytes;
source = p_open(file->path_original, O_RDONLY);
if (source < 0) {
giterr_set(GITERR_OS,
"Failed to open file '%s' for reading",
file->path_original);
return -1;
}
while ((read_bytes = p_read(source, buffer, sizeof(buffer))) > 0) {
p_write(file->fd, buffer, read_bytes);
if (file->digest)
git_hash_update(file->digest, buffer, read_bytes);
}
p_close(source);
if (read_bytes < 0) {
giterr_set(GITERR_OS, "Failed to read file '%s'", file->path_original);
return -1;
}
}
return 0;
}
int git_hash_final(git_oid *out, git_hash_ctx *ctx)
{
static const unsigned char pad[64] = { 0x80 };
unsigned int padlen[2];
int i;
/* Pad with a binary 1 (ie 0x80), then zeroes, then length */
padlen[0] = htonl((uint32_t)(ctx->size >> 29));
padlen[1] = htonl((uint32_t)(ctx->size << 3));
i = ctx->size & 63;
git_hash_update(ctx, pad, 1+ (63 & (55 - i)));
git_hash_update(ctx, padlen, 8);
/* Output hash */
for (i = 0; i < 5; i++)
put_be32(out->id + i*4, ctx->H[i]);
return 0;
}
int git_odb_stream_write(git_odb_stream *stream, const char *buffer, size_t len)
{
git_hash_update((git_hash_ctx*)stream->hash_ctx, buffer, len);
stream->received_bytes += len;
if (stream->received_bytes > stream->declared_size)
return git_odb_stream__invalid_length(stream,
"stream_write()");
return stream->write(stream, buffer, len);
}
static int write_normal(git_filebuf *file, const void *source, size_t len)
{
int result = 0;
if (len > 0) {
result = gitfo_write(file->fd, (void *)source, len);
if (file->digest)
git_hash_update(file->digest, source, len);
}
return result;
}
void test_object_raw_hash__hash_by_blocks(void)
{
git_hash_ctx *ctx;
git_oid id1, id2;
cl_assert((ctx = git_hash_new_ctx()) != NULL);
/* should already be init'd */
git_hash_update(ctx, hello_text, strlen(hello_text));
git_hash_final(&id2, ctx);
cl_git_pass(git_oid_fromstr(&id1, hello_id));
cl_assert(git_oid_cmp(&id1, &id2) == 0);
/* reinit should permit reuse */
git_hash_init(ctx);
git_hash_update(ctx, bye_text, strlen(bye_text));
git_hash_final(&id2, ctx);
cl_git_pass(git_oid_fromstr(&id1, bye_id));
cl_assert(git_oid_cmp(&id1, &id2) == 0);
git_hash_free_ctx(ctx);
}
void test_object_raw_hash__hash_by_blocks(void)
{
git_hash_ctx ctx;
git_oid id1, id2;
cl_git_pass(git_hash_ctx_init(&ctx));
/* should already be init'd */
cl_git_pass(git_hash_update(&ctx, hello_text, strlen(hello_text)));
cl_git_pass(git_hash_final(&id2, &ctx));
cl_git_pass(git_oid_fromstr(&id1, hello_id));
cl_assert(git_oid_cmp(&id1, &id2) == 0);
/* reinit should permit reuse */
cl_git_pass(git_hash_init(&ctx));
cl_git_pass(git_hash_update(&ctx, bye_text, strlen(bye_text)));
cl_git_pass(git_hash_final(&id2, &ctx));
cl_git_pass(git_oid_fromstr(&id1, bye_id));
cl_assert(git_oid_cmp(&id1, &id2) == 0);
git_hash_ctx_cleanup(&ctx);
}
static int flush_buffer(git_filebuf *file)
{
int result = GIT_SUCCESS;
if (file->buf_pos > 0) {
result = gitfo_write(file->fd, file->buffer, file->buf_pos);
if (file->digest)
git_hash_update(file->digest, file->buffer, file->buf_pos);
file->buf_pos = 0;
}
return result;
}
static int write_normal(git_filebuf *file, void *source, size_t len)
{
if (len > 0) {
if (p_write(file->fd, (void *)source, len) < 0) {
file->last_error = BUFERR_WRITE;
return -1;
}
if (file->digest)
git_hash_update(file->digest, source, len);
}
return 0;
}
int git_hash_buf(git_oid *out, const void *data, size_t len)
{
git_hash_ctx ctx;
int error = 0;
if (git_hash_ctx_init(&ctx) < 0)
return -1;
if ((error = git_hash_update(&ctx, data, len)) >= 0)
error = git_hash_final(out, &ctx);
git_hash_ctx_cleanup(&ctx);
return error;
}
static int hash_object_stream(git_indexer*idx, git_packfile_stream *stream)
{
ssize_t read;
assert(idx && stream);
do {
if ((read = git_packfile_stream_read(stream, idx->objbuf, sizeof(idx->objbuf))) < 0)
break;
git_hash_update(&idx->hash_ctx, idx->objbuf, read);
} while (read > 0);
if (read < 0)
return (int)read;
return 0;
}
int git_hash_vec(git_oid *out, git_buf_vec *vec, size_t n)
{
git_hash_ctx ctx;
size_t i;
int error = 0;
if (git_hash_ctx_init(&ctx) < 0)
return -1;
for (i = 0; i < n; i++) {
if ((error = git_hash_update(&ctx, vec[i].data, vec[i].len)) < 0)
goto done;
}
error = git_hash_final(out, &ctx);
done:
git_hash_ctx_cleanup(&ctx);
return error;
}
static int lock_file(git_filebuf *file, int flags)
{
if (gitfo_exists(file->path_lock) == 0) {
if (flags & GIT_FILEBUF_FORCE)
gitfo_unlink(file->path_lock);
else
return git__throw(GIT_EOSERR, "Failed to lock file");
}
/* create path to the file buffer is required */
if (flags & GIT_FILEBUF_FORCE) {
file->fd = gitfo_creat_locked_force(file->path_lock, 0644);
} else {
file->fd = gitfo_creat_locked(file->path_lock, 0644);
}
if (file->fd < 0)
return git__throw(GIT_EOSERR, "Failed to create lock");
if ((flags & GIT_FILEBUF_APPEND) && gitfo_exists(file->path_original) == 0) {
git_file source;
char buffer[2048];
size_t read_bytes;
source = gitfo_open(file->path_original, O_RDONLY);
if (source < 0)
return git__throw(GIT_EOSERR, "Failed to lock file. Could not open %s", file->path_original);
while ((read_bytes = gitfo_read(source, buffer, 2048)) > 0) {
gitfo_write(file->fd, buffer, read_bytes);
if (file->digest)
git_hash_update(file->digest, buffer, read_bytes);
}
gitfo_close(source);
}
return GIT_SUCCESS;
}
static int write_deflate(git_filebuf *file, void *source, size_t len)
{
z_stream *zs = &file->zs;
if (len > 0 || file->flush_mode == Z_FINISH) {
zs->next_in = source;
zs->avail_in = (uInt)len;
do {
size_t have;
zs->next_out = file->z_buf;
zs->avail_out = (uInt)file->buf_size;
if (deflate(zs, file->flush_mode) == Z_STREAM_ERROR) {
file->last_error = BUFERR_ZLIB;
return -1;
}
have = file->buf_size - (size_t)zs->avail_out;
if (p_write(file->fd, file->z_buf, have) < 0) {
file->last_error = BUFERR_WRITE;
return -1;
}
} while (zs->avail_out == 0);
assert(zs->avail_in == 0);
if (file->digest)
git_hash_update(file->digest, source, len);
}
return 0;
}
static int lock_file(git_filebuf *file, int flags)
{
if (gitfo_exists(file->path_lock) == 0) {
if (flags & GIT_FILEBUF_FORCE)
gitfo_unlink(file->path_lock);
else
return GIT_EOSERR;
}
file->fd = gitfo_creat(file->path_lock, 0644);
if (file->fd < 0)
return GIT_EOSERR;
/* TODO: do a flock() in the descriptor file_lock */
if ((flags & GIT_FILEBUF_APPEND) && gitfo_exists(file->path_original) == 0) {
git_file source;
char buffer[2048];
size_t read_bytes;
source = gitfo_open(file->path_original, O_RDONLY);
if (source < 0)
return GIT_EOSERR;
while ((read_bytes = gitfo_read(source, buffer, 2048)) > 0) {
gitfo_write(file->fd, buffer, read_bytes);
if (file->digest)
git_hash_update(file->digest, buffer, read_bytes);
}
gitfo_close(source);
}
return GIT_SUCCESS;
}
int git_indexer_commit(git_indexer *idx, git_transfer_progress *stats)
{
git_mwindow *w = NULL;
unsigned int i, long_offsets = 0, left;
int error;
struct git_pack_idx_header hdr;
git_buf filename = GIT_BUF_INIT;
struct entry *entry;
git_oid trailer_hash, file_hash;
git_hash_ctx ctx;
git_filebuf index_file = {0};
void *packfile_trailer;
if (git_hash_ctx_init(&ctx) < 0)
return -1;
/* Test for this before resolve_deltas(), as it plays with idx->off */
if (idx->off < idx->pack->mwf.size - 20) {
giterr_set(GITERR_INDEXER, "Unexpected data at the end of the pack");
return -1;
}
packfile_trailer = git_mwindow_open(&idx->pack->mwf, &w, idx->pack->mwf.size - GIT_OID_RAWSZ, GIT_OID_RAWSZ, &left);
if (packfile_trailer == NULL) {
git_mwindow_close(&w);
goto on_error;
}
/* Compare the packfile trailer as it was sent to us and what we calculated */
git_oid_fromraw(&file_hash, (unsigned char*) packfile_trailer);
git_mwindow_close(&w);
git_hash_final(&trailer_hash, &idx->trailer);
if (git_oid_cmp(&file_hash, &trailer_hash)) {
giterr_set(GITERR_INDEXER, "packfile trailer mismatch");
return -1;
}
/* Freeze the number of deltas */
stats->total_deltas = stats->total_objects - stats->indexed_objects;
if ((error = resolve_deltas(idx, stats)) < 0)
return error;
if (stats->indexed_objects != stats->total_objects) {
giterr_set(GITERR_INDEXER, "early EOF");
return -1;
}
if (stats->local_objects > 0) {
if (update_header_and_rehash(idx, stats) < 0)
return -1;
git_hash_final(&trailer_hash, &idx->trailer);
write_at(idx, &trailer_hash, idx->pack->mwf.size - GIT_OID_RAWSZ, GIT_OID_RAWSZ);
}
git_vector_sort(&idx->objects);
git_buf_sets(&filename, idx->pack->pack_name);
git_buf_shorten(&filename, strlen("pack"));
git_buf_puts(&filename, "idx");
if (git_buf_oom(&filename))
return -1;
if (git_filebuf_open(&index_file, filename.ptr,
GIT_FILEBUF_HASH_CONTENTS, idx->mode) < 0)
goto on_error;
/* Write out the header */
hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
hdr.idx_version = htonl(2);
git_filebuf_write(&index_file, &hdr, sizeof(hdr));
/* Write out the fanout table */
for (i = 0; i < 256; ++i) {
uint32_t n = htonl(idx->fanout[i]);
git_filebuf_write(&index_file, &n, sizeof(n));
}
/* Write out the object names (SHA-1 hashes) */
git_vector_foreach(&idx->objects, i, entry, struct entry*) {
git_filebuf_write(&index_file, &entry->oid, sizeof(git_oid));
git_hash_update(&ctx, &entry->oid, GIT_OID_RAWSZ);
}
git_hash_final(&idx->hash, &ctx);
/* Write out the CRC32 values */
git_vector_foreach(&idx->objects, i, entry, struct entry*) {
git_filebuf_write(&index_file, &entry->crc, sizeof(uint32_t));
}
/* Write out the offsets */
git_vector_foreach(&idx->objects, i, entry, struct entry*) {
uint32_t n;
if (entry->offset == UINT32_MAX)
n = htonl(0x80000000 | long_offsets++);
else
n = htonl(entry->offset);
git_filebuf_write(&index_file, &n, sizeof(uint32_t));
}
/* Write out the long offsets */
git_vector_foreach(&idx->objects, i, entry, struct entry*) {
uint32_t split[2];
if (entry->offset != UINT32_MAX)
continue;
split[0] = htonl(entry->offset_long >> 32);
split[1] = htonl(entry->offset_long & 0xffffffff);
git_filebuf_write(&index_file, &split, sizeof(uint32_t) * 2);
}
/* Write out the packfile trailer to the index */
if (git_filebuf_write(&index_file, &trailer_hash, GIT_OID_RAWSZ) < 0)
goto on_error;
/* Write out the hash of the idx */
if (git_filebuf_hash(&trailer_hash, &index_file) < 0)
goto on_error;
git_filebuf_write(&index_file, &trailer_hash, sizeof(git_oid));
/* Figure out what the final name should be */
if (index_path(&filename, idx, ".idx") < 0)
goto on_error;
/* Commit file */
if (git_filebuf_commit_at(&index_file, filename.ptr) < 0)
goto on_error;
git_mwindow_free_all(&idx->pack->mwf);
/* We need to close the descriptor here so Windows doesn't choke on commit_at */
if (p_close(idx->pack->mwf.fd) < 0) {
giterr_set(GITERR_OS, "failed to close packfile");
goto on_error;
}
idx->pack->mwf.fd = -1;
if (index_path(&filename, idx, ".pack") < 0)
goto on_error;
/* And don't forget to rename the packfile to its new place. */
p_rename(idx->pack->pack_name, git_buf_cstr(&filename));
git_buf_free(&filename);
git_hash_ctx_cleanup(&ctx);
return 0;
on_error:
git_mwindow_free_all(&idx->pack->mwf);
git_filebuf_cleanup(&index_file);
git_buf_free(&filename);
git_hash_ctx_cleanup(&ctx);
return -1;
}
