static int update_header_and_rehash(git_indexer *idx, git_transfer_progress *stats) { void *ptr; size_t chunk = 1024*1024; git_off_t hashed = 0; git_mwindow *w = NULL; git_mwindow_file *mwf; unsigned int left; mwf = &idx->pack->mwf; git_hash_init(&idx->trailer); /* Update the header to include the numer of local objects we injected */ idx->hdr.hdr_entries = htonl(stats->total_objects + stats->local_objects); if (write_at(idx, &idx->hdr, 0, sizeof(struct git_pack_header)) < 0) return -1; /* * We now use the same technique as before to determine the * hash. We keep reading up to the end and let * hash_partially() keep the existing trailer out of the * calculation. */ git_mwindow_free_all(mwf); idx->inbuf_len = 0; while (hashed < mwf->size) { ptr = git_mwindow_open(mwf, &w, hashed, chunk, &left); if (ptr == NULL) return -1; hash_partially(idx, (unsigned char*) ptr, left); hashed += left; git_mwindow_close(&w); } return 0; }
GIT_INLINE(void) pack_window_free_all(struct pack_backend *GIT_UNUSED(backend), struct git_pack_file *p) { GIT_UNUSED_ARG(backend); git_mwindow_free_all(&p->mwf); }
int git_indexer_commit(git_indexer *idx, git_transfer_progress *stats) { git_mwindow *w = NULL; unsigned int i, long_offsets = 0, left; int error; struct git_pack_idx_header hdr; git_buf filename = GIT_BUF_INIT; struct entry *entry; git_oid trailer_hash, file_hash; git_hash_ctx ctx; git_filebuf index_file = {0}; void *packfile_trailer; if (git_hash_ctx_init(&ctx) < 0) return -1; /* Test for this before resolve_deltas(), as it plays with idx->off */ if (idx->off < idx->pack->mwf.size - 20) { giterr_set(GITERR_INDEXER, "Unexpected data at the end of the pack"); return -1; } packfile_trailer = git_mwindow_open(&idx->pack->mwf, &w, idx->pack->mwf.size - GIT_OID_RAWSZ, GIT_OID_RAWSZ, &left); if (packfile_trailer == NULL) { git_mwindow_close(&w); goto on_error; } /* Compare the packfile trailer as it was sent to us and what we calculated */ git_oid_fromraw(&file_hash, (unsigned char*) packfile_trailer); git_mwindow_close(&w); git_hash_final(&trailer_hash, &idx->trailer); if (git_oid_cmp(&file_hash, &trailer_hash)) { giterr_set(GITERR_INDEXER, "packfile trailer mismatch"); return -1; } /* Freeze the number of deltas */ stats->total_deltas = stats->total_objects - stats->indexed_objects; if ((error = resolve_deltas(idx, stats)) < 0) return error; if (stats->indexed_objects != stats->total_objects) { giterr_set(GITERR_INDEXER, "early EOF"); return -1; } if (stats->local_objects > 0) { if (update_header_and_rehash(idx, stats) < 0) return -1; git_hash_final(&trailer_hash, &idx->trailer); write_at(idx, &trailer_hash, idx->pack->mwf.size - GIT_OID_RAWSZ, GIT_OID_RAWSZ); } git_vector_sort(&idx->objects); git_buf_sets(&filename, idx->pack->pack_name); git_buf_shorten(&filename, strlen("pack")); git_buf_puts(&filename, "idx"); if (git_buf_oom(&filename)) return -1; if (git_filebuf_open(&index_file, filename.ptr, GIT_FILEBUF_HASH_CONTENTS, idx->mode) < 0) goto on_error; /* Write out the header */ hdr.idx_signature = htonl(PACK_IDX_SIGNATURE); hdr.idx_version = htonl(2); git_filebuf_write(&index_file, &hdr, sizeof(hdr)); /* Write out the fanout table */ for (i = 0; i < 256; ++i) { uint32_t n = htonl(idx->fanout[i]); git_filebuf_write(&index_file, &n, sizeof(n)); } /* Write out the object names (SHA-1 hashes) */ git_vector_foreach(&idx->objects, i, entry, struct entry*) { git_filebuf_write(&index_file, &entry->oid, sizeof(git_oid)); git_hash_update(&ctx, &entry->oid, GIT_OID_RAWSZ); } git_hash_final(&idx->hash, &ctx); /* Write out the CRC32 values */ git_vector_foreach(&idx->objects, i, entry, struct entry*) { git_filebuf_write(&index_file, &entry->crc, sizeof(uint32_t)); } /* Write out the offsets */ git_vector_foreach(&idx->objects, i, entry, struct entry*) { uint32_t n; if (entry->offset == UINT32_MAX) n = htonl(0x80000000 | long_offsets++); else n = htonl(entry->offset); git_filebuf_write(&index_file, &n, sizeof(uint32_t)); } /* Write out the long offsets */ git_vector_foreach(&idx->objects, i, entry, struct entry*) { uint32_t split[2]; if (entry->offset != UINT32_MAX) continue; split[0] = htonl(entry->offset_long >> 32); split[1] = htonl(entry->offset_long & 0xffffffff); git_filebuf_write(&index_file, &split, sizeof(uint32_t) * 2); } /* Write out the packfile trailer to the index */ if (git_filebuf_write(&index_file, &trailer_hash, GIT_OID_RAWSZ) < 0) goto on_error; /* Write out the hash of the idx */ if (git_filebuf_hash(&trailer_hash, &index_file) < 0) goto on_error; git_filebuf_write(&index_file, &trailer_hash, sizeof(git_oid)); /* Figure out what the final name should be */ if (index_path(&filename, idx, ".idx") < 0) goto on_error; /* Commit file */ if (git_filebuf_commit_at(&index_file, filename.ptr) < 0) goto on_error; git_mwindow_free_all(&idx->pack->mwf); /* We need to close the descriptor here so Windows doesn't choke on commit_at */ if (p_close(idx->pack->mwf.fd) < 0) { giterr_set(GITERR_OS, "failed to close packfile"); goto on_error; } idx->pack->mwf.fd = -1; if (index_path(&filename, idx, ".pack") < 0) goto on_error; /* And don't forget to rename the packfile to its new place. */ p_rename(idx->pack->pack_name, git_buf_cstr(&filename)); git_buf_free(&filename); git_hash_ctx_cleanup(&ctx); return 0; on_error: git_mwindow_free_all(&idx->pack->mwf); git_filebuf_cleanup(&index_file); git_buf_free(&filename); git_hash_ctx_cleanup(&ctx); return -1; }
/** * Rewind the packfile by the trailer, as we might need to fix the * packfile by injecting objects at the tail and must overwrite it. */ static void seek_back_trailer(git_indexer *idx) { idx->pack->mwf.size -= GIT_OID_RAWSZ; git_mwindow_free_all(&idx->pack->mwf); }
int git_indexer_append(git_indexer *idx, const void *data, size_t size, git_transfer_progress *stats) { int error = -1; size_t processed; struct git_pack_header *hdr = &idx->hdr; git_mwindow_file *mwf = &idx->pack->mwf; assert(idx && data && stats); processed = stats->indexed_objects; if ((error = append_to_pack(idx, data, size)) < 0) return error; hash_partially(idx, (unsigned char*) data, (int)size); /* Make sure we set the new size of the pack */ idx->pack->mwf.size += size; if (!idx->parsed_header) { unsigned int total_objects; if ((unsigned)idx->pack->mwf.size < sizeof(struct git_pack_header)) return 0; if ((error = parse_header(&idx->hdr, idx->pack)) < 0) return error; idx->parsed_header = 1; idx->nr_objects = ntohl(hdr->hdr_entries); idx->off = sizeof(struct git_pack_header); /* for now, limit to 2^32 objects */ assert(idx->nr_objects == (size_t)((unsigned int)idx->nr_objects)); if (idx->nr_objects == (size_t)((unsigned int)idx->nr_objects)) total_objects = (unsigned int)idx->nr_objects; else total_objects = UINT_MAX; idx->pack->idx_cache = git_oidmap_alloc(); GITERR_CHECK_ALLOC(idx->pack->idx_cache); idx->pack->has_cache = 1; if (git_vector_init(&idx->objects, total_objects, objects_cmp) < 0) return -1; if (git_vector_init(&idx->deltas, total_objects / 2, NULL) < 0) return -1; stats->received_objects = 0; stats->local_objects = 0; stats->total_deltas = 0; stats->indexed_deltas = 0; processed = stats->indexed_objects = 0; stats->total_objects = total_objects; if ((error = do_progress_callback(idx, stats)) != 0) return error; } /* Now that we have data in the pack, let's try to parse it */ /* As the file grows any windows we try to use will be out of date */ git_mwindow_free_all(mwf); while (processed < idx->nr_objects) { git_packfile_stream *stream = &idx->stream; git_off_t entry_start = idx->off; size_t entry_size; git_otype type; git_mwindow *w = NULL; if (idx->pack->mwf.size <= idx->off + 20) return 0; if (!idx->have_stream) { error = git_packfile_unpack_header(&entry_size, &type, mwf, &w, &idx->off); if (error == GIT_EBUFS) { idx->off = entry_start; return 0; } if (error < 0) goto on_error; git_mwindow_close(&w); idx->entry_start = entry_start; git_hash_init(&idx->hash_ctx); if (type == GIT_OBJ_REF_DELTA || type == GIT_OBJ_OFS_DELTA) { error = advance_delta_offset(idx, type); if (error == GIT_EBUFS) { idx->off = entry_start; return 0; } if (error < 0) goto on_error; idx->have_delta = 1; } else { idx->have_delta = 0; hash_header(&idx->hash_ctx, entry_size, type); } idx->have_stream = 1; error = git_packfile_stream_open(stream, idx->pack, idx->off); if (error < 0) goto on_error; } if (idx->have_delta) { error = read_object_stream(idx, stream); } else { error = hash_object_stream(idx, stream); } idx->off = stream->curpos; if (error == GIT_EBUFS) return 0; /* We want to free the stream reasorces no matter what here */ idx->have_stream = 0; git_packfile_stream_free(stream); if (error < 0) goto on_error; if (idx->have_delta) { error = store_delta(idx); } else { error = store_object(idx); } if (error < 0) goto on_error; if (!idx->have_delta) { stats->indexed_objects = (unsigned int)++processed; } stats->received_objects++; if ((error = do_progress_callback(idx, stats)) != 0) goto on_error; } return 0; on_error: git_mwindow_free_all(mwf); return error; }