void multiround_hash_table::_next_round (file_reader & reader
, hasher_stream & stream
, std::set<hole_t> & holes
, const int32_t blk_len) const
{
if (holes.empty ()) {
std::string errmsg = fmt_string ("Holes mustn't be empty.");
THROW_XDELTA_EXCEPTION (errmsg);
}
try {
stream.next_round (blk_len);
typedef std::set<hole_t>::const_iterator cit_t;
for (cit_t begin = holes.begin (); begin != holes.end (); ++begin) {
const hole_t & hole = *begin;
if (hole.length < (uint32_t)blk_len)
continue;
uint64_t offset = reader.seek_file (hole.offset, FILE_BEGIN);
if (offset != hole.offset) {
std::string errmsg = fmt_string ("Can't seek file %s(%s)."
, reader.get_fname ().c_str (), error_msg ().c_str ());
THROW_XDELTA_EXCEPTION (errmsg);
}
read_and_hash (reader, stream, hole.length, blk_len, hole.offset, 0);
}
stream.end_one_round ();
}
catch (xdelta_exception &e) {
stream.on_error (e.what (), e.get_errno ());
}
}
static void read_and_hash (file_reader & reader
, hasher_stream & stream
, uint64_t to_read_bytes
, int32_t blk_len
, uint64_t t_offset
, rs_mdfour_t * pctx)
{
//
// read huge block one time and calc hash block after block length of f_blk_len;
//
uint32_t remain, buflen;
char_buffer<uchar_t> buf (XDELTA_BUFFER_LEN);
uint64_t index = 0;
uint32_t size;
uchar_t * rdbuf = buf.begin ();
buflen = (uint32_t)(to_read_bytes > XDELTA_BUFFER_LEN ? XDELTA_BUFFER_LEN : to_read_bytes);
while (to_read_bytes != 0) {
size = reader.read_file (rdbuf, buflen);
if (size != buflen) {
std::string errmsg = fmt_string ("Can't read file %s(%s)."
, reader.get_fname ().c_str (), error_msg ().c_str ());
THROW_XDELTA_EXCEPTION (errmsg);
}
if (pctx != 0)
rs_mdfour_update(pctx, rdbuf, size);
to_read_bytes -= size;
const uchar_t * endbuf = rdbuf + size;
rdbuf = buf.begin ();
while ((int32_t)(endbuf - rdbuf) >= blk_len) {
uint32_t fhash = rolling_hasher::hash (rdbuf, blk_len);
struct slow_hash bsh;
bsh.tpos.index = index;
bsh.tpos.t_offset = t_offset;
get_slow_hash (rdbuf, blk_len, bsh.hash);
stream.add_block (fhash, bsh);
++index;
rdbuf += blk_len;
}
remain = (int32_t)(endbuf - rdbuf);
if (remain > 0)
memmove (buf.begin (), rdbuf, remain);
rdbuf = buf.begin () + remain;
buflen = XDELTA_BUFFER_LEN - remain;
buflen = (uint32_t)(to_read_bytes > buflen ? buflen : to_read_bytes);
}
}
int32_t multiround_hash_table::_haser_first_round (file_reader & reader
, hasher_stream & stream
, std::set<hole_t> & holes) const
{
if (!holes.empty ()) {
std::string errmsg = fmt_string ("Holes must be empty.");
THROW_XDELTA_EXCEPTION (errmsg);
}
hole_t hole;
hole.offset = 0;
bool round = true;
try {
reader.open_file (); // sometimes this will throw.
hole.length = reader.get_file_size ();
holes.insert (hole);
}
catch (...) {
round = false;
}
if (!round) {
hash_table::hash_it (reader, stream);
return -1;
}
int32_t blk_len = (uint32_t)(hole.length / multiround_base ());
if (blk_len > MULTIROUND_MAX_BLOCK_SIZE)
blk_len = MULTIROUND_MAX_BLOCK_SIZE;
if (blk_len < XDELTA_BLOCK_SIZE)
blk_len = XDELTA_BLOCK_SIZE;
stream.start_hash_stream (reader.get_fname (), blk_len);
rs_mdfour_t ctx;
rs_mdfour_begin(&ctx);
read_and_hash (reader, stream, hole.length, blk_len, hole.offset, &ctx);
uchar_t file_hash[DIGEST_BYTES];
memset (&file_hash, 0, sizeof (file_hash));
rs_mdfour_result (&ctx, file_hash);
bool neednextround = stream.end_first_round (file_hash);
return (neednextround ? blk_len : -1);
}
uint32_t seek_and_read (file_reader & reader, uint64_t offset, uint32_t length, uchar_t * data)
{
uint64_t s_offset = reader.seek_file (offset, FILE_BEGIN);
if (offset != s_offset) {
std::string errmsg = fmt_string ("Can't seek file %s(%s)."
, reader.get_fname ().c_str (), error_msg ().c_str ());
THROW_XDELTA_EXCEPTION (errmsg);
}
uint32_t ret = reader.read_file (data, length);
if (ret < length) {
std::string errmsg = fmt_string ("Bytes read from %s(%s) is less than demand."
, reader.get_fname ().c_str (), error_msg ().c_str ());
THROW_XDELTA_EXCEPTION (errmsg);
}
return ret;
}
void multiround_hash_table::hash_it (file_reader & reader, hasher_stream & stream) const
{
std::set<hole_t> holes;
stream.set_holes (&holes);
int32_t blk_len = _haser_first_round (reader, stream, holes);
//
// -1, means file is equal, so
// no need to goto the next round.
//
if (blk_len != -1) {
blk_len /= multiround_base ();
while (blk_len > minimal_multiround_block ()) {
_next_round (reader, stream, holes, blk_len);
blk_len /= multiround_base ();
}
uchar_t file_hash[DIGEST_BYTES];
memset (file_hash, 0, DIGEST_BYTES);
uint64_t filsize = reader.get_file_size ();
reader.close_file ();
stream.end_hash_stream (file_hash, filsize);
}
}
void DLL_EXPORT get_file_digest (file_reader & reader
, uchar_t digest[DIGEST_BYTES])
{
const int buf_len = XDELTA_BUFFER_LEN; // four times of block.
char_buffer <uchar_t> buf (XDELTA_BUFFER_LEN);
rs_mdfour_t ctx;
rs_mdfour_begin(&ctx);
while (true) {
int size = reader.read_file (buf.begin (), buf_len);
if (size <= 0)
break;
rs_mdfour_update(&ctx, buf.begin (), size);
}
rs_mdfour_result (&ctx, digest);
}
//
// used the hash_table object receive from remote host
// and send a hash stream to the remote host.
//
void hash_table::hash_it (file_reader & reader, hasher_stream & stream) const
{
rs_mdfour_t ctx;
rs_mdfour_begin(&ctx);
uint64_t filsize = 0;
try {
int32_t f_blk_len = 0;
if (reader.exist_file ()) {
try {
reader.open_file (); // sometimes this will throw.
filsize = reader.get_file_size ();
f_blk_len = get_xdelta_block_size (filsize);
}
catch (...) {
stream.start_hash_stream (reader.get_fname (), 0);
throw;
}
stream.start_hash_stream (reader.get_fname (), f_blk_len);
}
else {
std::string mesg = fmt_string ("File %s not exists."
, reader.get_fname ().c_str ());
stream.start_hash_stream (reader.get_fname (), 0);
stream.on_error (mesg, ENOENT);
goto end;
}
read_and_hash (reader, stream, filsize, f_blk_len, 0, &ctx);
}
catch (xdelta_exception &e) {
stream.on_error (e.what (), e.get_errno ());
}
end:
uchar_t file_hash[DIGEST_BYTES];
memset (file_hash, 0, sizeof (file_hash));
rs_mdfour_result (&ctx, file_hash);
reader.close_file ();
stream.end_hash_stream (file_hash, filsize);
return;
}
void read_and_delta (file_reader & reader
, xdelta_stream & stream
, const hash_table & hashes
, std::set<hole_t> & hole_set
, const int blk_len
, bool need_split_hole)
{
bool adddiff = !need_split_hole;
char_buffer<uchar_t> buf (XDELTA_BUFFER_LEN);
typedef std::set<hole_t>::iterator it_t;
std::list<hole_t> holes2remove;
for (it_t begin = hole_set.begin (); begin != hole_set.end (); ++begin) {
const hole_t & hole = *begin;
uint64_t offset = reader.seek_file (hole.offset, FILE_BEGIN);
if (offset != hole.offset) {
std::string errmsg = fmt_string ("Can't seek file %s(%s)."
, reader.get_fname ().c_str (), error_msg ().c_str ());
THROW_XDELTA_EXCEPTION (errmsg);
}
uint32_t buflen = XDELTA_BUFFER_LEN;
uint32_t to_read_bytes = (uint32_t)hole.length;
buflen = to_read_bytes > buflen ? buflen : to_read_bytes;
uchar_t * rdbuf = buf.begin ();
uint32_t size = reader.read_file (rdbuf, buflen);
if (size != buflen) {
std::string errmsg = fmt_string ("Can't read file %s(%s)."
, reader.get_fname ().c_str (), error_msg ().c_str ());
THROW_XDELTA_EXCEPTION (errmsg);
}
to_read_bytes -= size;
const uchar_t * endbuf = rdbuf + size;
rdbuf = buf.begin ();
if ((int32_t)(endbuf - rdbuf) >= blk_len) {
uchar_t * sentrybuf = rdbuf;
rolling_hasher hasher;
hasher.eat_hash (rdbuf, blk_len);
while (true) {
bool newhash = false;
const slow_hash * bsh = hashes.find_block (hasher.hash_value (), rdbuf, blk_len);
uchar_t outchar = 0;
if (bsh) {
// a match was found.
uint32_t slipsize = (uint32_t)(rdbuf - sentrybuf);
if (slipsize > 0 && adddiff)
stream.add_block (sentrybuf, slipsize, offset);
offset += slipsize;
stream.add_block (bsh->tpos, blk_len, offset);
if (need_split_hole) {
hole_t newhole;
newhole.offset = offset;
newhole.length = blk_len;
holes2remove.push_back (newhole);
}
rdbuf += blk_len;
sentrybuf = rdbuf;
newhash = true;
offset += blk_len;
}
else {
// slip the window by one bytes which size is blk_len.
outchar = *rdbuf;
++rdbuf;
}
//
// beyond the buffer.
int remain = (int)(endbuf - rdbuf);
if (remain < blk_len) {
if (to_read_bytes == 0) {
// no more to read.
uint32_t slipsize = (uint32_t)(endbuf - sentrybuf);
if (slipsize > 0 && adddiff)
stream.add_block (sentrybuf, slipsize, offset);
goto end;
}
else {
memmove (buf.begin (), rdbuf, remain);
rdbuf = buf.begin ();
sentrybuf = rdbuf;
buflen = XDELTA_BUFFER_LEN - remain;
buflen = to_read_bytes > buflen ? buflen : to_read_bytes;
size = reader.read_file (rdbuf + remain, buflen);
if (size != buflen) {
std::string errmsg = fmt_string ("Can't read file %s(%s)."
, reader.get_fname ().c_str (), error_msg ().c_str ());
THROW_XDELTA_EXCEPTION (errmsg);
}
to_read_bytes -= size;
endbuf = rdbuf + remain + size;
remain += size;
if (remain >= blk_len) {
if (newhash)
hasher.eat_hash (rdbuf, blk_len);
else
hasher.update (outchar, *(rdbuf + blk_len));
}
else {
//
// one read must complement data which length plus
// remain must be more than one block length of @f_blk_len,
// so if remain less than that, it must be reach the end of
// file
//
if (adddiff)
stream.add_block (rdbuf, remain, offset);
offset += remain;
goto end;
}
}
}
else {
if (newhash)
hasher.eat_hash (rdbuf, blk_len);
else
hasher.update (outchar, *(rdbuf + blk_len - 1));
}
}
}
else {
if (adddiff)
stream.add_block (rdbuf, size, offset);
}
end:
continue;
}
if (need_split_hole) {
typedef std::list<hole_t>::iterator it_t;
for (it_t begin = holes2remove.begin (); begin != holes2remove.end (); ++begin)
split_hole (hole_set, *begin);
}
return;
}
