void cbn_consoleOutput(unordered_multimap<string, string> &folder)
{
std::cout << "Folder contains:"<<endl;
for ( auto it = folder.begin(); it != folder.end(); ++it ){
std::cout << " " << it->first << ":" << it->second;
std::cout << std::endl;}
}
void eraseIfVal(unordered_multimap<K, V, H, E, A>& umap, EraseIfFn const& f) {
for (typename unordered_multimap<K, V, H, E, A>::iterator i = umap.begin(), e = umap.end();
i != e;) // no ++i - intentional
if (f(i->second))
i = umap.erase(i);
else
++i;
}
void umm_buckets(unordered_multimap<string, string> &folder)
{
std::cout << "folder buckets contain:\n";
for ( unsigned i = 0; i < folder.bucket_count(); ++i) {
std::cout << "bucket #" << i << " contains:";
for ( auto local_it = folder.begin(i); local_it!= folder.end(i); ++local_it )
std::cout << " " << local_it->first << ":" << local_it->second;
std::cout << std::endl;
}
}
/** Removes a value from the collection. Returns true if the collection contained the specified element. */
bool remove(int val) {
auto inSet = map.find(val);
if(inSet != map.end()){
int pos = map.find(val)->second;
map.erase (map.find(val), ++map.find(val));
int last = elements.back(); elements.pop_back();
elements[pos] = last;
for(auto findPos = map.find(last); findPos != map.end(); ++findPos){
if(findPos->second == elements.size()){
map.erase( findPos, std::next(findPos));
map.insert( make_pair(last, pos));
break;
}
}
}
return inSet != map.end();
}
uint64_t sketchUnorderedComparisonError(const unordered_multimap<string, string>& map1, const unordered_multimap<string, string>& map2){
uint64_t res(0);
string beg,end;
for (auto it=map1.begin(); it!=map1.end(); ++it){
beg=it->first;
end=it->second;
auto ret = map2.equal_range(beg);
for (auto it2=ret.first; it2!=ret.second; ++it2){
if(isCorrect(end,it2->second)){
++res;
}
}
}
return res;
}
inline void insert_hash(hash_t key,hashtgt &val){
#if PERFORMANCE_STATISTICS
++totalh;
#endif
for(auto it=hashes.find(key);it!=hashes.end();++it){
if(memcmp(val.digest,it->second.digest,sizeof(val.digest))==0){
#if PERFORMANCE_STATISTICS
++conflict;
#endif
//already have the same one inserted!
return;
}
}
hashes.insert(make_pair(key,val));
}
void eraseIf(unordered_multimap<K, V, H, E, A>& unordered, EraseIfFn const& f) {
for (typename unordered_multimap<K, V, H, E, A>::iterator i = unordered.begin(), e = unordered.end();
i != e;) // no ++i - intentional
if (f(*i))
i = unordered.erase(i);
else
++i;
}
/*Scan and generate the diff file
more(buf,start,len,buffsize): read more date from input
*/
void Scan(function<offset_t(byte*,int64_t,offset_t,offset_t)> more,int64_t len){
int64_t fpos=0;
const int preread=5*1024*1024;
byte * input=new byte[preread];
{
int toread=(int32_t)min(preread,len-fpos);
toread=max(bsbmax,toread);
fpos+=more(input,0,toread,preread);
}
int bs=max_bs;
int bsb=1<<bs;
hash_t a=1,b=0;
for(int di=0;di<bsb;++di){
a=(a+input[di%preread])%hash_magic;
b=(b+a)%hash_magic;
}
offset_t written=0;
int64_t written64=0;
while(written64<len){
int64_t fileleft=len-fpos;
//preread when reach 10*bsbmax byte from fpos
if(fileleft>0 && written64+10*bsbmax>fpos){
int toread=(int)min(preread-20*bsbmax,fileleft);
fpos+=more(input,fpos,toread,preread);
}
bool find=false;
unsigned char digest[16];
bool digestgen=false;
bool after_a_match=false;
for(auto it=hashes.find((b<<16)+a);it!=hashes.end();++it){
#if WITH_FIRST16_BYTE_CHECK
{
int s0=written%preread;
int e0=(s0+sizeof(it->second.first16))%preread;
//test only for this case
if(s0<e0){
if(memcmp(it->second.first16, &input[s0], sizeof(it->second.first16))!=0){
continue;
}
}
}
#endif
if(!digestgen){
digestgen=true;
MD5_CTX context;
MD5_Init(&context);
{
int s0=written%preread;
int e0=(s0+bsb)%preread;
if(s0<e0){
MD5_Update(&context, &input[s0], bsb);
}else{
MD5_Update(&context, &input[s0], preread-s0);
MD5_Update(&context, &input[0], e0);
}
}
MD5_Final(digest, &context);
}
if(memcmp(it->second.digest,digest,sizeof(digest))==0){
find=true;
Coder::writeoff(it->second.offsetx);
written+=bsb;
written64+=bsb;
a=1,b=0;
for(int di=0;di<bsb;++di){
a=(a+input[(written+di)%preread])%hash_magic;
b=(b+a)%hash_magic;
}
after_a_match=true;
break;
}else{
}
}
if(!find){
int writesize=1;
if(after_a_match){
(int)((len/(1024*1024))&0x8fffffff);
writesize=rand()%writesize+1;
}
after_a_match=false;
for(int i=0;i<writesize && written64<len;++i){
byte wbyte=input[written%preread];
Coder::writebyte(wbyte);
++written;
++written64;
offset_t scan=written+bsb-1;
//(32 bits trap) for(;scan<written+bsbmax && scan<len;++scan){
//for [scan-bsb+1,scan]
int bs=max_bs;
int bsb=1<<bs;
hash_t olda=a;
hash_t oldb=b;
a=(olda+hash_magic-wbyte+input[scan%preread])%hash_magic;
b=(oldb
+hash_magic-(wbyte*bsb)%hash_magic
+a-1
)%hash_magic;
//}
}
}
}
Coder::writeclose();
}
