void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end,
ZopfliHash* h) {
unsigned short hpos = pos & ZOPFLI_WINDOW_MASK;
#ifdef ZOPFLI_HASH_SAME
size_t amount = 0;
#endif
UpdateHashValue(h, pos + ZOPFLI_MIN_MATCH <= end ?
array[pos + ZOPFLI_MIN_MATCH - 1] : 0);
h->hashval[hpos] = h->val;
if (h->head[h->val] != -1 && h->hashval[h->head[h->val]] == h->val) {
h->prev[hpos] = h->head[h->val];
}
else h->prev[hpos] = hpos;
h->head[h->val] = hpos;
#ifdef ZOPFLI_HASH_SAME
/* Update "same". */
if (h->same[(pos - 1) & ZOPFLI_WINDOW_MASK] > 1) {
amount = h->same[(pos - 1) & ZOPFLI_WINDOW_MASK] - 1;
}
while (pos + amount + 1 < end &&
array[pos] == array[pos + amount + 1] && amount < (unsigned short)(-1)) {
amount++;
}
h->same[hpos] = amount;
#endif
#ifdef ZOPFLI_HASH_SAME_HASH
h->val2 = ((h->same[hpos] - ZOPFLI_MIN_MATCH) & 255) ^ h->val;
h->hashval2[hpos] = h->val2;
if (h->head2[h->val2] != -1 && h->hashval2[h->head2[h->val2]] == h->val2) {
h->prev2[hpos] = h->head2[h->val2];
}
else h->prev2[hpos] = hpos;
h->head2[h->val2] = hpos;
#endif
}
void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end,
ZopfliHash* h) {
(void)end;
UpdateHashValue(h, array[pos + 0]);
UpdateHashValue(h, array[pos + 1]);
}
bool IndexBitConnector::OnInitConnectionToDataBase()
{
UInt32 tr,gr,colIndex;
UInt64 cIndex;
GML::Utils::GTFVector<GML::DB::DBRecord> VectPtr;
GML::Utils::GString temp;
GML::DB::RecordHash cHash;
double cValue;
IndexBitCounter ibc;
bool Label;
CacheMemory = 0; // ca sa nu am probleme la GetRecord
memset(&ibc,0,sizeof(ibc));
notifier->StartProcent("[%s] -> Analizing DataBase : ",ObjectName);
for (tr=0;tr<nrRecords;tr++)
{
if (database->ReadNextRecord(VectPtr)==false)
{
notifier->Error("[%s] -> Unable to read #d record from database!",ObjectName,tr);
return false;
}
// pentru fiecare record pun valorile
for (gr=0;gr<columns.nrFeatures;gr++)
{
colIndex = columns.indexFeature[gr];
if (UpdateDoubleValue(VectPtr,colIndex,cValue)==false)
return false;
if (cValue!=0.0)
Update(ibc,gr);
}
if ((tr % 10000)==0)
notifier->SetProcent(tr,nrRecords);
}
notifier->EndProcent();
notifier->Info("[%s] -> Total number of features: %d",ObjectName,ibc.countInt32);
ComputeMemory(ibc,MemToAlloc);
if (AllocMemory(MemToAlloc)==false)
return false;
if (database->BeginIteration()==false)
{
notifier->Error("[%s] -> Error on DataBase::BeginIteration()",ObjectName);
return false;
}
cIndex = 0;
notifier->StartProcent("[%s] -> Loading DataBase : ",ObjectName);
for (tr=0;tr<nrRecords;tr++)
{
Indexes[tr] = cIndex;
if (database->ReadNextRecord(VectPtr)==false)
{
notifier->Error("[%s] -> Unable to read #d record from database!",ObjectName,tr);
return false;
}
// pentru fiecare record pun valorile
for (gr=0;gr<columns.nrFeatures;gr++)
{
colIndex = columns.indexFeature[gr];
if (UpdateDoubleValue(VectPtr,colIndex,cValue)==false)
return false;
if (cValue!=0.0)
{
if (AddIndex(gr,cIndex)==false)
{
temp.Set("");
temp.AddFormatedEx("[%{str}] -> Unable to add Index to list : Records:%{uint32,dec} , Feature:%{uint32,dec} , Poz:%{uint64,dec,G3} , Alloc:%{uint64,dec,G3}",ObjectName,tr,gr,cIndex,MemToAlloc);
notifier->Error("%s",temp.GetText());
return false;
}
}
}
// pun si label-ul
if (UpdateDoubleValue(VectPtr,columns.indexLabel,cValue)==false)
return false;
if (UpdateTwoClassLabelValue(cValue,Label)==false)
return false;
if (Labels.Set(tr,Label)==false)
{
notifier->Error("[%s] -> Unable to set label for record #%d",ObjectName,tr);
return false;
}
// adaug si Hash-ul
if (StoreRecordHash)
{
if (UpdateHashValue(VectPtr,columns.indexHash,cHash)==false)
return false;
if (Hashes.PushByRef(cHash)==false)
{
notifier->Error("[%s] -> Unable to save Hash with id %d",ObjectName,tr);
return false;
}
}
// trecem la urmatorul record
if ((tr % 10000)==0)
notifier->SetProcent(tr,nrRecords);
}
notifier->EndProcent();
if (cIndex!=MemToAlloc)
{
notifier->Error("[%s] -> Internal Error computing compressed size ... (cIndex=%d,MemToAlloc=%d)",ObjectName,(UInt32)cIndex,(UInt32)MemToAlloc);
return false;
}
// all ok , am incarcat datele
dataMemorySize = (UInt64)nrRecords * sizeof(UInt64) + MemToAlloc+Labels.GetAllocated();
return true;
}
