void QueryTermVector::processTerms(Collection<String> queryTerms)
{
if (queryTerms)
{
std::sort(queryTerms.begin(), queryTerms.end());
MapStringInt tmpSet(MapStringInt::newInstance());
// filter out duplicates
Collection<String> tmpList(Collection<String>::newInstance());
Collection<int32_t> tmpFreqs(Collection<int32_t>::newInstance());
int32_t j = 0;
for (int32_t i = 0; i < queryTerms.size(); ++i)
{
String term(queryTerms[i]);
MapStringInt::iterator position = tmpSet.find(term);
if (position == tmpSet.end())
{
tmpSet.put(term, j++);
tmpList.add(term);
tmpFreqs.add(1);
}
else
{
int32_t freq = tmpFreqs[position->second];
tmpFreqs[position->second] = freq + 1;
}
}
terms = tmpList;
termFreqs = Collection<int32_t>::newInstance(tmpFreqs.size());
int32_t i = 0;
for (Collection<int32_t>::iterator freq = tmpFreqs.begin(); freq != tmpFreqs.end(); ++freq)
termFreqs[i++] = *freq;
}
}
void ReadImplicationsAndSets(std::vector<FCA::ImplicationInd> &implications, std::vector<FCA::BitSet> &bitsets, std::ifstream &in)
{
size_t implNumber;
size_t setsNumber;
size_t attrNumber;
in >> setsNumber >> implNumber >> attrNumber;
implications.clear();
bitsets.clear();
for (size_t setInd = 0; setInd < setsNumber; ++setInd)
{
FCA::BitSet tmpSet(attrNumber);
tmpSet.reset();
size_t setNumber;
in >> setNumber;
for (size_t i = 0 ; i < setNumber; ++i)
{
size_t tmp;
in >> tmp;
tmpSet.set(tmp);
}
bitsets.push_back(tmpSet);
}
FCA::ImplicationInd implTmp;
implTmp.Premise().resize(attrNumber);
implTmp.Conclusion().resize(attrNumber);
for (size_t implInd = 0; implInd < implNumber; ++implInd)
{
implTmp.Premise().reset();
implTmp.Conclusion().reset();
size_t premiseNumber;
size_t conclusionNumber;
in >> premiseNumber >> conclusionNumber;
for (size_t i = 0; i < premiseNumber; ++i)
{
size_t tmp;
in >> tmp;
implTmp.Premise().set(tmp);
}
for (size_t i = 0; i < conclusionNumber; ++i)
{
size_t tmp;
in >> tmp;
implTmp.Conclusion().set(tmp);
}
implTmp.Conclusion() |= implTmp.Premise();
implications.push_back(implTmp);
}
}
int LinearAffineEq::findLinearEquivalences(bsetElem * S1, bsetElem * S1inv,
bsetElem * S2, bsetElem * S2inv,
linearEquivalencesList * list){
int count = 0;
int i;
bset Ua, Ub, Na, Nb, Ca, Cb;
bsetElem guesses1[size-1]; // random guesses for A(x) mapping
randomPermutationT(guesses1, size-1, 1); // make it random - random permutation
// recursive stack for guesses
recStack_t recStack;
recStack.resize(1); // force stack to contain 1 root element - guess
recStack.reserve(dim); // we will need at most <dim> vectors, but 2 should be enough!
int stackIdx=-1; // -1 initialization -> has to be initialized in routine
// Known mapping
smap mapA, mapB;
// init Ua, Ub, Na, Nb
// Our Sboxes don't map 0 to 0, so we can save this mapping.
Ca.insert(0); Na.insert(0);
Cb.insert(0); Nb.insert(0);
mapA.insert(smapElem(0, 0));
mapB.insert(smapElem(0, 0));
// Random initialization of Ua, Ub
bsetElem rndInit[255];
randomPermutationT(rndInit, 255, 1);
for(i=0; i<255; i++){
Ua.insert(rndInit[i]);
Ub.insert(rndInit[i]);
}
bool guessRejected=false;
bset::const_iterator it1, it2, it3;
while(Ua.empty()==false && Ub.empty()==false){
if (verbosity){
cout << endl << "===================================================================================="
<< endl << "Main cycle started " << endl;
}
//
// Starting with new guess
//
if (Na.empty() && Nb.empty()){
//
// 1. If previous guess rejected, restore Ca, Cb, Ua, Ub
// Guess A(x) for some x \in Ua
// Set Na = {x}, Ua = Ua / {x}
//
bsetElem x;
//
// Guess rejected? recover last backups
//
if (guessRejected){
// Is everything done?
if (stackIdx < 0) {
if (verbosity) cout << "All possible guesses exhausted" << endl;
break;
}
// Revert backups for state variables
linEqGuess_t & gs = recStack[stackIdx];
Ca = gs.Ca; Cb = gs.Cb;
Ua = gs.Ua; Ub = gs.Ub;
mapA = gs.mapA; mapB = gs.mapB;
// select same X as in previous case
x = gs.guessKey;
Na.insert(x);
guessRejected=false;
gs.idx+=1;
// refresh 0 if applicable in Na, Nb
if (S1[0] != 0 && S2[0] != 0){
Na.insert(0);
Nb.insert(0);
}
if (verbosity){
cout << "#GuessWasRejected" << endl;
}
} else {
// Guess was not rejected
// -> descent in recursive stack
recStack.resize( recStack.size() + 1);
stackIdx+=1;
linEqGuess_t & gs = recStack[stackIdx];
// At first, backup Ca, Cb, Ua, Ub - will be restored in case of incorrect guess
gs.Ua = Ua; gs.Ub = Ub;
gs.Ca = Ca; gs.Cb = Cb;
gs.mapA = mapA; gs.mapB = mapB;
gs.idx = 0;
// Chose new X and pick value for it
// Keep in mind linearity of mapping, so avoid duplicities.
if (randomizeXGuess){
int rnd = phrand() % Ua.size();
it1 = Ua.begin(); for(i=0; i<rnd; ++i, ++it1);
x = *it1; Ua.erase(it1);
Na.insert(x);
} else {
// No X randomization, just pick basis vectors if possible
bool baseFound=false;
for(i=0; i<(signed int)dim; i++){
if (Ua.count(1<<i)>0){
x = 1<<i;
Ua.erase(x);
Na.insert(x);
baseFound=true;
break;
}
}
// No basis vector is available in Ua, just pick first one
if (!baseFound){
it1 = Ua.begin(); x = *it1; Ua.erase(it1);
Na.insert(x);
}
}
}
linEqGuess_t & gs = recStack[stackIdx];
if (verbosity){
cout << "Guess index=" << gs.idx << "; stackIdx=" << stackIdx << endl;
}
// Guess A(x) value, avoid duplicate with guesses in recursive stack
gs.guessKey = x;
gs.guessVal = 0;
for(bool freeGuess=true; gs.idx < size; gs.idx++, freeGuess=true){
gs.guessVal = guesses1[gs.idx];
for(i=0; i<stackIdx; i++) {
if (recStack[i].guessVal == gs.guessVal) { freeGuess=false; break; }
}
if (freeGuess) break;
}
// Is possible to guess ? check exhaustion
if (gs.idx >= (size-1)){
// terminate this stack level, invalid guess one level above
guessRejected=true;
Na.clear(); Nb.clear();
stackIdx-=1;
if (verbosity) cout << "Decrementing stackIdx to = " << stackIdx << "; index is exhausted; idx: " << gs.idx << endl;
continue;
}
mapA.insert(smapElem(gs.guessKey, gs.guessVal));
if (verbosity){
cout << "New guess;" << endl;
for(i=0; i<=stackIdx; i++) {
cout << "G["<<i<<"]: x=" << (recStack[i].guessKey) << endl;
cout << "G["<<i<<"]: A(x) = " << (recStack[i].guessVal) << endl;
cout << "G["<<i<<"]: idx = " << recStack[i].idx << endl;
}
}
}
//
// Na cycle
//
while(Na.empty() == false){
bsetElem x;
if (verbosity) cout << endl << "A: Cycle 1 start, |Na| = " << Na.size() << endl;
// Pick x \in Na; Na = Na \ {x};
it1 = Na.begin(); x = *it1; Na.erase(it1);
if (verbosity) cout << "A: newX is [" << x << "]" << endl;
// Nb = S2( x + Ca ) \ Cb
if (Ca.size() > 0){
bset tmpSet;
for (it1=Ca.begin(); it1!=Ca.end(); ++it1){
bsetElem curr = *it1;
bsetElem tmp = x ^ curr;
tmpSet.insert(S2[tmp]);
// Use linearity of A to build mapping for tmp, we will need this values later
mapA.insert(smapElem(tmp, mapA[x] ^ mapA[curr]));
if (verbosity){
cout << " curr=" << setw(2) << (curr)
<< "; tmp = " << setw(2) << tmp
<< "; S2[tmp] = " << setw(2) << S2[tmp] << endl;
cout << "A: adding mapping for A(x^curr) = A(" << (tmp) << ") = A(x) ^ A(curr) = " << mapA[x] << " ^ " << mapA[curr] << " = " << (mapA[x] ^ mapA[curr]) << endl;
}
//
// A(x) = S^{-1}_1 (B(S_2(x)))
// S1 * A = B * S2
if (Cb.count(S2[tmp])==0){
const bsetElem amap = mapA[tmp];
mapB.insert(smapElem(S2[tmp], S1[amap]));
if (verbosity) cout << "A: adding mapping for B(S2(tmp)) = B("<< S2[tmp] <<") = S1(A(tmp)) = S1(" << amap << ") = " << S1[amap] << endl;
}
}
Nb = LinearAffineEq::setDiff(tmpSet, Cb);
}
// Ca = Ca U (x + Ca)
bset tmpSet(Ca);
for(it1=tmpSet.begin(); it1 != tmpSet.end(); ++it1){
Ca.insert( x ^ (*it1) );
}
Ca.insert(x);
// Check if we have enough vectors to build A and B mappings from them. We need 8 linearly independent
// vectors in A mapping to build its matrix representation and to continue with computation.
//
// Check if B is invertible linear mapping, if yes, derive A and check A,B on all points that left in Ua, Ub
double vectKnown = Nb.size() + ceil(log2(Cb.size()));
if (verbosity) cout << "A: vect knownB: " << vectKnown << endl;
if (vectKnown >= dim){
mat_GF2 Ta, Tb, Tbinv; smap mapBinv;
if (verbosity) cout << "A: checking whether B is linear invertible: " << vectKnown << endl;
int result = checkInvertibleLinear(Ub, Ua, mapB, mapA, S1, S1inv, S2, S2inv, Tb, Ta, Tbinv, mapBinv, false);
if (result==-1 || result==-2) continue;
if (result==-3 || result==-4 || result==-5){
guessRejected=true;
Na.clear(); Nb.clear();
break;
}
if (verbosity) cout << "Process results here!! Everything OK" << endl;
if (list!=NULL){
linearEquiv_t tmpStruct;
tmpStruct.Ta = Ta; tmpStruct.Tb = Tb;
tmpStruct.TaV = mapA; tmpStruct.TbV = mapB;
tmpStruct.Tbinv = Tbinv; tmpStruct.TbinvV = mapBinv;
list->push_back(tmpStruct);
}
count+=1;
relationsCount+=1;
guessRejected=true;
Na.clear(); Nb.clear();
}
}
if (verbosity) cout << endl;
//
// Nb cycle
//
while(Nb.empty() == false){
bsetElem x;
it1 = Nb.begin(); x = *it1; Nb.erase(it1);
if (verbosity){
cout << "B: Cycle 2 start, |Nb| = " << Nb.size() << endl;
cout << "B: newX is [" << x << "]" << endl;
}
if (Cb.size() > 0){
bset tmpSet;
for (it1=Cb.begin(); it1!=Cb.end(); ++it1){
bsetElem curr = *it1;
bsetElem tmp = x ^ curr;
tmpSet.insert(S2inv[tmp]);
// Use linearity of B to build mapping for tmp
mapB.insert(smapElem(tmp, mapB[x] ^ mapB[curr]));
if (verbosity){
cout << " curr=" << setw(2) << (curr)
<< "; tmp = " << setw(2) << tmp
<< "; S2inv[tmp] = " << setw(2) << S2inv[tmp] << endl;
cout << "B: adding mapping for B(x^curr) = B(" << (tmp) << ") = B(x) ^ B(curr) = " << mapB[x] << " ^ " << mapB[curr] << " = " << (mapB[x] ^ mapB[curr]) << endl;
}
//
// A = S^{-1}_1 * B * S_2
// A * S_2^{-1} = S^{-1}_1 * B
if (Ca.count(S2inv[tmp])==0){
const bsetElem bmap = mapB[tmp];
mapA.insert(smapElem(S2inv[tmp], S1inv[bmap]));
if (verbosity) cout << "B: adding mapping for A(S2inv(tmp)) = A(" << S2inv[tmp] << ") = S1inv(B(tmp)) = S1inv(" << bmap << ") = " << (S1inv[bmap]) << endl;
}
}
Na = LinearAffineEq::setDiff(tmpSet, Ca);
}
// Cb = Cb U (x + Cb)
bset tmpSet(Cb);
for(it1=tmpSet.begin(); it1 != tmpSet.end(); ++it1){
Cb.insert( x ^ *it1 );
}
Cb.insert(x);
// Check if we have enough vectors to build A and B mappings from them. We need 8 linearly independent
// vectors in B mapping to build its matrix representation and to continue with computation.
//
// Check if A is invertible linear mapping, if yes, derive B and check A,B on all points that left in Ua, Ub
double vectKnown = Na.size() + ceil(log2(Ca.size()));
if (verbosity) cout << "B: vect knownA: " << vectKnown << endl;
if (vectKnown>=dim){
if (verbosity) cout << "B: ## check linearity of A, derive,..." << endl;
mat_GF2 Ta, Tb, Tbinv; smap mapBinv;
if (verbosity) cout << "B: checking whether A is linear invertible: " << vectKnown << endl;
int result = checkInvertibleLinear(Ua, Ub, mapA, mapB, S1, S1inv, S2, S2inv, Ta, Tb, Tbinv, mapBinv, true);
if (result==-1 || result==-2) continue;
if (result==-3 || result==-4 || result==-5){
guessRejected=true;
Na.clear(); Nb.clear();
break;
}
if (verbosity) cout << "Process results here!! Everything OK" << endl;
if (list!=NULL){
linearEquiv_t tmpStruct;
tmpStruct.Ta = Ta; tmpStruct.Tb = Tb;
tmpStruct.TaV = mapA; tmpStruct.TbV = mapB;
tmpStruct.Tbinv = Tbinv; tmpStruct.TbinvV = mapBinv;
list->push_back(tmpStruct);
}
count += 1;
relationsCount+=1;
guessRejected=true;
Na.clear(); Nb.clear();
}
}
// Ua = Ua \ Ca
// Ub = Ub \ Cb
Ua = LinearAffineEq::setDiff(Ua, Ca);
Ub = LinearAffineEq::setDiff(Ub, Cb);
if (verbosity){
cout << endl << "EEEnd of both cycles, remove Ca, Cb from Ua Ub" << endl;
cout << " |Ua| = " << setw(3) << Ua.size()
<< " |Ub| = " << setw(3) << Ub.size()
<< " |Ca| = " << setw(3) << Ca.size()
<< " |Cb| = " << setw(3) << Cb.size()
<< " |Na| = " << setw(3) << Na.size()
<< " |Nb| = " << setw(3) << Nb.size() << endl;
cout << "Dump mapA: " << endl;
LinearAffineEq::dumpMap(mapA);
cout << "Dump mapB: " << endl;
LinearAffineEq::dumpMap(mapB);
}
}
if (verbosity) cout << "Finishing linear equivalence cycle " << endl;
return count;
}