// YES I COPIED AND PASTED CODE my section leader would be so ashamed :D
void LSH::MinHash(THash<TMd5Sig, TIntSet>& ShingleToQuoteIds,
TVec<THash<TIntV, TIntSet> >& SignatureBandBuckets) {
TRnd RandomGenerator; // TODO: make this "more random" by incorporating time
for (int i = 0; i < NumBands; ++i) {
THash < TInt, TIntV > Inverted; // (QuoteID, QuoteSignatureForBand)
THash < TIntV, TIntSet > BandBuckets; // (BandSignature, QuoteIDs)
for (int j = 0; j < BandSize; ++j) {
// Create new signature
TVec < TMd5Sig > Signature;
ShingleToQuoteIds.GetKeyV(Signature);
Signature.Shuffle(RandomGenerator);
// Place in bucket - not very efficient
int SigLen = Signature.Len();
for (int k = 0; k < SigLen; ++k) {
TIntSet CurSet = ShingleToQuoteIds.GetDat(Signature[k]);
for (TIntSet::TIter l = CurSet.BegI(); l < CurSet.EndI(); l++) {
TInt Key = l.GetKey();
if (Inverted.IsKey(Key)) {
TIntV CurSignature = Inverted.GetDat(Key);
if (CurSignature.Len() <= j) {
CurSignature.Add(k);
Inverted.AddDat(Key, CurSignature);
}
} else {
TIntV NewSignature;
NewSignature.Add(k);
Inverted.AddDat(Key, NewSignature);
}
}
}
}
TIntV InvertedKeys;
Inverted.GetKeyV(InvertedKeys);
TInt InvertedLen = InvertedKeys.Len();
for (int k = 0; k < InvertedLen; ++k) {
TIntSet Bucket;
TIntV Signature = Inverted.GetDat(InvertedKeys[k]);
if (BandBuckets.IsKey(Signature)) {
Bucket = BandBuckets.GetDat(Signature);
}
Bucket.AddKey(InvertedKeys[k]);
BandBuckets.AddDat(Signature, Bucket);
}
SignatureBandBuckets.Add(BandBuckets);
Err("%d out of %d band signatures computed\n", i + 1, NumBands);
}
Err("Minhash step complete!\n");
}
void LSH::ComputeSignatures(THashSet<TMd5Sig>& Shingles,
THash<TMd5Sig, TIntV>& Signatures, int NumSignatures) {
if (NumSignatures < 1)
return;
TRnd RandomGenerator; // TODO: make this "more random" by incorporating time
TInt NumShingles = Shingles.Len();
for (int i = 0; i < NumSignatures; ++i) {
// Create new signature
TVec < TMd5Sig > Shuffle;
Shingles.GetKeyV(Shuffle);
Shuffle.Shuffle(RandomGenerator);
for (int j = 0; j < NumShingles; j++) {
TIntV Signature;
Signatures.IsKeyGetDat(Shuffle[j], Signature);
Signature.Add(j);
Signatures.AddDat(Shuffle[j], Signature);
}
}
Err("Computed %d signatures!\n", NumSignatures);
}
