nsresult VariableLengthPrefixSet::GetPrefixes(PrefixStringMap& aPrefixMap) { MutexAutoLock lock(mLock); // 4-bytes prefixes are handled by nsUrlClassifierPrefixSet. FallibleTArray<uint32_t> array; nsresult rv = mFixedPrefixSet->GetPrefixesNative(array); NS_ENSURE_SUCCESS(rv, rv); size_t count = array.Length(); if (count) { nsCString* prefixes = new nsCString(); if (!prefixes->SetLength(PREFIX_SIZE_FIXED * count, fallible)) { return NS_ERROR_OUT_OF_MEMORY; } // Writing integer array to character array uint32_t* begin = reinterpret_cast<uint32_t*>(prefixes->BeginWriting()); for (uint32_t i = 0; i < count; i++) { begin[i] = NativeEndian::swapToBigEndian(array[i]); } aPrefixMap.Put(PREFIX_SIZE_FIXED, prefixes); } // Copy variable-length prefix set for (auto iter = mVLPrefixSet.ConstIter(); !iter.Done(); iter.Next()) { aPrefixMap.Put(iter.Key(), new nsCString(*iter.Data())); } return NS_OK; }
void PrefixArrayToPrefixStringMap(const nsTArray<nsCString>& prefixArray, PrefixStringMap& out) { out.Clear(); for (uint32_t i = 0; i < prefixArray.Length(); i++) { const nsCString& prefix = prefixArray[i]; nsCString* prefixString = out.LookupOrAdd(prefix.Length()); prefixString->Append(prefix.BeginReading(), prefix.Length()); } }
static void CheckContent(VariableLengthPrefixSet* pset, PrefixStringMap& expected) { PrefixStringMap vlPSetMap; pset->GetPrefixes(vlPSetMap); for (auto iter = vlPSetMap.Iter(); !iter.Done(); iter.Next()) { nsCString* expectedPrefix = expected.Get(iter.Key()); nsCString* resultPrefix = iter.Data(); ASSERT_TRUE(resultPrefix->Equals(*expectedPrefix)); } }
VLPrefixSet::VLPrefixSet(const PrefixStringMap& aMap) : mCount(0) { for (auto iter = aMap.ConstIter(); !iter.Done(); iter.Next()) { uint32_t size = iter.Key(); mMap.Put(size, new PrefixString(*iter.Data(), size)); mCount += iter.Data()->Length() / size; } }
static void AppendPrefixToMap(PrefixStringMap& prefixes, nsDependentCSubstring& prefix) { if (!prefix.Length()) { return; } nsCString* prefixString = prefixes.LookupOrAdd(prefix.Length()); prefixString->Append(prefix.BeginReading(), prefix.Length()); }
void VLPrefixSet::Merge(PrefixStringMap& aPrefixMap) { for (auto iter = mMap.ConstIter(); !iter.Done(); iter.Next()) { nsCString* prefixString = aPrefixMap.LookupOrAdd(iter.Key()); PrefixString* str = iter.Data(); if (str->get()) { prefixString->Append(str->get(), str->remaining()); } } }
// Test setting prefix set with only 5~32 bytes prefixes TEST(VariableLengthPrefixSet, VariableLengthSet) { RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet; pset->Init(NS_LITERAL_CSTRING("test")); PrefixStringMap map; _PrefixArray array = { _Prefix("bravo"), _Prefix("charlie"), _Prefix("delta"), _Prefix("EchoEchoEchoEchoEcho"), _Prefix("foxtrot"), _Prefix("GolfGolfGolfGolfGolfGolfGolfGolf"), _Prefix("hotel"), _Prefix("november"), _Prefix("oscar"), _Prefix("quebec"), _Prefix("romeo"), _Prefix("sierrasierrasierrasierrasierra"), _Prefix("Tango"), _Prefix("whiskey"), _Prefix("yankee"), _Prefix("ZuluZuluZuluZulu") }; SetupPrefixMap(array, map); pset->SetPrefixes(map); DoExpectedLookup(pset, array); DoRandomLookup(pset, 1000, array); CheckContent(pset, map); // Run random test array.Clear(); map.Clear(); RandomPrefixes(1500, 5, 32, array); SetupPrefixMap(array, map); pset->SetPrefixes(map); DoExpectedLookup(pset, array); DoRandomLookup(pset, 1000, array); CheckContent(pset, map); }
// Test setting prefix set with both 4-bytes prefixes and 5~32 bytes prefixes TEST(VariableLengthPrefixSet, MixedPrefixSet) { RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet; pset->Init(NS_LITERAL_CSTRING("test")); PrefixStringMap map; _PrefixArray array = { _Prefix("enus"), _Prefix("apollo"), _Prefix("mars"), _Prefix("Hecatonchires cyclopes"), _Prefix("vesta"), _Prefix("neptunus"), _Prefix("jupiter"), _Prefix("diana"), _Prefix("minerva"), _Prefix("ceres"), _Prefix("Aidos,Adephagia,Adikia,Aletheia"), _Prefix("hecatonchires"), _Prefix("alcyoneus"), _Prefix("hades"), _Prefix("vulcanus"), _Prefix("juno"), _Prefix("mercury"), _Prefix("Stheno, Euryale and Medusa") }; SetupPrefixMap(array, map); pset->SetPrefixes(map); DoExpectedLookup(pset, array); DoRandomLookup(pset, 1000, array); CheckContent(pset, map); // Run random test array.Clear(); map.Clear(); RandomPrefixes(1500, 4, 32, array); SetupPrefixMap(array, map); pset->SetPrefixes(map); DoExpectedLookup(pset, array); DoRandomLookup(pset, 1000, array); CheckContent(pset, map); }
static void SetupPrefixMap(const _PrefixArray& array, PrefixStringMap& map) { map.Clear(); // Buckets are keyed by prefix length and contain an array of // all prefixes of that length. nsClassHashtable<nsUint32HashKey, _PrefixArray> table; for (uint32_t i = 0; i < array.Length(); i++) { _PrefixArray* prefixes = table.Get(array[i].Length()); if (!prefixes) { prefixes = new _PrefixArray(); table.Put(array[i].Length(), prefixes); } prefixes->AppendElement(array[i]); } // The resulting map entries will be a concatenation of all // prefix data for the prefixes of a given size. for (auto iter = table.Iter(); !iter.Done(); iter.Next()) { uint32_t size = iter.Key(); uint32_t count = iter.Data()->Length(); _Prefix* str = new _Prefix(); str->SetLength(size * count); char* dst = str->BeginWriting(); iter.Data()->Sort(); for (uint32_t i = 0; i < count; i++) { memcpy(dst, iter.Data()->ElementAt(i).get(), size); dst += size; } map.Put(size, str); } }
// Test setting prefix set with only 4-bytes prefixes TEST(VariableLengthPrefixSet, FixedLengthSet) { srand(time(nullptr)); RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet; pset->Init(NS_LITERAL_CSTRING("test")); PrefixStringMap map; _PrefixArray array = { _Prefix("alph"), _Prefix("brav"), _Prefix("char"), _Prefix("delt"), _Prefix("echo"), _Prefix("foxt"), }; SetupPrefixMap(array, map); pset->SetPrefixes(map); DoExpectedLookup(pset, array); DoRandomLookup(pset, 1000, array); CheckContent(pset, map); // Run random test array.Clear(); map.Clear(); RandomPrefixes(1500, 4, 4, array); SetupPrefixMap(array, map); pset->SetPrefixes(map); DoExpectedLookup(pset, array); DoRandomLookup(pset, 1000, array); CheckContent(pset, map); }
// Please see https://bug1287058.bmoattachments.org/attachment.cgi?id=8795366 // for detail about partial update algorithm. nsresult LookupCacheV4::ApplyPartialUpdate(TableUpdateV4* aTableUpdate, PrefixStringMap& aInputMap, PrefixStringMap& aOutputMap) { MOZ_ASSERT(aOutputMap.IsEmpty()); // oldPSet contains prefixes we already have or we just merged last round. // addPSet contains prefixes stored in tableUpdate which should be merged with oldPSet. VLPrefixSet oldPSet(aInputMap); VLPrefixSet addPSet(aTableUpdate->Prefixes()); // RemovalIndiceArray is a sorted integer array indicating the index of prefix we should // remove from the old prefix set(according to lexigraphic order). // |removalIndex| is the current index of RemovalIndiceArray. // |numOldPrefixPicked| is used to record how many prefixes we picked from the old map. TableUpdateV4::RemovalIndiceArray& removalArray = aTableUpdate->RemovalIndices(); uint32_t removalIndex = 0; int32_t numOldPrefixPicked = -1; nsDependentCSubstring smallestOldPrefix; nsDependentCSubstring smallestAddPrefix; // This is used to avoid infinite loop for partial update algorithm. // The maximum loops will be the number of old prefixes plus the number of add prefixes. uint32_t index = oldPSet.Count() + addPSet.Count() + 1; for(;index > 0; index--) { // Get smallest prefix from the old prefix set if we don't have one if (smallestOldPrefix.IsEmpty()) { // If prefixes from the old prefix set are all merged, // then we can merge the entire add prefix set directly. if (!oldPSet.GetSmallestPrefix(smallestOldPrefix)) { AppendPrefixToMap(aOutputMap, smallestAddPrefix); addPSet.Merge(aOutputMap); break; } } // Get smallest prefix from add prefix set if we don't have one if (smallestAddPrefix.IsEmpty()) { // If add prefixes are all merged and there is no removalIndices left, // then merge the entire old prefix set directly. If there are still // removalIndices left, we should still merge prefixes one by one // to know which prefix from old prefix set should be removed. if (!addPSet.GetSmallestPrefix(smallestAddPrefix) && removalIndex >= removalArray.Length()) { AppendPrefixToMap(aOutputMap, smallestOldPrefix); oldPSet.Merge(aOutputMap); break; } } // Compare the smallest string in old prefix set and add prefix set, merge the // smaller one into new map to ensure merged string still follows // lexigraphic order. if (smallestOldPrefix < smallestAddPrefix || smallestAddPrefix.IsEmpty()) { numOldPrefixPicked++; // If the number of picks from old map matches the removalIndex, then this prefix // will be removed by not merging it to new map. if (removalIndex < removalArray.Length() && numOldPrefixPicked == removalArray[removalIndex]) { removalIndex++; } else { AppendPrefixToMap(aOutputMap, smallestOldPrefix); } smallestOldPrefix.SetLength(0); } else if (smallestOldPrefix > smallestAddPrefix || smallestOldPrefix.IsEmpty()){ AppendPrefixToMap(aOutputMap, smallestAddPrefix); smallestAddPrefix.SetLength(0); } else { NS_WARNING("Add prefix should not exist in the original prefix set."); Telemetry::Accumulate(Telemetry::URLCLASSIFIER_UPDATE_ERROR_TYPE, DUPLICATE_PREFIX); return NS_ERROR_FAILURE; } } // We expect index will be greater to 0 because max number of runs will be // the number of original prefix plus add prefix. if (index <= 0) { NS_WARNING("There are still prefixes remaining after reaching maximum runs."); Telemetry::Accumulate(Telemetry::URLCLASSIFIER_UPDATE_ERROR_TYPE, INFINITE_LOOP); return NS_ERROR_FAILURE; } if (removalIndex < removalArray.Length()) { NS_WARNING("There are still prefixes to remove after exhausting the old PrefixSet."); Telemetry::Accumulate(Telemetry::URLCLASSIFIER_UPDATE_ERROR_TYPE, WRONG_REMOVAL_INDICES); return NS_ERROR_FAILURE; } return NS_OK; }