nsresult
VariableLengthPrefixSet::GetPrefixes(PrefixStringMap& aPrefixMap)
{
MutexAutoLock lock(mLock);
// 4-bytes prefixes are handled by nsUrlClassifierPrefixSet.
FallibleTArray<uint32_t> array;
nsresult rv = mFixedPrefixSet->GetPrefixesNative(array);
NS_ENSURE_SUCCESS(rv, rv);
size_t count = array.Length();
if (count) {
nsCString* prefixes = new nsCString();
if (!prefixes->SetLength(PREFIX_SIZE_FIXED * count, fallible)) {
return NS_ERROR_OUT_OF_MEMORY;
}
// Writing integer array to character array
uint32_t* begin = reinterpret_cast<uint32_t*>(prefixes->BeginWriting());
for (uint32_t i = 0; i < count; i++) {
begin[i] = NativeEndian::swapToBigEndian(array[i]);
}
aPrefixMap.Put(PREFIX_SIZE_FIXED, prefixes);
}
// Copy variable-length prefix set
for (auto iter = mVLPrefixSet.ConstIter(); !iter.Done(); iter.Next()) {
aPrefixMap.Put(iter.Key(), new nsCString(*iter.Data()));
}
return NS_OK;
}
void
PrefixArrayToPrefixStringMap(const nsTArray<nsCString>& prefixArray,
PrefixStringMap& out)
{
out.Clear();
for (uint32_t i = 0; i < prefixArray.Length(); i++) {
const nsCString& prefix = prefixArray[i];
nsCString* prefixString = out.LookupOrAdd(prefix.Length());
prefixString->Append(prefix.BeginReading(), prefix.Length());
}
}
static void CheckContent(VariableLengthPrefixSet* pset,
PrefixStringMap& expected)
{
PrefixStringMap vlPSetMap;
pset->GetPrefixes(vlPSetMap);
for (auto iter = vlPSetMap.Iter(); !iter.Done(); iter.Next()) {
nsCString* expectedPrefix = expected.Get(iter.Key());
nsCString* resultPrefix = iter.Data();
ASSERT_TRUE(resultPrefix->Equals(*expectedPrefix));
}
}
VLPrefixSet::VLPrefixSet(const PrefixStringMap& aMap)
: mCount(0)
{
for (auto iter = aMap.ConstIter(); !iter.Done(); iter.Next()) {
uint32_t size = iter.Key();
mMap.Put(size, new PrefixString(*iter.Data(), size));
mCount += iter.Data()->Length() / size;
}
}
static void
AppendPrefixToMap(PrefixStringMap& prefixes, nsDependentCSubstring& prefix)
{
if (!prefix.Length()) {
return;
}
nsCString* prefixString = prefixes.LookupOrAdd(prefix.Length());
prefixString->Append(prefix.BeginReading(), prefix.Length());
}
void
VLPrefixSet::Merge(PrefixStringMap& aPrefixMap) {
for (auto iter = mMap.ConstIter(); !iter.Done(); iter.Next()) {
nsCString* prefixString = aPrefixMap.LookupOrAdd(iter.Key());
PrefixString* str = iter.Data();
if (str->get()) {
prefixString->Append(str->get(), str->remaining());
}
}
}
// Test setting prefix set with only 5~32 bytes prefixes
TEST(VariableLengthPrefixSet, VariableLengthSet)
{
RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet;
pset->Init(NS_LITERAL_CSTRING("test"));
PrefixStringMap map;
_PrefixArray array = { _Prefix("bravo"), _Prefix("charlie"), _Prefix("delta"),
_Prefix("EchoEchoEchoEchoEcho"), _Prefix("foxtrot"),
_Prefix("GolfGolfGolfGolfGolfGolfGolfGolf"),
_Prefix("hotel"), _Prefix("november"),
_Prefix("oscar"), _Prefix("quebec"), _Prefix("romeo"),
_Prefix("sierrasierrasierrasierrasierra"),
_Prefix("Tango"), _Prefix("whiskey"), _Prefix("yankee"),
_Prefix("ZuluZuluZuluZulu")
};
SetupPrefixMap(array, map);
pset->SetPrefixes(map);
DoExpectedLookup(pset, array);
DoRandomLookup(pset, 1000, array);
CheckContent(pset, map);
// Run random test
array.Clear();
map.Clear();
RandomPrefixes(1500, 5, 32, array);
SetupPrefixMap(array, map);
pset->SetPrefixes(map);
DoExpectedLookup(pset, array);
DoRandomLookup(pset, 1000, array);
CheckContent(pset, map);
}
// Test setting prefix set with both 4-bytes prefixes and 5~32 bytes prefixes
TEST(VariableLengthPrefixSet, MixedPrefixSet)
{
RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet;
pset->Init(NS_LITERAL_CSTRING("test"));
PrefixStringMap map;
_PrefixArray array = { _Prefix("enus"), _Prefix("apollo"), _Prefix("mars"),
_Prefix("Hecatonchires cyclopes"),
_Prefix("vesta"), _Prefix("neptunus"), _Prefix("jupiter"),
_Prefix("diana"), _Prefix("minerva"), _Prefix("ceres"),
_Prefix("Aidos,Adephagia,Adikia,Aletheia"),
_Prefix("hecatonchires"), _Prefix("alcyoneus"), _Prefix("hades"),
_Prefix("vulcanus"), _Prefix("juno"), _Prefix("mercury"),
_Prefix("Stheno, Euryale and Medusa")
};
SetupPrefixMap(array, map);
pset->SetPrefixes(map);
DoExpectedLookup(pset, array);
DoRandomLookup(pset, 1000, array);
CheckContent(pset, map);
// Run random test
array.Clear();
map.Clear();
RandomPrefixes(1500, 4, 32, array);
SetupPrefixMap(array, map);
pset->SetPrefixes(map);
DoExpectedLookup(pset, array);
DoRandomLookup(pset, 1000, array);
CheckContent(pset, map);
}
static void SetupPrefixMap(const _PrefixArray& array,
PrefixStringMap& map)
{
map.Clear();
// Buckets are keyed by prefix length and contain an array of
// all prefixes of that length.
nsClassHashtable<nsUint32HashKey, _PrefixArray> table;
for (uint32_t i = 0; i < array.Length(); i++) {
_PrefixArray* prefixes = table.Get(array[i].Length());
if (!prefixes) {
prefixes = new _PrefixArray();
table.Put(array[i].Length(), prefixes);
}
prefixes->AppendElement(array[i]);
}
// The resulting map entries will be a concatenation of all
// prefix data for the prefixes of a given size.
for (auto iter = table.Iter(); !iter.Done(); iter.Next()) {
uint32_t size = iter.Key();
uint32_t count = iter.Data()->Length();
_Prefix* str = new _Prefix();
str->SetLength(size * count);
char* dst = str->BeginWriting();
iter.Data()->Sort();
for (uint32_t i = 0; i < count; i++) {
memcpy(dst, iter.Data()->ElementAt(i).get(), size);
dst += size;
}
map.Put(size, str);
}
}
// Test setting prefix set with only 4-bytes prefixes
TEST(VariableLengthPrefixSet, FixedLengthSet)
{
srand(time(nullptr));
RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet;
pset->Init(NS_LITERAL_CSTRING("test"));
PrefixStringMap map;
_PrefixArray array = { _Prefix("alph"), _Prefix("brav"), _Prefix("char"),
_Prefix("delt"), _Prefix("echo"), _Prefix("foxt"),
};
SetupPrefixMap(array, map);
pset->SetPrefixes(map);
DoExpectedLookup(pset, array);
DoRandomLookup(pset, 1000, array);
CheckContent(pset, map);
// Run random test
array.Clear();
map.Clear();
RandomPrefixes(1500, 4, 4, array);
SetupPrefixMap(array, map);
pset->SetPrefixes(map);
DoExpectedLookup(pset, array);
DoRandomLookup(pset, 1000, array);
CheckContent(pset, map);
}
// Please see https://bug1287058.bmoattachments.org/attachment.cgi?id=8795366
// for detail about partial update algorithm.
nsresult
LookupCacheV4::ApplyPartialUpdate(TableUpdateV4* aTableUpdate,
PrefixStringMap& aInputMap,
PrefixStringMap& aOutputMap)
{
MOZ_ASSERT(aOutputMap.IsEmpty());
// oldPSet contains prefixes we already have or we just merged last round.
// addPSet contains prefixes stored in tableUpdate which should be merged with oldPSet.
VLPrefixSet oldPSet(aInputMap);
VLPrefixSet addPSet(aTableUpdate->Prefixes());
// RemovalIndiceArray is a sorted integer array indicating the index of prefix we should
// remove from the old prefix set(according to lexigraphic order).
// |removalIndex| is the current index of RemovalIndiceArray.
// |numOldPrefixPicked| is used to record how many prefixes we picked from the old map.
TableUpdateV4::RemovalIndiceArray& removalArray = aTableUpdate->RemovalIndices();
uint32_t removalIndex = 0;
int32_t numOldPrefixPicked = -1;
nsDependentCSubstring smallestOldPrefix;
nsDependentCSubstring smallestAddPrefix;
// This is used to avoid infinite loop for partial update algorithm.
// The maximum loops will be the number of old prefixes plus the number of add prefixes.
uint32_t index = oldPSet.Count() + addPSet.Count() + 1;
for(;index > 0; index--) {
// Get smallest prefix from the old prefix set if we don't have one
if (smallestOldPrefix.IsEmpty()) {
// If prefixes from the old prefix set are all merged,
// then we can merge the entire add prefix set directly.
if (!oldPSet.GetSmallestPrefix(smallestOldPrefix)) {
AppendPrefixToMap(aOutputMap, smallestAddPrefix);
addPSet.Merge(aOutputMap);
break;
}
}
// Get smallest prefix from add prefix set if we don't have one
if (smallestAddPrefix.IsEmpty()) {
// If add prefixes are all merged and there is no removalIndices left,
// then merge the entire old prefix set directly. If there are still
// removalIndices left, we should still merge prefixes one by one
// to know which prefix from old prefix set should be removed.
if (!addPSet.GetSmallestPrefix(smallestAddPrefix) &&
removalIndex >= removalArray.Length()) {
AppendPrefixToMap(aOutputMap, smallestOldPrefix);
oldPSet.Merge(aOutputMap);
break;
}
}
// Compare the smallest string in old prefix set and add prefix set, merge the
// smaller one into new map to ensure merged string still follows
// lexigraphic order.
if (smallestOldPrefix < smallestAddPrefix ||
smallestAddPrefix.IsEmpty()) {
numOldPrefixPicked++;
// If the number of picks from old map matches the removalIndex, then this prefix
// will be removed by not merging it to new map.
if (removalIndex < removalArray.Length() &&
numOldPrefixPicked == removalArray[removalIndex]) {
removalIndex++;
} else {
AppendPrefixToMap(aOutputMap, smallestOldPrefix);
}
smallestOldPrefix.SetLength(0);
} else if (smallestOldPrefix > smallestAddPrefix ||
smallestOldPrefix.IsEmpty()){
AppendPrefixToMap(aOutputMap, smallestAddPrefix);
smallestAddPrefix.SetLength(0);
} else {
NS_WARNING("Add prefix should not exist in the original prefix set.");
Telemetry::Accumulate(Telemetry::URLCLASSIFIER_UPDATE_ERROR_TYPE,
DUPLICATE_PREFIX);
return NS_ERROR_FAILURE;
}
}
// We expect index will be greater to 0 because max number of runs will be
// the number of original prefix plus add prefix.
if (index <= 0) {
NS_WARNING("There are still prefixes remaining after reaching maximum runs.");
Telemetry::Accumulate(Telemetry::URLCLASSIFIER_UPDATE_ERROR_TYPE,
INFINITE_LOOP);
return NS_ERROR_FAILURE;
}
if (removalIndex < removalArray.Length()) {
NS_WARNING("There are still prefixes to remove after exhausting the old PrefixSet.");
Telemetry::Accumulate(Telemetry::URLCLASSIFIER_UPDATE_ERROR_TYPE,
WRONG_REMOVAL_INDICES);
return NS_ERROR_FAILURE;
}
return NS_OK;
}
