void FlowNetwork::finalize(const Network& network, const Config& config, bool normalizeNodeFlow) { // TODO: Skip bipartite flow adjustment for directed / rawdir / .. ? if (network.isBipartite() && !config.skipAdjustBipartiteFlow) { // Only links between ordinary nodes and feature nodes in bipartite network // Don't code feature nodes -> distribute all flow from those to ordinary nodes unsigned int minBipartiteNodeIndex = network.numNodes() - network.numBipartiteNodes(); for (LinkVec::iterator linkIt(m_flowLinks.begin()); linkIt != m_flowLinks.end(); ++linkIt) { Link& link = *linkIt; link.flow *= 2; // Markov time 2 on the full network will correspond to markov time 1 between the real nodes. if (link.source >= minBipartiteNodeIndex) { m_nodeFlow[link.target] += link.flow; m_nodeFlow[link.source] = 0.0; // Doesn't matter if done multiple times on each node. } else if (config.parseAsUndirected()) { m_nodeFlow[link.source] += link.flow; m_nodeFlow[link.target] = 0.0; // Doesn't matter if done multiple times on each node. } } normalizeNodeFlow = true; } if (normalizeNodeFlow) { double sumNodeFlow = 0.0; for (unsigned int i = 0; i < m_nodeFlow.size(); ++i) sumNodeFlow += m_nodeFlow[i]; for (unsigned int i = 0; i < m_nodeFlow.size(); ++i) m_nodeFlow[i] /= sumNodeFlow; } }
QString MessageWidget::_filterText(QString rawText) { rawText = rawText .replace("&", "&") .replace("<", "<") .replace(">", ">") .replace("\"", """) .replace("'", "'") .replace(" ", " "); QRegExp linkPattern("((https?|ftp):(\\/\\/)+[\\w\\d:#@%/;$()~_?\\+-=\\\\\\.&]*)"); QStringList linkMatches; int pos = 0; while ((pos = linkPattern.indexIn(rawText, pos)) != -1) { QString link = linkPattern.cap(1); if (!linkMatches.contains(link)) { linkMatches.append(link); } pos += linkPattern.matchedLength(); } QStringListIterator linkIt(linkMatches); while (linkIt.hasNext()) { QString link(linkIt.next()); rawText = rawText.replace( link, QString("<a style=\"color: #008b9e; font: 700 10px; text-decoration: none;\" href=\"%1\">%1</a>").arg(link) ); } QRegExp userPattern("\\[\\[user:([0-9]+)\\]\\]"); QStringList userMatches; pos = 0; while ((pos = userPattern.indexIn(rawText, pos)) != -1) { userMatches.append(userPattern.cap(1)); pos += userPattern.matchedLength(); } QStringListIterator it(userMatches); while (it.hasNext()) { int uid = it.next().toInt(); if (uid > 0) { YammerNS::User *user = Yawner::getInstance()->getUserManager()->getUserById(uid); rawText = rawText.replace( QString("[[user:%1]]").arg(uid), QString("<a style=\"color: #008b9e; font: 700 10px; text-decoration: none;\" href=\"user:%1\">%2</a>") .arg(QString::number(uid), user->getName()) ); } } QString repliedToName; YammerNS::Message *repliedToMessage = Yawner::getInstance()->getMessageManager()->getRepliedToMessage(_message); if (repliedToMessage != 0) { repliedToName = repliedToMessage->getUser()->getName(); } if (!repliedToName.isEmpty()) { int threadId = _message->getThreadStarterId(); rawText.prepend(QString("@<a style=\"color: #FF5800; font: 700 10px; text-decoration: none;\" href=\"thread:%1\">%2</a>: ") .arg(QString::number(threadId), repliedToName)); } return rawText.replace(QString("\n"), QString("<br />")); }
void FlowNetwork::calculateFlow(const Network& network, const Config& config) { std::cout << "Calculating global flow... "; // Prepare data in sequence containers for fast access of individual elements unsigned int numNodes = network.numNodes(); m_nodeOutDegree.assign(numNodes, 0); m_sumLinkOutWeight.assign(numNodes, 0.0); m_nodeFlow.assign(numNodes, 0.0); m_nodeTeleportRates.assign(numNodes, 0.0); const LinkMap& linkMap = network.linkMap(); unsigned int numLinks = network.numLinks(); m_flowLinks.resize(numLinks); double totalLinkWeight = network.totalLinkWeight(); double sumUndirLinkWeight = 2 * totalLinkWeight - network.totalSelfLinkWeight(); unsigned int linkIndex = 0; for (LinkMap::const_iterator linkIt(linkMap.begin()); linkIt != linkMap.end(); ++linkIt) { unsigned int linkEnd1 = linkIt->first; const std::map<unsigned int, double>& subLinks = linkIt->second; for (std::map<unsigned int, double>::const_iterator subIt(subLinks.begin()); subIt != subLinks.end(); ++subIt, ++linkIndex) { unsigned int linkEnd2 = subIt->first; double linkWeight = subIt->second; ++m_nodeOutDegree[linkEnd1]; m_sumLinkOutWeight[linkEnd1] += linkWeight; if (linkEnd1 != linkEnd2) { // Possibly aggregate if no self-link if (config.isUndirected()) { m_sumLinkOutWeight[linkEnd2] += linkWeight; ++m_nodeOutDegree[linkEnd2]; } if (!config.outdirdir) m_nodeFlow[linkEnd2] += linkWeight / sumUndirLinkWeight; } m_nodeFlow[linkEnd1] += linkWeight / sumUndirLinkWeight; m_flowLinks[linkIndex] = Link(linkEnd1, linkEnd2, linkWeight); } } if (config.rawdir) { // Treat the link weights as flow (after global normalization) and // do one power iteration to set the node flow m_nodeFlow.assign(numNodes, 0.0); for (LinkVec::iterator linkIt(m_flowLinks.begin()); linkIt != m_flowLinks.end(); ++linkIt) { Link& link = *linkIt; link.flow /= totalLinkWeight; m_nodeFlow[link.target] += link.flow; } //Normalize node flow double sumNodeRank = 0.0; for (unsigned int i = 0; i < numNodes; ++i) sumNodeRank += m_nodeFlow[i]; for (unsigned int i = 0; i < numNodes; ++i) m_nodeFlow[i] /= sumNodeRank; std::cout << "using directed links with raw flow... done!" << std::endl; std::cout << "Total link weight: " << totalLinkWeight << "\n"; return; } if (!config.directed) { if (config.undirdir || config.outdirdir) { //Take one last power iteration std::vector<double> nodeFlowSteadyState(m_nodeFlow); m_nodeFlow.assign(numNodes, 0.0); for (LinkVec::iterator linkIt(m_flowLinks.begin()); linkIt != m_flowLinks.end(); ++linkIt) { Link& link = *linkIt; m_nodeFlow[link.target] += nodeFlowSteadyState[link.source] * link.flow / m_sumLinkOutWeight[link.source]; } //Normalize node flow double sumNodeRank = 0.0; for (unsigned int i = 0; i < numNodes; ++i) sumNodeRank += m_nodeFlow[i]; for (unsigned int i = 0; i < numNodes; ++i) m_nodeFlow[i] /= sumNodeRank; // Update link data to represent flow instead of weight for (LinkVec::iterator linkIt(m_flowLinks.begin()); linkIt != m_flowLinks.end(); ++linkIt) { Link& link = *linkIt; link.flow *= nodeFlowSteadyState[link.source] / m_sumLinkOutWeight[link.source] / sumNodeRank; } } else // undirected { for (unsigned int i = 0; i < numLinks; ++i) m_flowLinks[i].flow /= sumUndirLinkWeight; } if (config.outdirdir) std::cout << "counting only ingoing links... done!" << std::endl; else std::cout << "using undirected links" << (config.undirdir? ", switching to directed after steady state... done!" : "... done!") << std::endl; return; } std::cout << "using " << (config.recordedTeleportation ? "recorded" : "unrecorded") << " teleportation to " << (config.teleportToNodes ? "nodes" : "links") << "... " << std::flush; // Calculate the teleport rate distribution if (config.teleportToNodes) { const std::vector<double>& nodeWeights = network.nodeWeights(); if (nodeWeights.empty()) { double rate = 1.0 / numNodes; for (unsigned int i = 0; i < numNodes; ++i) m_nodeTeleportRates[i] = rate; } else { for (unsigned int i = 0; i < numNodes; ++i) m_nodeTeleportRates[i] = nodeWeights[i] / network.sumNodeWeights(); } } else // Teleport to nodes { // Teleport proportionally to out-degree, or in-degree if recorded teleportation. for (LinkVec::iterator linkIt(m_flowLinks.begin()); linkIt != m_flowLinks.end(); ++linkIt) { unsigned int toNode = config.recordedTeleportation ? linkIt->target : linkIt->source; m_nodeTeleportRates[toNode] += linkIt->flow / totalLinkWeight; } } // Normalize link weights with respect to its source nodes total out-link weight; for (LinkVec::iterator linkIt(m_flowLinks.begin()); linkIt != m_flowLinks.end(); ++linkIt) { linkIt->flow /= m_sumLinkOutWeight[linkIt->source]; } // Collect dangling nodes std::vector<unsigned int> danglings; for (unsigned int i = 0; i < numNodes; ++i) { if (m_nodeOutDegree[i] == 0) danglings.push_back(i); } // Calculate PageRank std::vector<double> nodeFlowTmp(numNodes, 0.0); unsigned int numIterations = 0; double alpha = config.teleportationProbability; double beta = 1.0 - alpha; double sqdiff = 1.0; double danglingRank = 0.0; do { // Calculate dangling rank danglingRank = 0.0; for (std::vector<unsigned int>::iterator danglingIt(danglings.begin()); danglingIt != danglings.end(); ++danglingIt) { danglingRank += m_nodeFlow[*danglingIt]; } // Flow from teleportation for (unsigned int i = 0; i < numNodes; ++i) { nodeFlowTmp[i] = (alpha + beta*danglingRank) * m_nodeTeleportRates[i]; } // Flow from links for (LinkVec::iterator linkIt(m_flowLinks.begin()); linkIt != m_flowLinks.end(); ++linkIt) { Link& link = *linkIt; nodeFlowTmp[link.target] += beta * link.flow * m_nodeFlow[link.source]; } // Update node flow from the power iteration above and check if converged double sum = 0.0; double sqdiff_old = sqdiff; sqdiff = 0.0; for (unsigned int i = 0; i < numNodes; ++i) { sum += nodeFlowTmp[i]; sqdiff += std::abs(nodeFlowTmp[i] - m_nodeFlow[i]); m_nodeFlow[i] = nodeFlowTmp[i]; } // Normalize if needed if (std::abs(sum - 1.0) > 1.0e-10) { std::cout << "(Normalizing ranks after " << numIterations << " power iterations with error " << (sum-1.0) << ") "; for (unsigned int i = 0; i < numNodes; ++i) { m_nodeFlow[i] /= sum; } } // Perturb the system if equilibrium if(sqdiff == sqdiff_old) { alpha += 1.0e-10; beta = 1.0 - alpha; } numIterations++; } while((numIterations < 200) && (sqdiff > 1.0e-15 || numIterations < 50)); double sumNodeRank = 1.0; if (!config.recordedTeleportation) { //Take one last power iteration excluding the teleportation (and normalize node flow to sum 1.0) sumNodeRank = 1.0 - danglingRank; m_nodeFlow.assign(numNodes, 0.0); for (LinkVec::iterator linkIt(m_flowLinks.begin()); linkIt != m_flowLinks.end(); ++linkIt) { Link& link = *linkIt; m_nodeFlow[link.target] += link.flow * nodeFlowTmp[link.source] / sumNodeRank; } beta = 1.0; } // Update the links with their global flow from the PageRank values. (Note: beta is set to 1 if unrec) for (LinkVec::iterator linkIt(m_flowLinks.begin()); linkIt != m_flowLinks.end(); ++linkIt) { linkIt->flow *= beta * nodeFlowTmp[linkIt->source] / sumNodeRank; } std::cout << "done in " << numIterations << " iterations!" << std::endl; }
void MemFlowNetwork::calculateFlow(const Network& net, const Config& config) { if (!config.isMemoryNetwork()) { FlowNetwork::calculateFlow(net, config); return; } Log() << "Calculating global flow... " << std::flush; const MemNetwork& network = static_cast<const MemNetwork&>(net); // Prepare data in sequence containers for fast access of individual elements unsigned int numM2Nodes = network.numM2Nodes(); // Copy to be able to modify std::vector<double> nodeOutDegree(network.outDegree()); std::vector<double> sumLinkOutWeight(network.sumLinkOutWeight()); m_nodeFlow.assign(numM2Nodes, 0.0); m_nodeTeleportRates.assign(numM2Nodes, 0.0); const MemNetwork::M2LinkMap& linkMap = network.m2LinkMap(); unsigned int numLinks = network.numM2Links(); m_flowLinks.resize(numLinks); double totalM2LinkWeight = network.totalM2LinkWeight(); double sumUndirLinkWeight = 2 * totalM2LinkWeight - network.totalMemorySelfLinkWeight(); unsigned int linkIndex = 0; const MemNetwork::M2NodeMap& nodeMap = network.m2NodeMap(); for (MemNetwork::M2LinkMap::const_iterator linkIt(linkMap.begin()); linkIt != linkMap.end(); ++linkIt) { const M2Node& m2source = linkIt->first; const std::map<M2Node, double>& subLinks = linkIt->second; for (std::map<M2Node, double>::const_iterator subIt(subLinks.begin()); subIt != subLinks.end(); ++subIt, ++linkIndex) { const M2Node& m2target = subIt->first; double linkWeight = subIt->second; // Get the indices for the m2 nodes MemNetwork::M2NodeMap::const_iterator nodeMapIt = nodeMap.find(m2source); if (nodeMapIt == nodeMap.end()) throw InputDomainError(io::Str() << "Couldn't find mapped index for source M2 node " << m2source); unsigned int sourceIndex = nodeMapIt->second; nodeMapIt = nodeMap.find(m2target); if (nodeMapIt == nodeMap.end()) throw InputDomainError(io::Str() << "Couldn't find mapped index for target M2 node " << m2target); unsigned int targetIndex = nodeMapIt->second; m_nodeFlow[sourceIndex] += linkWeight;// / sumUndirLinkWeight; m_flowLinks[linkIndex] = Link(sourceIndex, targetIndex, linkWeight); if (sourceIndex != targetIndex && !config.outdirdir) m_nodeFlow[targetIndex] += linkWeight;// / sumUndirLinkWeight; } } m_m2nodes.resize(numM2Nodes); for (MemNetwork::M2NodeMap::const_iterator m2nodeIt(nodeMap.begin()); m2nodeIt != nodeMap.end(); ++m2nodeIt) { m_m2nodes[m2nodeIt->second] = m2nodeIt->first; } unsigned int numM1Nodes = network.numNodes(); typedef std::multimap<double, unsigned int> PhysToMemWeightMap; std::vector<PhysToMemWeightMap> netPhysToMem(numM1Nodes); const LinkMap& m1LinkMap = network.linkMap(); // Map middle column in trigrams to target m2 nodes (source to link for m1 links) for (LinkMap::const_iterator linkIt(m1LinkMap.begin()); linkIt != m1LinkMap.end(); ++linkIt) { unsigned int linkEnd1 = linkIt->first; const std::map<unsigned int, double>& subLinks = linkIt->second; for (std::map<unsigned int, double>::const_iterator subIt(subLinks.begin()); subIt != subLinks.end(); ++subIt) { unsigned int linkEnd2 = subIt->first; double linkWeight = subIt->second; MemNetwork::M2NodeMap::const_iterator m2it = nodeMap.find(M2Node(linkEnd1, linkEnd2)); if (m2it == nodeMap.end()) throw InputDomainError(io::Str() << "Memory node (" << linkEnd1 << ", " << linkEnd2 << ") not indexed!"); unsigned int m2nodeIndex = m2it->second; PhysToMemWeightMap& physMap = netPhysToMem[linkEnd1]; physMap.insert(std::make_pair(linkWeight, m2nodeIndex)); } } // Add M1 flow to dangling M2 nodes unsigned int numDanglingM2Nodes = 0; unsigned int numSelfLinks = 0; double sumExtraLinkWeight = 0.0; for (unsigned int i = 0; i < numM2Nodes; ++i) { if (nodeOutDegree[i] == 0) { ++numDanglingM2Nodes; // We are in physIndex, lookup all mem nodes in that physical node // and add a link to the target node of those mem nodes (pre-mapped above) const PhysToMemWeightMap& physToMem = netPhysToMem[m_m2nodes[i].physIndex]; for(PhysToMemWeightMap::const_iterator it = physToMem.begin(); it != physToMem.end(); it++) { unsigned int from = i; unsigned int to = it->second; double linkWeight = it->first; if(linkWeight > 0.0) { if(from == to) { ++numSelfLinks; } else { ++nodeOutDegree[from]; sumLinkOutWeight[from] += linkWeight; sumExtraLinkWeight += linkWeight; m_nodeFlow[from] += linkWeight; if (config.isUndirected()) { ++nodeOutDegree[to]; sumLinkOutWeight[to] += linkWeight; } if (!config.outdirdir) { m_nodeFlow[to] += linkWeight; } if (from >= numM2Nodes || to >= numM2Nodes) { Log() << "\nRange error adding dangling links " << from << " " << to << " !!!"; } m_flowLinks.push_back(Link(from, to, linkWeight)); } } } } } if (m_flowLinks.size() - numLinks != 0) Log() << "\n -> Added " << (m_flowLinks.size() - numLinks) << " links to " << numDanglingM2Nodes << " dangling memory nodes -> " << m_flowLinks.size() << " links" << std::flush; totalM2LinkWeight += sumExtraLinkWeight; sumUndirLinkWeight = 2 * totalM2LinkWeight - network.totalMemorySelfLinkWeight(); numLinks = m_flowLinks.size(); // Normalize node flow for (unsigned int i = 0; i < numM2Nodes; ++i) m_nodeFlow[i] /= sumUndirLinkWeight; if (config.rawdir) { // Treat the link weights as flow (after global normalization) and // do one power iteration to set the node flow m_nodeFlow.assign(numM2Nodes, 0.0); for (LinkVec::iterator linkIt(m_flowLinks.begin()); linkIt != m_flowLinks.end(); ++linkIt) { Link& link = *linkIt; link.flow /= totalM2LinkWeight; m_nodeFlow[link.target] += link.flow; } //Normalize node flow double sumNodeRank = 0.0; for (unsigned int i = 0; i < numM2Nodes; ++i) sumNodeRank += m_nodeFlow[i]; for (unsigned int i = 0; i < numM2Nodes; ++i) m_nodeFlow[i] /= sumNodeRank; Log() << "\n -> Using directed links with raw flow."; Log() << "\n -> Total link weight: " << totalM2LinkWeight << "."; Log() << std::endl; return; } if (!config.directed) { if (config.undirdir || config.outdirdir) { //Take one last power iteration std::vector<double> nodeFlowSteadyState(m_nodeFlow); m_nodeFlow.assign(numM2Nodes, 0.0); for (LinkVec::iterator linkIt(m_flowLinks.begin()); linkIt != m_flowLinks.end(); ++linkIt) { Link& link = *linkIt; m_nodeFlow[link.target] += nodeFlowSteadyState[link.source] * link.flow / sumLinkOutWeight[link.source]; } //Normalize node flow double sumNodeRank = 0.0; for (unsigned int i = 0; i < numM2Nodes; ++i) sumNodeRank += m_nodeFlow[i]; for (unsigned int i = 0; i < numM2Nodes; ++i) m_nodeFlow[i] /= sumNodeRank; // Update link data to represent flow instead of weight for (LinkVec::iterator linkIt(m_flowLinks.begin()); linkIt != m_flowLinks.end(); ++linkIt) { Link& link = *linkIt; link.flow *= nodeFlowSteadyState[link.source] / sumLinkOutWeight[link.source] / sumNodeRank; } } else // undirected { for (unsigned int i = 0; i < numLinks; ++i) m_flowLinks[i].flow /= sumUndirLinkWeight; } if (config.outdirdir) Log() << "\n -> Counting only ingoing links."; else Log() << "\n -> Using undirected links" << (config.undirdir? ", switching to directed after steady state." : "."); Log() << std::endl; return; } Log() << "\n -> Using " << (config.recordedTeleportation ? "recorded" : "unrecorded") << " teleportation to memory " << (config.teleportToNodes ? "nodes" : "links") << " " << std::flush; if (config.originallyUndirected) { if (config.recordedTeleportation || !config.teleportToNodes) Log() << "(warning: should be unrecorded teleportation to nodes to correspond to undirected flow on physical network) " << std::flush; else Log() << "(corresponding to undirected flow on physical network) " << std::flush; } // Calculate the teleport rate distribution if (config.teleportToNodes) { const std::vector<double>& nodeWeights = network.m2NodeWeights(); for (unsigned int i = 0; i < numM2Nodes; ++i) { // m_nodeTeleportRates[i] = sumLinkOutWeight[i] / totalM2LinkWeight; // Use original m2 weights (without m1-completed weights for dangling m2 nodes) m_nodeTeleportRates[i] = nodeWeights[i] / network.totalM2NodeWeight(); } } else // Teleport to links { // Teleport proportionally to out-degree, or in-degree if recorded teleportation. for (LinkVec::iterator linkIt(m_flowLinks.begin()); linkIt != m_flowLinks.end(); ++linkIt) { unsigned int toNode = config.recordedTeleportation ? linkIt->target : linkIt->source; m_nodeTeleportRates[toNode] += linkIt->flow / totalM2LinkWeight; } } // Normalize link weights with respect to its source nodes total out-link weight; for (LinkVec::iterator linkIt(m_flowLinks.begin()); linkIt != m_flowLinks.end(); ++linkIt) { linkIt->flow /= sumLinkOutWeight[linkIt->source]; } // Collect dangling nodes std::vector<unsigned int> danglings; for (unsigned int i = 0; i < numM2Nodes; ++i) { if (nodeOutDegree[i] == 0) danglings.push_back(i); } // Calculate PageRank std::vector<double> nodeFlowTmp(numM2Nodes, 0.0); unsigned int numIterations = 0; double alpha = config.teleportationProbability; double beta = 1.0 - alpha; double sqdiff = 1.0; double danglingRank = 0.0; do { // Calculate dangling rank danglingRank = 0.0; for (std::vector<unsigned int>::iterator danglingIt(danglings.begin()); danglingIt != danglings.end(); ++danglingIt) { danglingRank += m_nodeFlow[*danglingIt]; } // Flow from teleportation for (unsigned int i = 0; i < numM2Nodes; ++i) { nodeFlowTmp[i] = (alpha + beta*danglingRank) * m_nodeTeleportRates[i]; } // Flow from links for (LinkVec::iterator linkIt(m_flowLinks.begin()); linkIt != m_flowLinks.end(); ++linkIt) { Link& link = *linkIt; nodeFlowTmp[link.target] += beta * link.flow * m_nodeFlow[link.source]; } // Update node flow from the power iteration above and check if converged double sum = 0.0; double sqdiff_old = sqdiff; sqdiff = 0.0; for (unsigned int i = 0; i < numM2Nodes; ++i) { sum += nodeFlowTmp[i]; sqdiff += std::abs(nodeFlowTmp[i] - m_nodeFlow[i]); m_nodeFlow[i] = nodeFlowTmp[i]; } // Normalize if needed if (std::abs(sum - 1.0) > 1.0e-10) { Log() << "(Normalizing ranks after " << numIterations << " power iterations with error " << (sum-1.0) << ") "; for (unsigned int i = 0; i < numM2Nodes; ++i) { m_nodeFlow[i] /= sum; } std::cerr << "DEBUG BREAK!!" << std::endl; break; } // Perturb the system if equilibrium if(sqdiff == sqdiff_old) { alpha += 1.0e-10; beta = 1.0 - alpha; } numIterations++; } while((numIterations < 200) && (sqdiff > 1.0e-15 || numIterations < 50)); double sumNodeRank = 1.0; if (!config.recordedTeleportation) { //Take one last power iteration excluding the teleportation (and normalize node flow to sum 1.0) sumNodeRank = 1.0 - danglingRank; m_nodeFlow.assign(numM2Nodes, 0.0); for (LinkVec::iterator linkIt(m_flowLinks.begin()); linkIt != m_flowLinks.end(); ++linkIt) { Link& link = *linkIt; m_nodeFlow[link.target] += link.flow * nodeFlowTmp[link.source] / sumNodeRank; } beta = 1.0; } // Update the links with their global flow from the PageRank values. (Note: beta is set to 1 if unrec) for (LinkVec::iterator linkIt(m_flowLinks.begin()); linkIt != m_flowLinks.end(); ++linkIt) { linkIt->flow *= beta * nodeFlowTmp[linkIt->source] / sumNodeRank; } Log() << "\n -> PageRank calculation done in " << numIterations << " iterations." << std::endl; }