void Graph::setSize(size_t n)
{
// If the graph is being made smaller we first need to remove all of the edges
// from the soon to be removed vertices.
for (size_t i = n; i < m_adjacencyList.size(); i++)
removeEdges(i);
m_adjacencyList.resize(n);
}
void Graph::removeVertex(size_t index)
{
assert(index < size());
// Remove the edges to the vertex.
removeEdges(index);
// Remove vertex's adjacency list.
m_adjacencyList.erase(m_adjacencyList.begin() + index);
}
/**
* Hides functions called from the current function node.
* This slot is connected to the "Hide Called Functions" popup menu
* action.
*/
void GraphWidget::slotHideCalled()
{
GraphNode* pNode;
// Make sure the menu item is a node
pNode = dynamic_cast<GraphNode*>(m_pMenuItem);
if (pNode == NULL)
return;
// Remove edges and redraw the graph
removeEdges(pNode, true);
draw();
}
//----------------------------------------------------------------
void GraphView::removeNode(const node n, const std::vector<edge> &ee) {
removeEdges(ee);
removeNode(n);
}
TopicExpander(string seedWordsFile, string graphFile, string freqFile,
double edgeThreshold, double minFreqThreshold, double maxFreqThreshold,
double affinityThreshold, int minConnections, int minEdges) {
cerr << "Creating Topic expander" << endl;
maxFreqThreshold_ = maxFreqThreshold;
minFreqThreshold_ = minFreqThreshold;
affinityThreshold_ = affinityThreshold;
wfUtil = new WordFrequencyUtil(freqFile);
StreamingFileUtil fUtil(graphFile);
cerr << "Creating graph" << endl;
unsigned lineCnt = 0;
while (!fUtil.done()) {
string line = fUtil.getNextLine();
vector<string> parts = StringUtil::split(line, "\t");
vector<string> words = StringUtil::split(parts[0], ":");
string word1 = StringUtil::clean(words[0]);
string word2 = StringUtil::clean(words[1]);
double score = atof(StringUtil::clean(parts[1]).c_str());
if (score > edgeThreshold) {
adjList[word1][word2] = score;
}
cerr << lineCnt++ << "\t\r";
}
while (removeEdges(minEdges));
fUtil.close();
fUtil.open(seedWordsFile);
unsigned int maxFreq = (unsigned int)(numWords * maxFreqThreshold_);
unsigned int minFreq = (unsigned int)(numWords * minFreqThreshold_);
while (!fUtil.done()) {
string line = fUtil.getNextLine();
vector<string> parts = StringUtil::split(line, ":");
string topicWord = StringUtil::clean(parts[0]);
set<WordScore> &clusterWords = clusters[topicWord];
clusterWords.insert(WordScore(topicWord, 1.0));
if (parts.size() > 1 && StringUtil::clean(parts[1]).size() > 0) {
vector<string> otherWords = StringUtil::split(parts[1], ",");
for (int i = 0; i < otherWords.size(); ++i) {
clusterWords.insert(WordScore(StringUtil::clean(otherWords[i]), 1.0));
}
}
}
fUtil.close();
fUtil.open(freqFile);
while (!fUtil.done()) {
string line = fUtil.getNextLine();
string currentWord = StringUtil::clean(StringUtil::split(line, "\t")[0]);
set<WordScore> topics = getBestKClustersForWord(currentWord, 10, minConnections, maxFreq);
for (set<WordScore>::iterator i = topics.begin();
i != topics.end(); ++i) {
string topic = i->word;
clusters[topic].insert(WordScore(currentWord, i->score));
}
}
for (map<string, set<WordScore> >::iterator i = clusters.begin();
i != clusters.end(); ++i) {
unsigned int clusterFrequency = getClusterFrequency(clusters[i->first]);
if (clusterFrequency < minFreq) {
weakClusters[i->first] = clusters[i->first];
}
}
for (map<string, set<WordScore> >::iterator i = weakClusters.begin();
i != weakClusters.end(); ++i) {
clusters.erase(i->first);
}
}
bool CreateIdiomaticAlternative::operator()(Automaton::RecognizerMatch& result,
AnalysisContent& analysis) const
{
#ifdef DEBUG_LP
MORPHOLOGINIT;
LDEBUG << "CreateIdiomaticAlternative, match is " << result;
LDEBUG << " expression is " << (result.isContiguous()?"":"non") <<
" contiguous and" << (result.isContextual()?" non":"") << " absolute";
#endif
if (result.empty()) return false;
const LinguisticAnalysisStructure::AnalysisGraph& graph = *(result.getGraph());
AnnotationData* annotationData = static_cast< AnnotationData* >(analysis.getData("AnnotationData"));
if (annotationData->dumpFunction("IdiomExpr") == 0)
{
annotationData->dumpFunction("IdiomExpr", new DumpIdiomaticExpressionAnnotation());
}
RecognizerData* recoData=static_cast<RecognizerData*>(analysis.getData("RecognizerData"));
std::set<LinguisticGraphVertex> addedVertices;
// initialize the vertices to clear
if (result.isContiguous())
{
// MORPHOLOGINIT;
// LDEBUG << "contiguous idiomatic expression found: "
// << result.concatString();
// only one part : terms in expression are adjacent -> easy part
// check if there is an overlap first
if (recoData->matchOnRemovedVertices(result))
{
// ignore current idiomatic expression, continue
MORPHOLOGINIT;
LWARN << "idiomatic expression ignored: " << Common::Misc::limastring2utf8stdstring(result.concatString())
<< ": overlapping with a previous one";
return false;
}
// create the new token
std::pair<Token*,MorphoSyntacticData*> newToken = createAlternativeToken(result);
if (newToken.second->empty())
{
// ignore current idiomatic expression, continue
MORPHOLOGINIT;
LERROR << "CreateIdiomaticAlternative::operator() Got empty morphosyntactic data. Abort";
delete newToken.first;
delete newToken.second;
return false;
}
// add the vertex
LinguisticGraphVertex idiomaticVertex =
addAlternativeVertex(newToken.first, newToken.second, const_cast<LinguisticGraph*>(graph.getGraph()));
AnnotationGraphVertex agv = annotationData->createAnnotationVertex();
annotationData->addMatching("AnalysisGraph", idiomaticVertex, "annot", agv);
annotationData->annotate(agv, Common::Misc::utf8stdstring2limastring("AnalysisGraph"), idiomaticVertex);
IdiomaticExpressionAnnotation annot(result);
GenericAnnotation ga(annot);
annotationData->annotate(agv, Common::Misc::utf8stdstring2limastring("IdiomExpr"), ga);
addedVertices.insert(idiomaticVertex);
//create the alternative with this only vertex
createBeginAlternative(result.front().getVertex(),
idiomaticVertex,const_cast<LinguisticGraph&>(*graph.getGraph()));
attachEndOfAlternative(idiomaticVertex,
result.back().getVertex(),const_cast<LinguisticGraph&>(*graph.getGraph()));
// if expression is not contextual, only keep alternative
if (! result.isContextual())
{
recoData->storeVerticesToRemove(result,const_cast<LinguisticGraph*>(graph.getGraph()));
removeEdges(const_cast<LinguisticGraph&>(*graph.getGraph()),
result, analysis);
//recoData->setNextVertex(idiomaticVertex);
// if match was on single token, use next vertices (to avoid loops)
if (result.size() > 1)
{
recoData->setNextVertex(idiomaticVertex);
}
else
{
LinguisticGraphOutEdgeIt outItr,outItrEnd;
boost::tie(outItr,outItrEnd) = out_edges(idiomaticVertex,*(graph.getGraph()));
for (;outItr!=outItrEnd;outItr++)
{
recoData->setNextVertex(target(*outItr, *(graph.getGraph())));
}
}
}
}
else
{
// several parts : tough case
// MORPHOLOGINIT;
// LDEBUG << "non contiguous idiomatic expression found: "
// << result.concatString();
// check if there is an overlap first
if (recoData->matchOnRemovedVertices(result))
{
// ignore current idiomatic expression, continue
MORPHOLOGINIT;
LWARN << "idiomatic expression ignored: " << Common::Misc::limastring2utf8stdstring(result.concatString())
<< ": overlapping with a previous one";
return false;
}
// create the new token
pair<Token*,MorphoSyntacticData*> newToken = createAlternativeToken(result);
if (newToken.second->empty())
{
// ignore current idiomatic expression, continue
MORPHOLOGINIT;
LERROR << "CreateIdiomaticAlternative::operator() Got empty morphosyntactic data. Abort";
delete newToken.first;
delete newToken.second;
return false;
}
// add the vertex
LinguisticGraphVertex idiomaticVertex =
addAlternativeVertex(newToken.first,newToken.second,const_cast<LinguisticGraph*>(graph.getGraph()));
addedVertices.insert(idiomaticVertex);
AnnotationGraphVertex agv = annotationData->createAnnotationVertex();
annotationData->addMatching("AnalysisGraph", idiomaticVertex, "annot", agv);
annotationData->annotate(agv, Common::Misc::utf8stdstring2limastring("AnalysisGraph"), idiomaticVertex);
IdiomaticExpressionAnnotation annot(result);
GenericAnnotation ga(annot);
annotationData->annotate(agv, Common::Misc::utf8stdstring2limastring("IdiomExpr"), ga);
//create the alternative with this vertex and duplicate of other vertices
deque<LinguisticGraphVertex> idiomAlternative;
LinguisticGraphVertex headVertex=result.getHead();
#ifdef DEBUG_LP
LDEBUG << "headVertex = " << headVertex;
if (headVertex!=0)
{
LDEBUG << "=> " << Common::Misc::limastring2utf8stdstring(get(vertex_token,*graph.getGraph(),headVertex)->stringForm());
}
#endif
bool foundHead=false;
bool keeping = false;
std::pair< LinguisticGraphVertex, LinguisticGraphVertex > idiomPartBounds;
std::set< std::pair< LinguisticGraphVertex, LinguisticGraphVertex > > edgesToRemove;
RecognizerMatch::const_iterator matchItr=result.begin();
for (; matchItr!=result.end(); matchItr++)
{
if (!matchItr->isKept())
{
if (keeping)
{
RecognizerMatch::const_iterator prevItr = matchItr - 1;
idiomPartBounds.second = prevItr->getVertex();
keeping = false;
#ifdef DEBUG_LP
LDEBUG << "adding " << idiomPartBounds.first << " -> " << idiomPartBounds.second << " in edgesToRemove";
#endif
edgesToRemove.insert(idiomPartBounds);
}
// duplicate this vertex
#ifdef DEBUG_LP
LDEBUG << "duplication of vertex " << matchItr->getVertex();;
#endif
Token* token=get(vertex_token,*graph.getGraph(),matchItr->getVertex());
MorphoSyntacticData* data =
new MorphoSyntacticData(*get(vertex_data,*graph.getGraph(),matchItr->getVertex()));
LinguisticGraphVertex dupVx = add_vertex(const_cast<LinguisticGraph&>(*graph.getGraph()));
put(vertex_token,const_cast<LinguisticGraph&>(*graph.getGraph()),dupVx,token);
put(vertex_data,const_cast<LinguisticGraph&>(*graph.getGraph()),dupVx,data);
idiomAlternative.push_back(dupVx);
AnnotationGraphVertex agv = annotationData->createAnnotationVertex();
annotationData->addMatching("AnalysisGraph", dupVx, "annot", agv);
annotationData->annotate(agv, Common::Misc::utf8stdstring2limastring("AnalysisGraph"), dupVx);
std::set< LinguisticGraphVertex > annotMatches =
annotationData->matches("AnalysisGraph",matchItr->getVertex(),"annot");
for (std::set< LinguisticGraphVertex >::const_iterator annotIt(annotMatches.begin());
annotIt != annotMatches.end(); annotIt++)
{
std::set< std::string > excepted;
excepted.insert("AnalysisGraph");
annotationData->cloneAnnotations(*annotIt, agv, excepted);
}
addedVertices.insert(dupVx);
// verticesToRemove.insert(matchItr->getVertex());
}
else
{
if (!keeping)
{
idiomPartBounds.first = matchItr->getVertex();
keeping = true;
}
#ifdef DEBUG_LP
LDEBUG << "kept vertex " << matchItr->getVertex();
#endif
if (matchItr->getVertex()==headVertex)
{
foundHead=true;
#ifdef DEBUG_LP
LDEBUG << "add head vertex " << idiomaticVertex;
#endif
idiomAlternative.push_back(idiomaticVertex);
}
}
}
if (!foundHead)
{
MORPHOLOGINIT;
LWARN << "head token has not been found in non contiguous expression. "
<< "Idiomatic token is placed first";
idiomAlternative.push_front(idiomaticVertex);
}
if (keeping)
{
RecognizerMatch::const_iterator prevItr = matchItr - 1;
idiomPartBounds.second = prevItr->getVertex();
keeping = false;
#ifdef DEBUG_LP
LDEBUG << "adding " << idiomPartBounds.first << " -> " << idiomPartBounds.second << " in edgesToRemove";
#endif
edgesToRemove.insert(idiomPartBounds);
}
// link alternatives
#ifdef DEBUG_LP
LDEBUG << "idiomAlternative has " << idiomAlternative.size() << " vertex";
#endif
createBeginAlternative(result.front().getVertex(),
idiomAlternative.front(),const_cast<LinguisticGraph&>(*graph.getGraph()));
{
deque<LinguisticGraphVertex>::const_iterator idItr=idiomAlternative.begin();
LinguisticGraphVertex lastIdiomVx=*idItr;
idItr++;
while (idItr!=idiomAlternative.end())
{
LinguisticGraphEdge newEdge;
bool ok;
boost::tie(newEdge, ok) = add_edge(lastIdiomVx,*idItr,const_cast<LinguisticGraph&>(*graph.getGraph()));
#ifdef DEBUG_LP
LDEBUG << "added new edge in alternatives linking: " << newEdge.m_source << " -> " << newEdge.m_target;
#endif
lastIdiomVx=*idItr;
idItr++;
}
}
attachEndOfAlternative(idiomAlternative.back(),
result.back().getVertex(),const_cast<LinguisticGraph&>(*graph.getGraph()));
// if expression is not contextual, only keep alternative
if (! result.isContextual())
{
#ifdef DEBUG_LP
LDEBUG << "expression is not contextual, only keep alternative";
#endif
std::set< std::pair< LinguisticGraphVertex, LinguisticGraphVertex > >::const_iterator edgesToRemoveIt, edgesToRemoveIt_end;
edgesToRemoveIt = edgesToRemove.begin(); edgesToRemoveIt_end = edgesToRemove.end();
for (; edgesToRemoveIt != edgesToRemoveIt_end; edgesToRemoveIt++)
{
#ifdef DEBUG_LP
LDEBUG << "Removing edge " << (*edgesToRemoveIt).first << " -> " << (*edgesToRemoveIt).second;
#endif
removeEdges(const_cast<LinguisticGraph&>(*graph.getGraph()),
result, analysis);
}
// recoData->storeVerticesToRemove(result,*graph);
// no need to check size: if several parts, more than one vertex
recoData->setNextVertex(idiomaticVertex);
}
}
RecognizerMatch::const_iterator matchItr=result.begin();
for (; matchItr!=result.end(); matchItr++)
{
recoData->clearUnreachableVertices( analysis, (*matchItr).getVertex());
}
// recoData->clearUnreachableVertices( analysis, result.front().getVertex(), result.back().getVertex(), storedEdges);
return true;
}
void removeNode(node * n) {
removeEdges(n->edges);
free(n);
}
