Пример #1
0
PatternTree::PatternTree(earl::EventTrace* trace, Analyzer& analyzer)
{
    vector<Pattern*> pv;

    get_patternv(pv, trace, analyzer.get_opt() );

    for ( size_t i = 0; i < pv.size(); i++ )
        if ( pv[i]->isenabled() )
            this->add_node(pv[i]->get_id(), pv[i], pv[i]->get_parent_id());
        else
            delete pv[i];

    // register callbacks
    this->regcb(analyzer);
}
Пример #2
0
void TestVariation::onPlayerConnected()
{
    Analyzer *analyzer = sender();

    team.importFromTxt(getFileContent("team1.txt").trimmed());

    TeamHolder holder;
    holder.team() = team;

    if (count == 0) {
        holder.name() = "Weavile";
    } else if (count == 1) {
        holder.name() = "daleck";
    }

    ++count;

    /* Use different IPs to make sure we can find a battle against each other */
    analyzer->notify(NetworkCli::SetIP, holder.name());
    analyzer->login(holder,true);

    /* Timeout on test */
    setTimeout(10);
}
Пример #3
0
/**
 * Test Analyzer::segmentString().
 * \param analyzer analyzer
 * \param is input stream
 */
void testSentenceLength(Analyzer& analyzer, istream& is)
{
    string line;
    while(getline(is, line))
    {
        const char* p = line.c_str();
        string sentStr;
        while(size_t len = analyzer.sentenceLength(p))
        {
            sentStr.assign(p, len); // get each sentence
            cout << sentStr << endl; // print each sentence
            p += len; // move to the begining of next sentence
        }
        cout << endl;
    }
}
Пример #4
0
/**
 * Test Analyzer::segmentString().
 * \param analyzer analyzer
 */
void testSegmentString(Analyzer& analyzer)
{
    vector<LanguageRegion> regionVec;

    string line;
    while(getline(cin, line))
    {
        if(! analyzer.segmentString(line.c_str(), regionVec))
        {
            cerr << "error to segment string " << line << endl;
            exit(1);
        }

        printStringRegion(line.c_str(), regionVec);
        cout << endl;
    }
}
Пример #5
0
/**
 * Test Analyzer::languageListFromFile().
 * \param analyzer analyzer
 * \param fileName file name
 */
void testLanguageListFromFile(Analyzer& analyzer, const char* fileName)
{
    vector<LanguageID> idVec;
    if(! analyzer.languageListFromFile(fileName, idVec))
    {
        cerr << "error to get language ID list from file " << fileName << endl;
        exit(1);
    }

    cout << "raw file name: " << fileName << endl;
    cout << "multiple language types in descending order of sentence count:" << endl;
    for(unsigned int i=0; i<idVec.size(); ++i)
    {
        cout << i << ": " << Knowledge::getLanguageNameFromID(idVec[i]) << endl;
    }
    cout << endl;
}
Пример #6
0
void check_analyzer_expected(Analyzer& ana, corpus::document doc,
                             uint64_t num_unique, uint64_t length)
{
    ana.tokenize(doc);
    ASSERT_EQUAL(doc.counts().size(), num_unique);
    ASSERT_EQUAL(doc.length(), length);
    ASSERT_EQUAL(doc.id(), 47ul);
    if (doc.contains_content())
    {
        ASSERT_EQUAL(doc.path(), "/home/person/filename.txt");
        ASSERT_EQUAL(doc.name(), "filename.txt");
    }
    else
    {
        ASSERT_EQUAL(doc.path(), "../data/sample-document.txt");
        ASSERT_EQUAL(doc.name(), "sample-document.txt");
    }
}
Пример #7
0
/**
 * Test Analyzer::languageListFromString().
 * \param analyzer analyzer
 */
void testLanguageListFromString(Analyzer& analyzer)
{
    vector<LanguageID> idVec;
    string line;
    while(getline(cin, line))
    {
        if(! analyzer.languageListFromString(line.c_str(), idVec))
        {
            cerr << "error to get language ID list from string " << line << endl;
            exit(1);
        }

        cout << "multiple language types in descending order of sentence count:" << endl;
        for(unsigned int i=0; i<idVec.size(); ++i)
        {
            cout << i << ": " << Knowledge::getLanguageNameFromID(idVec[i]) << endl;
        }
        cout << endl;
    }
}
Пример #8
0
void Analyze::results_json(Json::Value &root)
{
    Json::Value tests;
    uint16_t total_evilness= 0;
    for (std::vector<Analyzer*>::iterator it = _analyzers.begin();
         it != _analyzers.end();
         ++it) {
        Analyzer *analyzer = *it;
        const std::string name = analyzer->name();
        if (tests[name].isNull()) {
            Json::Value test_info(Json::objectValue);
            test_info["description"] = analyzer->description();
            test_info["name"] = name;
            test_info["results"] = Json::Value(Json::arrayValue);
            tests[name] = test_info;
        }

        analyzer->results_json_results(tests[name]["results"]);
        tests[name]["status"] = analyzer->status_as_string();
        tests[name]["severity-score"] = tests[name]["severity-score"].asLargestUInt() + analyzer->severity_score();
        if (!pure_output()) {
            tests[name]["evilness"] = tests[name]["severity-score"];
            tests[name]["evilness-is-deprecated"] = "Use severity-score";
        }
        total_evilness += analyzer->severity_score();
    }
    
    root["severity-score"] = total_evilness;
    if (!pure_output()) {
        root["evilness"] = root["severity-score"];
        root["evilness-is-deprecated"] = "Use severity-score";
    }
    root["tests"] = tests;
    results_json_add_version(root);

    results_json_add_statistics(root);
}
Пример #9
0
int main( int argc, char * argv[] ) {
  string ltl_file;
  try {
	Timer timer;
	timer.start();

  // Command line option handling.
  po::options_description desc
    ("CodeThorn V1.2\n"
     "Written by Markus Schordan and Adrian Prantl 2012\n"
     "Supported options");

  desc.add_options()
    ("help,h", "produce this help message")
    ("rose-help", "show help for compiler frontend options")
    ("version,v", "display the version")
    ("internal-checks", "run internal consistency checks (without input program)")
    ("verify", po::value< string >(), "verify all LTL formulae in the file [arg]")
    ("ltl-verifier",po::value< int >(),"specify which ltl-verifier to use [=1|2]")
    ("debug-mode",po::value< int >(),"set debug mode [arg]")
    ("csv-ltl", po::value< string >(), "output LTL verification results into a CSV file [arg]")
    ("csv-assert", po::value< string >(), "output assert reachability results into a CSV file [arg]")
    ("csv-assert-live", po::value< string >(), "output assert reachability results during analysis into a CSV file [arg]")
    ("csv-stats",po::value< string >(),"output statistics into a CSV file [arg]")
    ("tg1-estate-address", po::value< string >(), "transition graph 1: visualize address [=yes|no]")
    ("tg1-estate-id", po::value< string >(), "transition graph 1: visualize estate-id [=yes|no]")
    ("tg1-estate-properties", po::value< string >(), 
     "transition graph 1: visualize all estate-properties [=yes|no]")
    ("tg2-estate-address", po::value< string >(), "transition graph 2: visualize address [=yes|no]")
    ("tg2-estate-id", po::value< string >(), "transition graph 2: visualize estate-id [=yes|no]")
    ("tg2-estate-properties", po::value< string >(),
     "transition graph 2: visualize all estate-properties [=yes|no]")
    ("colors",po::value< string >(),"use colors in output [=yes|no]")
    ("report-stdout",po::value< string >(),"report stdout estates during analysis [=yes|no]")
    ("report-stderr",po::value< string >(),"report stderr estates during analysis [=yes|no]")
    ("report-failed-assert",po::value< string >(),
     "report failed assert estates during analysis [=yes|no]")
    ("precision-intbool",po::value< string >(),
     "use precise top with intbool-(and/or) operators (used in int-analyzer) [=yes|no]")
    ("precision-exact-constraints",po::value< string >(),
     "(experimental) use precise constraint extraction [=yes|no]")
    ("tg-ltl-reduced",po::value< string >(),"(experimental) compute LTL-reduced transition graph based on a subset of computed estates [=yes|no]")
    ("semantic-fold",po::value< string >(),"compute semantically folded state transition graph [=yes|no]")
    ("post-semantic-fold",po::value< string >(),"compute semantically folded state transition graph only after the complete transition graph has been computed. [=yes|no]")
    ("report-semantic-fold",po::value< string >(),"report each folding operation with the respective number of estates. [=yes|no]")
    ("semantic-fold-threshold",po::value< int >(),"Set threshold with <arg> for semantic fold operation (experimental)")
    ("post-collapse-stg",po::value< string >(),"compute collapsed state transition graph after the complete transition graph has been computed. [=yes|no]")
    ("viz",po::value< string >(),"generate visualizations (dot) outputs [=yes|no]")
    ("update-input-var",po::value< string >(),"For testing purposes only. Default is Yes. [=yes|no]")
    ("run-rose-tests",po::value< string >(),"Run ROSE AST tests. [=yes|no]")
    ("reduce-cfg",po::value< string >(),"Reduce CFG nodes which are not relevant for the analysis. [=yes|no]")
    ("threads",po::value< int >(),"Run analyzer in parallel using <arg> threads (experimental)")
    ("display-diff",po::value< int >(),"Print statistics every <arg> computed estates.")
	("ltl-output-dot",po::value< string >(),"LTL visualization: generate dot output.")
	("ltl-show-derivation",po::value< string >(),"LTL visualization: show derivation in dot output.")
	("ltl-show-node-detail",po::value< string >(),"LTL visualization: show node detail in dot output.")
	("ltl-collapsed-graph",po::value< string >(),"LTL visualization: show collapsed graph in dot output.")
	("input-var-values",po::value< string >(),"specify a set of input values (e.g. \"{1,2,3}\")")
    ("input-var-values-as-constraints",po::value<string >(),"represent input var values as constraints (otherwise as constants in PState)")
    ("arith-top",po::value< string >(),"Arithmetic operations +,-,*,/,% always evaluate to top [=yes|no]")
    ("abstract-interpreter",po::value< string >(),"Run analyzer in abstract interpreter mode. Use [=yes|no]")
    ("rers-binary",po::value< string >(),"Call rers binary functions in analysis. Use [=yes|no]")
	("print-all-options",po::value< string >(),"print all yes/no command line options.")
    ;

  po::store(po::command_line_parser(argc, argv).
	    options(desc).allow_unregistered().run(), args);
  po::notify(args);

  if (args.count("help")) {
    cout << desc << "\n";
    return 0;
  }

  if (args.count("rose-help")) {
    argv[1] = strdup("--help");
  }

  if (args.count("version")) {
    cout << "CodeThorn version 1.2\n";
    cout << "Written by Markus Schordan and Adrian Prantl 2012\n";
    return 0;
  }

  boolOptions.init(argc,argv);
  boolOptions.registerOption("tg1-estate-address",false);
  boolOptions.registerOption("tg1-estate-id",false);
  boolOptions.registerOption("tg1-estate-properties",true);
  boolOptions.registerOption("tg2-estate-address",false);
  boolOptions.registerOption("tg2-estate-id",true);
  boolOptions.registerOption("tg2-estate-properties",false);
  boolOptions.registerOption("colors",true);
  boolOptions.registerOption("report-stdout",false);
  boolOptions.registerOption("report-stderr",false);
  boolOptions.registerOption("report-failed-assert",false);
  boolOptions.registerOption("precision-intbool",true);
  boolOptions.registerOption("precision-exact-constraints",false);
  boolOptions.registerOption("tg-ltl-reduced",false);
  boolOptions.registerOption("semantic-fold",false);
  boolOptions.registerOption("post-semantic-fold",false);
  boolOptions.registerOption("report-semantic-fold",false);
  boolOptions.registerOption("post-collapse-stg",true);

  boolOptions.registerOption("viz",false);
  boolOptions.registerOption("update-input-var",true);
  boolOptions.registerOption("run-rose-tests",false);
  boolOptions.registerOption("reduce-cfg",true);
  boolOptions.registerOption("print-all-options",false);

  boolOptions.registerOption("ltl-output-dot",false);
  boolOptions.registerOption("ltl-show-derivation",true);
  boolOptions.registerOption("ltl-show-node-detail",true);
  boolOptions.registerOption("ltl-collapsed-graph",false);
  boolOptions.registerOption("input-var-values-as-constraints",false);

  boolOptions.registerOption("arith-top",false);
  boolOptions.registerOption("abstract-interpreter",false);
  boolOptions.registerOption("rers-binary",false);
  boolOptions.registerOption("relop-constraints",false); // not accessible on command line yet

  boolOptions.processOptions();

  if(boolOptions["print-all-options"]) {
	cout<<boolOptions.toString(); // prints all bool options
  }
  
  if (args.count("internal-checks")) {
	if(CodeThorn::internalChecks(argc,argv)==false)
	  return 1;
	else
	  return 0;
  }

  Analyzer analyzer;
  
  // clean up verify and csv-ltl option in argv
  if (args.count("verify")) {
    ltl_file = args["verify"].as<string>();
	for (int i=1; i<argc; ++i) {
	  if ((string(argv[i]) == "--verify") || 
	      (string(argv[i]) == "--csv-ltl")) {
		// do not confuse ROSE frontend
		argv[i] = strdup("");
		assert(i+1<argc);
		argv[i+1] = strdup("");
	  }
	}
  }
  if(args.count("csv-assert-live")) {
	analyzer._csv_assert_live_file=args["csv-assert-live"].as<string>();
  }

  if(args.count("input-var-values")) {
	string setstring=args["input-var-values"].as<string>();
	cout << "STATUS: input-var-values="<<setstring<<endl;
	stringstream ss(setstring);
	if(ss.peek()=='{')
	  ss.ignore();
	else
	  throw "Error: option input-var-values: wrong input format (at start).";
	int i;
	while(ss>>i) {
	  //cout << "DEBUG: input-var-string:i:"<<i<<" peek:"<<ss.peek()<<endl;	
	  analyzer.insertInputVarValue(i);
	  if(ss.peek()==','||ss.peek()==' ')
		ss.ignore();
	}
#if 0
	if(ss.peek()=='}')
	  ss.ignore();
	else
	  throw "Error: option input-var-values: wrong input format (at end).";
#endif
  }

  int numberOfThreadsToUse=1;
  if(args.count("threads")) {
	numberOfThreadsToUse=args["threads"].as<int>();
  }
  analyzer.setNumberOfThreadsToUse(numberOfThreadsToUse);

  if(args.count("semantic-fold-threshold")) {
	int semanticFoldThreshold=args["semantic-fold-threshold"].as<int>();
	analyzer.setSemanticFoldThreshold(semanticFoldThreshold);
  }
  if(args.count("display-diff")) {
	int displayDiff=args["display-diff"].as<int>();
	analyzer.setDisplayDiff(displayDiff);
  }
  if(args.count("ltl-verifier")) {
	int ltlVerifier=args["ltl-verifier"].as<int>();
	analyzer.setLTLVerifier(ltlVerifier);
  }
  if(args.count("debug-mode")) {
	option_debug_mode=args["debug-mode"].as<int>();
  }

  // clean up string-options in argv
  for (int i=1; i<argc; ++i) {
	if (string(argv[i]) == "--csv-assert" 
		|| string(argv[i])=="--csv-stats" 
		|| string(argv[i])=="--csv-assert-live"
		|| string(argv[i])=="--threads" 
		|| string(argv[i])=="--display-diff"
		|| string(argv[i])=="--input-var-values"
		|| string(argv[i])=="--ltl-verifier"
		) {
	  // do not confuse ROSE frontend
	  argv[i] = strdup("");
	  assert(i+1<argc);
		argv[i+1] = strdup("");
	}
  }

  // Build the AST used by ROSE
  cout << "INIT: Parsing and creating AST."<<endl;
  SgProject* sageProject = frontend(argc,argv);
  double frontEndRunTime=timer.getElapsedTimeInMilliSec();
  
  if(boolOptions["run-rose-tests"]) {
	cout << "INIT: Running ROSE AST tests."<<endl;
	// Run internal consistency tests on AST
	AstTests::runAllTests(sageProject);
  }

  SgNode* root=sageProject;
  checkProgram(root);
  timer.start();

  cout << "INIT: Running variable<->symbol mapping check."<<endl;
  //VariableIdMapping varIdMap;
  analyzer.getVariableIdMapping()->computeVariableSymbolMapping(sageProject);
  cout << "STATUS: Variable<->Symbol mapping created."<<endl;
  if(!analyzer.getVariableIdMapping()->isUniqueVariableSymbolMapping()) {
	cerr << "WARNING: Variable<->Symbol mapping not bijective."<<endl;
	//varIdMap.reportUniqueVariableSymbolMappingViolations();
  }
#if 0
  analyzer.getVariableIdMapping()->toStream(cout);
#endif

  cout << "INIT: creating solver."<<endl;
  analyzer.initializeSolver1("main",root);
  analyzer.initLabeledAssertNodes(sageProject);
  double initRunTime=timer.getElapsedTimeInMilliSec();

  timer.start();
  cout << "=============================================================="<<endl;
  if(boolOptions["semantic-fold"]) {
	analyzer.runSolver2();
  } else {
	analyzer.runSolver1();
  }

  if(boolOptions["post-semantic-fold"]) {
	cout << "Performing post semantic folding (this may take some time):"<<endl;
	analyzer.semanticFoldingOfTransitionGraph();
  }
  double analysisRunTime=timer.getElapsedTimeInMilliSec();

  // since CT1.2 the ADT TransitionGraph ensures that no duplicates can exist
#if 0
  long removed=analyzer.getTransitionGraph()->removeDuplicates();
  cout << "Transitions reduced: "<<removed<<endl;
#endif

  cout << "=============================================================="<<endl;
  // TODO: reachability in presence of semantic folding
  if(!boolOptions["semantic-fold"] && !boolOptions["post-semantic-fold"]) {
	printAsserts(analyzer,sageProject);
  }
  if (args.count("csv-assert")) {
	string filename=args["csv-assert"].as<string>().c_str();
	generateAssertsCsvFile(analyzer,sageProject,filename);
	cout << "=============================================================="<<endl;
  }
  if(boolOptions["tg-ltl-reduced"]) {
	cout << "(Experimental) Reducing transition graph ..."<<endl;
	set<const EState*> xestates=analyzer.nonLTLRelevantEStates();
	cout << "Size of transition graph before reduction: "<<analyzer.getTransitionGraph()->size()<<endl;
	cout << "Number of EStates to be reduced: "<<xestates.size()<<endl;
	analyzer.getTransitionGraph()->reduceEStates(xestates);
	cout << "Size of transition graph after reduction : "<<analyzer.getTransitionGraph()->size()<<endl;
	cout << "=============================================================="<<endl;
  }
  timer.start();
  if (ltl_file.size()) {
	generateLTLOutput(analyzer,ltl_file);
	cout << "=============================================================="<<endl;
  }
  double ltlRunTime=timer.getElapsedTimeInMilliSec();
  // TODO: reachability in presence of semantic folding
  if(boolOptions["semantic-fold"] || boolOptions["post-semantic-fold"]) {
	  cout << "NOTE: no reachability results with semantic folding (not implemented yet)."<<endl;
	} else {
	  printAssertStatistics(analyzer,sageProject);
	}
  cout << "=============================================================="<<endl;

  double totalRunTime=frontEndRunTime+initRunTime+ analysisRunTime+ltlRunTime;

  long pstateSetSize=analyzer.getPStateSet()->size();
  long pstateSetBytes=analyzer.getPStateSet()->memorySize();
  long pstateSetMaxCollisions=analyzer.getPStateSet()->maxCollisions();
  long pstateSetLoadFactor=analyzer.getPStateSet()->loadFactor();
  long eStateSetSize=analyzer.getEStateSet()->size();
  long eStateSetBytes=analyzer.getEStateSet()->memorySize();
  long eStateSetMaxCollisions=analyzer.getEStateSet()->maxCollisions();
  double eStateSetLoadFactor=analyzer.getEStateSet()->loadFactor();
  long transitionGraphSize=analyzer.getTransitionGraph()->size();
  long transitionGraphBytes=transitionGraphSize*sizeof(Transition);
  long numOfconstraintSets=analyzer.getConstraintSetMaintainer()->numberOf();
  long constraintSetsBytes=analyzer.getConstraintSetMaintainer()->memorySize();
  long constraintSetsMaxCollisions=analyzer.getConstraintSetMaintainer()->maxCollisions();
  double constraintSetsLoadFactor=analyzer.getConstraintSetMaintainer()->loadFactor();

  cout << "Number of stdin-estates        : "<<color("cyan")<<(analyzer.getEStateSet()->numberOfIoTypeEStates(InputOutput::STDIN_VAR))<<color("white")<<endl;
  cout << "Number of stdout-estates       : "<<color("cyan")<<(analyzer.getEStateSet()->numberOfIoTypeEStates(InputOutput::STDOUT_VAR))<<color("white")<<endl;
  cout << "Number of stderr-estates       : "<<color("cyan")<<(analyzer.getEStateSet()->numberOfIoTypeEStates(InputOutput::STDERR_VAR))<<color("white")<<endl;
  cout << "Number of failed-assert-estates: "<<color("cyan")<<(analyzer.getEStateSet()->numberOfIoTypeEStates(InputOutput::FAILED_ASSERT))<<color("white")<<endl;
  cout << "=============================================================="<<endl;
  cout << "Number of pstates              : "<<color("magenta")<<pstateSetSize<<color("white")<<" (memory: "<<color("magenta")<<pstateSetBytes<<color("white")<<" bytes)"<<" ("<<""<<pstateSetLoadFactor<<  "/"<<pstateSetMaxCollisions<<")"<<endl;
  cout << "Number of estates              : "<<color("cyan")<<eStateSetSize<<color("white")<<" (memory: "<<color("cyan")<<eStateSetBytes<<color("white")<<" bytes)"<<" ("<<""<<eStateSetLoadFactor<<  "/"<<eStateSetMaxCollisions<<")"<<endl;
  cout << "Number of transitions          : "<<color("blue")<<transitionGraphSize<<color("white")<<" (memory: "<<color("blue")<<transitionGraphBytes<<color("white")<<" bytes)"<<endl;
  cout << "Number of constraint sets      : "<<color("yellow")<<numOfconstraintSets<<color("white")<<" (memory: "<<color("yellow")<<constraintSetsBytes<<color("white")<<" bytes)"<<" ("<<""<<constraintSetsLoadFactor<<  "/"<<constraintSetsMaxCollisions<<")"<<endl;
  cout << "=============================================================="<<endl;
  long totalMemory=pstateSetBytes+eStateSetBytes+transitionGraphBytes+constraintSetsBytes;
  cout << "Memory total         : "<<color("green")<<totalMemory<<" bytes"<<color("normal")<<endl;
  cout << "Time total           : "<<color("green")<<readableruntime(totalRunTime)<<color("normal")<<endl;
  cout << "=============================================================="<<endl;

  if(args.count("csv-stats")) {
	string filename=args["csv-stats"].as<string>().c_str();
	stringstream text;
	text<<"Sizes,"<<pstateSetSize<<", "
		<<eStateSetSize<<", "
		<<transitionGraphSize<<", "
		<<numOfconstraintSets<<endl;
	text<<"Memory,"<<pstateSetBytes<<", "
		<<eStateSetBytes<<", "
		<<transitionGraphBytes<<", "
		<<constraintSetsBytes<<", "
		<<totalMemory<<endl;
	text<<"Runtime(readable),"
		<<readableruntime(frontEndRunTime)<<", "
		<<readableruntime(initRunTime)<<", "
		<<readableruntime(analysisRunTime)<<", "
		<<readableruntime(ltlRunTime)<<", "
		<<readableruntime(totalRunTime)<<endl;
	text<<"Runtime(ms),"
		<<frontEndRunTime<<", "
		<<initRunTime<<", "
		<<analysisRunTime<<", "
		<<ltlRunTime<<", "
		<<totalRunTime<<endl;
	text<<"hashset-collisions,"
		<<pstateSetMaxCollisions<<", "
		<<eStateSetMaxCollisions<<", "
		<<constraintSetsMaxCollisions<<endl;
	text<<"hashset-loadfactors,"
		<<pstateSetLoadFactor<<", "
		<<eStateSetLoadFactor<<", "
		<<constraintSetsLoadFactor<<endl;
	text<<"threads,"<<numberOfThreadsToUse<<endl;
	write_file(filename,text.str());
    cout << "generated "<<filename<<endl;
  }
  

  if(boolOptions["viz"]) {
    Visualizer visualizer(analyzer.getLabeler(),analyzer.getVariableIdMapping(),analyzer.getFlow(),analyzer.getPStateSet(),analyzer.getEStateSet(),analyzer.getTransitionGraph());
	cout << "generating graphviz files:"<<endl;
    string dotFile="digraph G {\n";
    dotFile+=visualizer.transitionGraphToDot();
    dotFile+="}\n";
    write_file("transitiongraph1.dot", dotFile);
    cout << "generated transitiongraph1.dot."<<endl;
    string dotFile3=visualizer.foldedTransitionGraphToDot();
    write_file("transitiongraph2.dot", dotFile3);
    cout << "generated transitiongraph2.dot."<<endl;

	string datFile1=(analyzer.getTransitionGraph())->toString();
	write_file("transitiongraph1.dat", datFile1);
    cout << "generated transitiongraph1.dat."<<endl;

    assert(analyzer.startFunRoot);
    //analyzer.generateAstNodeInfo(analyzer.startFunRoot);
    //dotFile=astTermWithNullValuesToDot(analyzer.startFunRoot);
    analyzer.generateAstNodeInfo(sageProject);
	cout << "generated node info."<<endl;
    dotFile=functionAstTermsWithNullValuesToDot(sageProject);
    write_file("ast.dot", dotFile);
    cout << "generated ast.dot."<<endl;
    
    write_file("cfg.dot", analyzer.flow.toDot(analyzer.cfanalyzer->getLabeler()));
    cout << "generated cfg.dot."<<endl;
  }

#if 0
  {
	cout << "EStateSet:\n"<<analyzer.getEStateSet()->toString()<<endl;
  }
#endif

#if 0
  {
    cout << "MAP:"<<endl;
    cout << analyzer.getLabeler()->toString();
  }
#endif

#if 0
  // check output var to be constant in transition graph
  TransitionGraph* tg=analyzer.getTransitionGraph();
  for(TransitionGraph::iterator i=tg->begin();i!=tg->end();++i) {
	const EState* es1=(*i).source;
	InputOutput myio=es1->io;
	assert(myio.op==InputOutput::STDOUT_VAR 
		   && 
		   es1->pstate->varIsConst(es1->io.var)
		   );
  }
#endif


  // reset terminal
  cout<<color("normal")<<"done."<<endl;
  
  } catch(char* str) {
Пример #10
0
void generateLTLOutput(Analyzer& analyzer, string ltl_file) {
  extern CodeThorn::LTL::Formula* ltl_val;
  //
  // Verification
  //
  int n = 0;
  int n_yes = 0;
  int n_no = 0;
  int n_undecided = 0;
  int n_failed = 0;

  assert(analyzer.getEStateSet());
  assert(analyzer.getTransitionGraph());
  if (ltl_file.size()) {
    CodeThorn::FixpointLTL::Checker* checker1 = 0; 
    CodeThorn::UnifiedLTL::UChecker* checker2 = 0; 
    switch(analyzer.getLTLVerifier()) {
    case 1: 
      checker1 = new CodeThorn::FixpointLTL::Checker(*analyzer.getEStateSet(), 
						     *analyzer.getTransitionGraph());
      break;
    case 2:
      checker2 = new CodeThorn::UnifiedLTL::UChecker(*analyzer.getEStateSet(),
						     *analyzer.getTransitionGraph());
      break;
    default: 
      cerr << "Error: unknown ltl-verifier specified with ltl-verifier option."<<endl;
      exit(1);
    }
    ltl_input = fopen(ltl_file.c_str(), "r");
    if (ltl_input == NULL)
      cerr<<"Error: could not open file "<<ltl_file.c_str()<<endl;
    assert(ltl_input);

    ofstream* csv = NULL;
    if (args.count("csv-ltl")) {
      csv = new ofstream();
      // use binary and \r\n tp enforce DOS line endings
      // http://tools.ietf.org/html/rfc4180
      csv->open(args["csv-ltl"].as<string>().c_str(), ios::trunc|ios::binary);
      //*csv << "Index,\"LTL formula\",Result,Confidence\r\n";
    }

    while ( !ltl_eof) {
      try { 
	ltl_label = 0;
        if (ltl_parse()) {
          cerr<<color("red")<< "Syntax error" <<color("normal")<<endl;
	  ++n;
	  ++n_failed;
	  continue;
        }
        if (ltl_val == NULL) {
	  // empty line
	  continue;
	}
      } catch(const char* s) {
        if (ltl_val) cout<<color("normal")<<string(*ltl_val)<<endl;
        cout<< s<<endl<<color("red")<< "Grammar Error" <<color("normal")<<endl;
	++n;
	++n_failed;
	continue;
      } catch(...) {
	cout<<color("red")<< "Parser exception" << endl;
	++n;
	++n_failed;
	continue;
      }  
	  
      ++n;
      string formula = *ltl_val;
      cout<<endl<<"Verifying formula "<<color("white")<<formula<<color("normal")<<"."<<endl;
      //if (csv) *csv << n <<";\"" <<formula<<"\";";
      if (csv) *csv << n+60 <<",";
      try {
	AType::BoolLattice result;
	if (checker1) result = checker1->verify(*ltl_val);
	if (checker2) result = checker2->verify(*ltl_val);

	if (result.isTrue()) {
	  ++n_yes;
	  cout<<color("green")<<"YES"<<color("normal")<<endl;
	  if (csv) *csv << "yes,9\r\n";
	} else if (result.isFalse()) {
	  ++n_no;
	  cout<<color("cyan")<<"NO"<<color("normal")<<endl;
	  if (csv) *csv << "no,9\r\n";
	} else {
	  ++n_undecided;
	  cout<<color("magenta")<<"UNKNOWN"<<color("normal")<<endl;
	  if (csv) *csv << "unknown,0\r\n";
	}
      } catch(const char* str) {
	++n_failed;
	cerr << "Exception raised: " << str << endl;
	cout<<color("red")<<"ERROR"<<color("normal")<<endl;
	if (csv) *csv << "error,0\r\n";
      } catch(string str) {
	++n_failed;
	cerr << "Exception raised: " << str << endl;
	cout<<color("red")<<"ERROR"<<color("normal")<<endl;
	if (csv) *csv << "error,0\r\n";
      } catch(...) {
	++n_failed;
	cout<<color("red")<<"ERROR"<<color("normal")<<endl;
	if (csv) *csv << "error,0\r\n";
      }  
    }
    fclose(ltl_input);
    if (csv) delete csv;
    if (checker1) delete checker1;
    if (checker2) delete checker2;
    assert(n_yes+n_no+n_undecided+n_failed == n);
    cout<<"\nStatistics "<<endl
        <<"========== "<<endl
	<<n_yes      <<"/"<<n<<color("green")  <<" YES, "       <<color("normal") 	
	<<n_no       <<"/"<<n<<color("cyan")   <<" NO, "        <<color("normal") 	
	<<n_undecided<<"/"<<n<<color("magenta")<<" UNKNOWN, "   <<color("normal") 	
	<<n_failed   <<"/"<<n<<color("red")    <<" ERROR"       <<color("normal") 	
	<<endl;

  } 
}
Пример #11
0
/**
 * Main function.
 */
int main(int argc, char* argv[])
{
    string inputFile, typeStr;
    bool isAnalyzeFile = false;
    bool isTestLine = false;

    try
    {
        po::options_description config("Allowed options");
        config.add_options()
            ("help,h", "print help message")
            ("file,f", po::value<string>(&inputFile), "use an input file instead of standard input")
            ("type,t", po::value<string>(&typeStr)->default_value("language"), "type of function: [encoding, language, list, segment, sentence]")
            ("line,l", "test each line in input file")
            ;

        po::options_description cmdline_options;
        cmdline_options.add(config);

        po::variables_map vm;
        store(po::command_line_parser(argc, argv).options(cmdline_options).run(), vm);
        po::notify(vm);    

        if(vm.count("help"))
        {
            cout << cmdline_options << endl;
            exit(1);
        }

        if(vm.count("file"))
        {
            isAnalyzeFile = true;
            //cout << "input file: " << inputFile << endl;
        }

        if(typeStr == "encoding" || typeStr == "language" || typeStr == "list" || typeStr == "segment" || typeStr == "sentence")
        {
            //cout << "function type: " << typeStr << endl;
        }
        else
        {
            cerr << "unknown function type: " << typeStr << endl;
            cout << cmdline_options << endl;
            exit(1);
        }

        if(vm.count("line"))
        {
            isTestLine = true;
            //cout << "testing each line" << endl;

            if(! isAnalyzeFile)
            {
                cerr << "error: no input file is given by -f." << endl;
                exit(1);
            }

            if(typeStr != "encoding" && typeStr != "language")
            {
                cerr << "error: line test mode is only valid for \"encoding\" or \"language\" type." << endl;
                exit(1);
            }
        }

        //cout << endl;
    }
    catch(std::exception& e)
    {
        cerr << "error: " << e.what() << "\n";
        exit(1);
    }

    // create instances
    Factory* factory = Factory::instance();
    Analyzer* analyzer = factory->createAnalyzer();
    Knowledge* knowledge = factory->createKnowledge();

    // model files
    const char* encodingModel = "../../db/langid/model/encoding.bin";
    const char* languageModel = "../../db/langid/model/language.bin";

    // load encoding model for encoding identification
    if(! knowledge->loadEncodingModel(encodingModel))
    {
        cerr << "error: fail to load file " << encodingModel << endl;
        exit(1);
    }

    // load language model for language identification or sentence tokenization
    if(! knowledge->loadLanguageModel(languageModel))
    {
        cerr << "error: fail to load file " << languageModel << endl;
        exit(1);
    }

    // not to limit analyze size
    //analyzer->setOption(Analyzer::OPTION_TYPE_LIMIT_ANALYZE_SIZE, 0);

    // set minimum block size
    //analyzer->setOption(Analyzer::OPTION_TYPE_BLOCK_SIZE_THRESHOLD, 100);

    // identify Chinese Traditional text as Chinese Simplified language
    //analyzer->setOption(Analyzer::OPTION_TYPE_NO_CHINESE_TRADITIONAL, 1);
    
    // set knowledge
    analyzer->setKnowledge(knowledge);
    
    // identify character encoding
    if(typeStr == "encoding")
    {
        if(isAnalyzeFile)
        {
            if(isTestLine)
                testEncodingFromFileLine(*analyzer, inputFile.c_str());
            else
                testEncodingFromFile(*analyzer, inputFile.c_str());
        }
        else
            testEncodingFromString(*analyzer);
    }
    // identify the single primary language in UTF-8 encoding
    else if(typeStr == "language")
    {
        if(isAnalyzeFile)
        {
            if(isTestLine)
                testLanguageFromFileLine(*analyzer, inputFile.c_str());
            else
                testLanguageFromFile(*analyzer, inputFile.c_str());
        }
        else
            testLanguageFromString(*analyzer);
    }
    // identify the list of multiple languages in UTF-8 encoding
    else if(typeStr == "list")
    {
        if(isAnalyzeFile)
            testLanguageListFromFile(*analyzer, inputFile.c_str());
        else
            testLanguageListFromString(*analyzer);
    }
    // segment the UTF-8 multi-lingual text into single-language regions
    else if(typeStr == "segment")
    {
        if(isAnalyzeFile)
            testSegmentFile(*analyzer, inputFile.c_str());
        else
            testSegmentString(*analyzer);
    }
    // tokenize the UTF-8 text into sentences
    else if(typeStr == "sentence")
    {
        if(isAnalyzeFile)
        {
            ifstream ifs(inputFile.c_str());
            if(! ifs)
            {
                cerr << "error in opening file " << inputFile << endl;
                exit(1);
            }
            testSentenceLength(*analyzer, ifs);
        }
        else
            testSentenceLength(*analyzer, cin);
    }
    
    delete knowledge;
    delete analyzer;

    return 0;
}
Пример #12
0
int main(int argc, char * argv[])
{
  // HbondMap map;

  // map.push_pair (0,1);
  // map.push_pair (1,2);
  // map.push_pair (2,3);
  // map.push_pair (3,4);
  // map.push_pair (4,5);
  // map.push_pair (5,0);
  // map.push_pair (0,2);
  // map.push_pair (1,3);
  // map.push_pair (2,4);
  // map.push_pair (3,5);
  // map.push_pair (4,0);
  // map.push_pair (5,1);

  // // map.push_pair (0, 1);
  // // map.push_pair (0, 2);
  // // map.push_pair (0, 3);
  // // map.push_pair (1, 3);
  // // map.push_pair (2, 3);
  // // map.push_pair (1, 4);
  // // map.push_pair (2, 5);
  // // map.push_pair (4, 6);
  // // map.push_pair (5, 6);

  // // map.push_pair (101, 102);
  // // map.push_pair (101, 103);
  // // map.push_pair (102, 103);

  // // map.push_pair (202, 203);

  // Tree tree;
  // Circles cs;

  // while (!map.empty()){
  //   tree.addRoot(map, 0);
  //   tree.addGenerations(map);
  //   // tree.print();
  //   Circles tmpc;
  //   tree.buildCircles (tmpc);
  //   tmpc.uniqueCircles();
  //   // std::cout << std::endl;
  //   // std::cout << "before simplified:" << std::endl;
  //   // tmpc.print();
  //   tmpc.simplifyCircles ();
  //   // std::cout << std::endl;
  //   // std::cout << "simplified circles:" << std::endl;
  //   // tmpc.print();    
  //   cs.add (tmpc);
  //   HbondMap newMap;
  //   tree.renewMap (map, newMap);
  //   map = newMap;
  // }

  // cs.uniqueCircles();
  // cs.sortCircles();
  // std::cout << std::endl;
  // std::cout << "simplified circles:" << std::endl;
  // cs.print ();


  std::vector<Hbond> bonds;
  bonds.push_back (Hbond(0,1));
  bonds.push_back (Hbond(1,2));
  bonds.push_back (Hbond(2,3));
  bonds.push_back (Hbond(3,4));
  bonds.push_back (Hbond(4,5));
  bonds.push_back (Hbond(5,0));
  bonds.push_back (Hbond(0,2));
  bonds.push_back (Hbond(1,3));
  bonds.push_back (Hbond(2,4));
  bonds.push_back (Hbond(3,5));
  bonds.push_back (Hbond(4,0));
  bonds.push_back (Hbond(5,1));


  // bonds.push_back (Hbond(0,1));
  // bonds.push_back (Hbond(0,2));
  // bonds.push_back (Hbond(1,3));
  // bonds.push_back (Hbond(2,3));
  // bonds.push_back (Hbond(2,4));
  // bonds.push_back (Hbond(1,4));

  Analyzer ana;
  ana.readData (bonds);
  std::cout << std::endl;
  std::cout << "simplified circles:" << std::endl;
  ana.getCircles().print ();
  

  return 0;
}
Пример #13
0
void Sonogram::demo(QPainter& p) {
  analyze(p, Scope(m_fht->size(), 0), new_frame_);
}
Пример #14
0
/**
 * Main function.
 */
int main(int argc, char* argv[])
{
    const char* sysdict = TEST_JMA_DEFAULT_SYSTEM_DICT;
    const char* stopdict = TEST_JMA_DEFAULT_STOPWORD_DICT;

    for(int optIndex=1; optIndex+1<argc; optIndex+=2)
    {
        if(! strcmp(argv[optIndex], "--stop"))
            stopdict = argv[optIndex+1];
        else if(! strcmp(argv[optIndex], "--dict"))
            sysdict = argv[optIndex+1];
        else
        {
            cerr << "unknown option: " << argv[optIndex] << endl;
            printUsage();
            exit(1);
        }
    }

    cout << "system dictionary: " << sysdict << endl;
    cout << "stop word dictionary: " << stopdict << endl;

    // create factory
    JMA_Factory* factory = JMA_Factory::instance();

    // create analyzer and knowledge
    Analyzer* analyzer = factory->createAnalyzer();
    Knowledge* knowledge = factory->createKnowledge();

    // load dictioanry files
    knowledge->setSystemDict(sysdict);
    if(knowledge->loadDict() == 0)
    {
        cerr << "error: fail to load dictionary files" << endl;
        exit(1);
    }
    cout << "encoding type of system dictionary: " << Knowledge::encodeStr(knowledge->getEncodeType()) << endl;

    // load stop word dictionary
    if(knowledge->loadStopWordDict(stopdict) == 0)
    {
        cerr << "error: fail to load stop word dictionary" << endl;
        exit(1);
    }

    // set knowledge
    if(analyzer->setKnowledge(knowledge) == 0)
    {
        cerr << "fail to set knowledge" << endl;
        exit(1);
    }

    Sentence s;
    string line;
    while(getline(cin, line))
    {
        s.setString(line.c_str());

        if(analyzer->runWithSentence(s) != 1)
        {
            cerr << "error: fail in Analyzer::runWithSentence()" << endl;
            exit(1);
        }

        // get one-best result
        int i= s.getOneBestIndex();
        if(i == -1)
            cout << "no one-best result exists." << endl;
        else
        {
            for(int j=0; j<s.getCount(i); ++j)
                cout << s.getLexicon(i, j) << "/" << s.getStrPOS(i, j) << "  ";

            cout << endl;
        }
    }

    delete knowledge;
    delete analyzer;

    return 0;
}
Пример #15
0
/**
 * Main function.
 */
int main(int argc, char* argv[])
{
    if(argc < 3)
    {
        printUsage();
        exit(1);
    }

    // set default dictionary file
    const char* sysdict = TEST_JMA_DEFAULT_SYSTEM_DICT;
    if(argc > 3)
    {
        if(argc == 5 && ! strcmp(argv[3], "--dict"))
            sysdict = argv[4];
        else
        {
            cerr << "unknown command option " << argv[3] << endl;
            printUsage();
            exit(1);
        }
    }

    // create factory
    JMA_Factory* factory = JMA_Factory::instance();

    // create analyzer and knowledge
    Analyzer* analyzer = factory->createAnalyzer();
    Knowledge* knowledge = factory->createKnowledge();

    // load dictioanry files
    knowledge->setSystemDict(sysdict);
    cout << "system dictionary: " << sysdict << endl;
    if(knowledge->loadDict() == 0)
    {
        cerr << "fail to load dictionary files" << endl;
        exit(1);
    }
    cout << "encoding type of system dictionary: " << Knowledge::encodeStr(knowledge->getEncodeType()) << endl;

    // set knowledge
    if(analyzer->setKnowledge(knowledge) == 0)
    {
        cerr << "fail to set knowledge" << endl;
        exit(1);
    }

    // open files
    const char* source = argv[1];
    const char* dest = argv[2];
    assert(source && dest);

    ifstream from(source);
    if(! from)
    {
        cerr << "error in opening file: " << source << endl;
        exit(1);
    }

    ofstream to(dest);
    if(! to)
    {
        cerr << "error in opening file: " << dest << endl;
        exit(1);
    }

    // split sentences
    string line;
    vector<Sentence> sentVec;
    while(getline(from, line))
    {
        sentVec.clear();
        analyzer->splitSentence(line.c_str(), sentVec);
        for(size_t i=0; i<sentVec.size(); ++i)
        {
            string str(sentVec[i].getString());
            trimString(str);

            if(!str.empty())
                to << str << endl;
        }
    }

    // destroy instances
    delete knowledge;
    delete analyzer;

    return 0;
}