void TranslationUnit::createTranslationUnitIfNeeded() const { if (!d->translationUnit) { d->translationUnit = CXTranslationUnit(); const auto args = commandLineArguments(); if (isVerboseModeEnabled()) args.print(); CXErrorCode errorCode = clang_parseTranslationUnit2(index(), NULL, args.data(), args.count(), unsavedFiles().cxUnsavedFiles(), unsavedFiles().count(), defaultOptions(), &d->translationUnit); checkTranslationUnitErrorCode(errorCode); updateIncludeFilePaths(); updateLastProjectPartChangeTimePoint(); } }
void TranslationUnitUpdater::createTranslationUnitIfNeeded() { if (!m_cxTranslationUnit) { m_cxTranslationUnit = CXTranslationUnit(); const auto args = commandLineArguments(); if (isVerboseModeEnabled()) args.print(); UnsavedFilesShallowArguments unsaved = m_in.unsavedFiles.shallowArguments(); m_parseErrorCode = clang_parseTranslationUnit2(m_cxIndex, NULL, args.data(), args.count(), unsaved.data(), unsaved.count(), defaultParseOptions(), &m_cxTranslationUnit); if (parseWasSuccessful()) { updateIncludeFilePaths(); m_out.parseTimePoint = Clock::now(); } else { qWarning() << "Parsing" << m_in.filePath << "failed:" << errorCodeToText(m_parseErrorCode); m_out.hasParseOrReparseFailed = true; } } }
/**@brief Construct the setUp with the generic argc and argv * * @param argc The number of arguments * @param argv The array of char pointers of the arguments * */ programSetUp(int argc, char *argv[]) : commands_(commandLineArguments(argc, argv)) { commands_.arguments_["-program"] = argv[0]; // get rid of the ./ if program is being called from current dir, it can // mess things up latter programName_ = replaceString(argv[0], "./", ""); init(); }
/**@brief Construct the setUp with a map of string pairs converted from argc *and argv * * @param inputCommands A map of string pairs object likely converted from argc *and argv * */ programSetUp(const std::map<std::string, std::string> &inputCommands) : commands_(commandLineArguments(inputCommands)) { auto progSearch = commands_.arguments_.find("-program"); // if no program name (though there really should be) just call it program if (commands_.arguments_.end() == progSearch) { programName_ = "program"; } else { programName_ = progSearch->second; } init(); }
int main(int argc, char* argv[]) { std::string compassRoot = cc::util::binaryPathToInstallDir(argv[0]); const std::string PARSER_PLUGIN_DIR = compassRoot + "/lib/parserplugin"; const std::string SQL_DIR = compassRoot + "/share/codecompass/sql"; cc::util::initLogger(); //--- Process command line arguments ---// po::options_description desc = commandLineArguments(); po::variables_map vm; po::store(po::command_line_parser(argc, argv) .options(desc).allow_unregistered().run(), vm); //--- Skip parser list ---// std::vector<std::string> skipParserList; if (vm.count("skip")) skipParserList = vm["skip"].as<std::vector<std::string>>(); //--- Load parsers ---// cc::parser::PluginHandler pHandler(PARSER_PLUGIN_DIR); pHandler.loadPlugins(skipParserList); //--- Add arguments of parsers ---// po::options_description pluginOptions = pHandler.getOptions(); desc.add(pluginOptions); po::store(po::parse_command_line(argc, argv, desc), vm); if (argc < 2 || vm.count("help")) { std::cout << desc << std::endl; return 0; } if (vm.count("loglevel")) { trivial::severity_level loglevel = vm["loglevel"].as<trivial::severity_level>(); boost::shared_ptr<boost::log::core> logger = boost::log::core::get(); logger->set_filter(boost::log::expressions::attr< trivial::severity_level>("Severity") >= loglevel); logger->add_global_attribute("Severity", boost::log::attributes::mutable_constant<trivial::severity_level>( loglevel)); } if (vm.count("list")) { std::cout << "Available plugins:" << std::endl; for (const std::string& pluginName : pHandler.getPluginNames()) std::cout << " - " << pluginName << std::endl; return 0; } try { po::notify(vm); } catch (const po::error& e) { LOG(error) << "Error in command line arguments: " << e.what(); return 1; } //--- Check database and project directory existence ---// bool isNewDb = cc::util::connectDatabase( vm["database"].as<std::string>(), false) == nullptr; bool isNewProject = !checkProjectDir(vm); if ((isNewProject ^ isNewDb) && !vm.count("force")) { LOG(error) << "Database and working directory existence are inconsistent. " "Use -f for reparsing!"; return 1; } if (!isNewDb && !vm.count("force")) { LOG(info) << "Project already exists, incremental parsing in action" << (vm.count("dry-run") ? " (DRY RUN)" : "") << "."; } if (isNewDb && vm.count("dry-run")) { LOG(error) << "Dry-run can be only used with incremental parsing, " "no project found. Try turning --dry-run off."; return 1; } //--- Prepare workspace and project directory ---// std::string projDir = prepareProjectDir(vm); if (projDir.empty()) return 1; //--- Create and init database ---// std::shared_ptr<odb::database> db = cc::util::connectDatabase( vm["database"].as<std::string>(), true); std::unordered_map<std::string, cc::parser::IncrementalStatus> fileStatus; if (!db) { LOG(error) << "Couldn't connect to database. Check the connection string. " "Connection string format: \"" + cc::util::getDbDriver() + ":<opt1>=<val1>;<opt2>=<val2>...\""; return 1; } if (vm.count("force")) cc::util::removeTables(db, SQL_DIR); if (vm.count("force") || isNewDb) cc::util::createTables(db, SQL_DIR); //--- Start parsers ---// cc::parser::SourceManager srcMgr(db); cc::parser::ParserContext ctx(db, srcMgr, compassRoot, vm); pHandler.createPlugins(ctx); // TODO: Handle errors returned by preparse(). std::vector<std::string> topologicalOrder = pHandler.getTopologicalOrder(); for (auto it = topologicalOrder.rbegin(); it != topologicalOrder.rend(); ++it) { LOG(info) << "[" << *it << "] preparse started!"; if (!pHandler.getParser(*it)->preparse(vm.count("dry-run"))) { LOG(error) << "[" << *it << "] preparse failed!"; return 2; } } if (vm.count("dry-run")) { incrementalList(ctx); } else { incrementalCleanup(ctx); // TODO: Handle errors returned by parse(). for (const std::string& parserName : pHandler.getTopologicalOrder()) { LOG(info) << "[" << parserName << "] parse started!"; pHandler.getParser(parserName)->parse(); } } //--- Add indexes to the database ---// if (vm.count("force") || isNewDb) cc::util::createIndexes(db, SQL_DIR); //--- Create project config file ---// boost::property_tree::ptree pt; if (vm.count("label")) { boost::property_tree::ptree labels; for (const std::string& label : vm["label"].as<std::vector<std::string>>()) { std::size_t pos = label.find('='); if (pos == std::string::npos) LOG(warning) << "Label doesn't contain '=' for separating label and the path: " << label; else labels.put(label.substr(0, pos), label.substr(pos + 1)); } pt.add_child("labels", labels); } std::string database = cc::util::connStrComponent(vm["database"].as<std::string>(), "database"); pt.put( "database", database.empty() ? vm["name"].as<std::string>() : database); if (vm.count("description")) pt.put("description", vm["description"].as<std::string>()); boost::property_tree::write_json(projDir + "/project_info.json", pt); // TODO: Print statistics. return 0; }
int main (int argc, char **argv) { Onyx::LinearLaRank::Classifier *classifier; Onyx::Example::Dataset datasetTrain; Onyx::Example::Dataset datasetTest; std::chrono::time_point<std::chrono::system_clock> start, end; // Timings std::string filenameTrainingData; std::string filenameTrainingLabels; std::string filenameTestData; std::string filenameTestLabels; std::string saveClassifier; std::string loadClassifier; bool doTraining = true; // Enable all by default bool doTesting = true; // Enable by default bool doOnlineTesting = false; // Disable by default unsigned int numEpochs = 10; double C = 1.0; double tau = 0.0001; // Random number generator (with random seed) std::mt19937 random_number_generator(std::random_device{}()); // *** Print banner *** std::cout << "Onyx v.1.0, (C) 2015 Rok Mandeljc <*****@*****.**>" << std::endl; std::cout << std::endl; // *** Command-line parser *** boost::program_options::options_description commandLineArguments("Onyx - Online Classifier Application"); boost::program_options::variables_map optionsMap; boost::program_options::positional_options_description positionalArguments; boost::program_options::options_description argArguments("Arguments"); argArguments.add_options() ("help", "produce help message") ("training-data", boost::program_options::value<std::string>(&filenameTrainingData), "name of training .data file") ("training-labels", boost::program_options::value<std::string>(&filenameTrainingLabels), "name of training .labels file") ("test-data", boost::program_options::value<std::string>(&filenameTestData), "name of test .data file") ("test-labels", boost::program_options::value<std::string>(&filenameTestLabels), "name of test .labels file") ("save-classifier", boost::program_options::value<std::string>(&saveClassifier), "optional filename to store classifier") ("load-classifier", boost::program_options::value<std::string>(&loadClassifier), "optional filename to load classifier") ("epochs", boost::program_options::value<unsigned int>(&numEpochs)->default_value(numEpochs), "number of epochs for (re)training") ("training", boost::program_options::value<bool>(&doTraining)->default_value(doTraining), "enable training (if training data is available)") ("test", boost::program_options::value<bool>(&doTesting)->default_value(doTesting), "enable testing (if testing data is available)") ("online-test", boost::program_options::value<bool>(&doOnlineTesting)->default_value(doOnlineTesting), "enable on-line testing (if testing data is available)") ; commandLineArguments.add(argArguments); boost::program_options::options_description argParameters("Algorithm parameters"); argParameters.add_options() ("C", boost::program_options::value<double>(&C)->default_value(C), "SVM regularization parameter") ("tau", boost::program_options::value<double>(&tau)->default_value(tau), "tolerance for choosing new support vectors") ; commandLineArguments.add(argParameters); positionalArguments.add("training-data", 1); positionalArguments.add("training-labels", 1); positionalArguments.add("test-data", 1); positionalArguments.add("test-labels", 1); // Parse command-line try { boost::program_options::store(boost::program_options::command_line_parser(argc, argv).options(commandLineArguments).positional(positionalArguments).run(), optionsMap); } catch (std::exception &error) { std::cout << commandLineArguments << std::endl << std::endl; std::cout << "Command-line error: " << error.what() << std::endl; return -1; } // Display help? if (optionsMap.count("help")) { std::cout << commandLineArguments << std::endl; return 1; } // Validate try { boost::program_options::notify(optionsMap); } catch (std::exception &error) { std::cout << commandLineArguments << std::endl << std::endl; std::cout << "Argument error: " << error.what() << std::endl; return -1; } bool trainingDataAvailable = !filenameTrainingData.empty() && !filenameTrainingLabels.empty(); bool testingDataAvailable = !filenameTestData.empty() && !filenameTestLabels.empty(); doTraining = doTraining && trainingDataAvailable; doTesting = doTesting && testingDataAvailable; doOnlineTesting = doOnlineTesting && testingDataAvailable; if (!doTraining && !doTesting && !doOnlineTesting) { std::cout << "Doing neither training nor testing; nothing to do!" << std::endl; return 1; } if (!doTraining && loadClassifier.empty()) { std::cout << "Neither training dataset nor pre-trained classifier provided!" << std::endl; return -1; } // *** Load datasets *** if (doTraining) { std::cout << "Loading training dataset..." << std::endl; try { datasetTrain.load(filenameTrainingData, filenameTrainingLabels); } catch (std::exception &error) { std::cout << "Failed to load training dataset: " << error.what() << std::endl; return -2; } std::cout << "Loaded training set:" << std::endl; std::cout << " data file: " << filenameTrainingData << std::endl; std::cout << " labels file: " << filenameTrainingLabels << std::endl; std::cout << " samples: " << datasetTrain.numSamples << std::endl; std::cout << " features: " << datasetTrain.numFeatures << std::endl; std::cout << " classes: " << datasetTrain.numClasses << std::endl; std::cout << std::endl; } if (doTesting || doOnlineTesting) { std::cout << "Loading testing dataset..." << std::endl; try { datasetTest.load(filenameTestData, filenameTestLabels); } catch (std::exception &error) { std::cout << "Failed to load testing dataset: " << error.what() << std::endl; return -2; } std::cout << "Loaded testing set:" << std::endl; std::cout << " data file: " << filenameTestData << std::endl; std::cout << " labels file: " << filenameTestLabels << std::endl; std::cout << " samples: " << datasetTest.numSamples << std::endl; std::cout << " features: " << datasetTest.numFeatures << std::endl; std::cout << " classes: " << datasetTest.numClasses << std::endl; std::cout << std::endl; } // *** Classifier *** classifier = Onyx::LinearLaRank::create_classifier(); if (!loadClassifier.empty()) { // Load from file std::cout << "Loading classifier from file: " << loadClassifier << std::endl; std::cout << std::endl; std::ifstream stream(loadClassifier, std::ios::binary); classifier->loadFromStream(stream); } else { // Create new classifier std::cout << "Creating new classifier..." << std::endl; std::cout << std::endl; classifier->setC(C); classifier->setTau(tau); } // *** Training (with optional testing) *** if (doTraining) { start = std::chrono::system_clock::now(); int sampleRatio = datasetTrain.numSamples / 10; std::vector<float> trainError(numEpochs, 0.0); std::vector<float> testError(numEpochs, 0.0); std::vector<int> sampleIndices(datasetTrain.numSamples); std::iota(sampleIndices.begin(), sampleIndices.end(), 0); for (unsigned int epoch = 0; epoch < numEpochs; epoch++) { std::cout << "Epoch " << epoch << std::endl; // *** Train *** // Randomly permute the sample indices std::vector<std::vector<int>::iterator> shuffledSampleIndices(sampleIndices.size()); std::iota(shuffledSampleIndices.begin(), shuffledSampleIndices.end(), sampleIndices.begin()); std::shuffle(shuffledSampleIndices.begin(), shuffledSampleIndices.end(), random_number_generator); for (unsigned int s = 0; s < shuffledSampleIndices.size(); s++) { int idx = *shuffledSampleIndices[s]; const Eigen::VectorXf &sampleFeature = datasetTrain.features[idx]; int sampleLabel = datasetTrain.labels[idx]; // Estimate training error int label = classifier->predict(sampleFeature); if (label != sampleLabel) { trainError[epoch]++; } // Update classifier->update(sampleFeature, sampleLabel, 1.0f); // Print progress if (s && sampleRatio && s % sampleRatio == 0) { std::cout << "Epoch: " << epoch << ": "; std::cout << (10 * s) / sampleRatio << "%"; std::cout << " -> training error: " << trainError[epoch]; std::cout << "/" << s; std::cout << " = " << trainError[epoch]/s*100 << "%"; std::cout << std::endl; } } if (doTesting) { // *** Test *** for (unsigned int s = 0; s < datasetTest.numSamples; s++) { const Eigen::VectorXf &sampleFeature = datasetTest.features[s]; int sampleLabel = datasetTest.labels[s]; if (classifier->predict(sampleFeature) != sampleLabel) { testError[epoch]++; } } std::cout << "Test error: " << testError[epoch] << "/" << datasetTest.numSamples << " = " << testError[epoch]/datasetTest.numSamples*100 << "%" << std::endl; std::cout << std::endl; } } end = std::chrono::system_clock::now(); std::cout << "Elapsed time: " << std::chrono::duration<float>(end - start).count() << " seconds." << std::endl; } // *** Save classifier *** if (!saveClassifier.empty()) { std::cout << "Saving classifier to file: " << saveClassifier << std::endl; std::ofstream stream(saveClassifier, std::ios::binary); classifier->saveToStream(stream); std::cout << "Done!" << std::endl; } // *** Test - only if we didn't do the training *** if (!doTraining && doTesting) { float testError = 0.0f; std::cout << "Performing off-line test..." << std::endl; // Experiment start = std::chrono::system_clock::now(); for (unsigned int s = 0; s < datasetTest.numSamples; s++) { const Eigen::VectorXf &sampleFeature = datasetTest.features[s]; int sampleLabel = datasetTest.labels[s]; if (classifier->predict(sampleFeature) != sampleLabel) { testError++; } } end = std::chrono::system_clock::now(); std::cout << "Test error: " << testError << "/" << datasetTest.numSamples << " = " << testError/datasetTest.numSamples*100 << "%" << std::endl; std::cout << "Elapsed time: " << std::chrono::duration<float>(end - start).count() << " seconds." << std::endl; std::cout << std::endl; } // *** Online test *** if (doOnlineTesting) { float testError = 0.0f; std::cout << "Performing on-line test..." << std::endl; // Randomly permute the sample indices std::vector<int> sampleIndices(datasetTest.numSamples); std::iota(sampleIndices.begin(), sampleIndices.end(), 0); std::vector<std::vector<int>::iterator> shuffledSampleIndices(sampleIndices.size()); std::iota(shuffledSampleIndices.begin(), shuffledSampleIndices.end(), sampleIndices.begin()); std::shuffle(shuffledSampleIndices.begin(), shuffledSampleIndices.end(), random_number_generator); // Experiment start = std::chrono::system_clock::now(); for (unsigned int s = 0; s < shuffledSampleIndices.size(); s++) { int idx = *shuffledSampleIndices[s]; const Eigen::VectorXf &sampleFeature = datasetTest.features[idx]; int sampleLabel = datasetTest.labels[idx]; // Predict if (classifier->predict(sampleFeature) != sampleLabel) { testError++; } // Update classifier->update(sampleFeature, sampleLabel); } end = std::chrono::system_clock::now(); std::cout << "Online test error: " << testError << "/" << datasetTest.numSamples << " = " << testError/datasetTest.numSamples*100 << "%" << std::endl; std::cout << "Elapsed time: " << std::chrono::duration<float>(end - start).count() << " seconds." << std::endl; std::cout << std::endl; } // Cleanup delete classifier; return 0; }