void TranslationUnit::createTranslationUnitIfNeeded() const
{
if (!d->translationUnit) {
d->translationUnit = CXTranslationUnit();
const auto args = commandLineArguments();
if (isVerboseModeEnabled())
args.print();
CXErrorCode errorCode = clang_parseTranslationUnit2(index(),
NULL,
args.data(),
args.count(),
unsavedFiles().cxUnsavedFiles(),
unsavedFiles().count(),
defaultOptions(),
&d->translationUnit);
checkTranslationUnitErrorCode(errorCode);
updateIncludeFilePaths();
updateLastProjectPartChangeTimePoint();
}
}
void TranslationUnitUpdater::createTranslationUnitIfNeeded()
{
if (!m_cxTranslationUnit) {
m_cxTranslationUnit = CXTranslationUnit();
const auto args = commandLineArguments();
if (isVerboseModeEnabled())
args.print();
UnsavedFilesShallowArguments unsaved = m_in.unsavedFiles.shallowArguments();
m_parseErrorCode = clang_parseTranslationUnit2(m_cxIndex,
NULL,
args.data(),
args.count(),
unsaved.data(),
unsaved.count(),
defaultParseOptions(),
&m_cxTranslationUnit);
if (parseWasSuccessful()) {
updateIncludeFilePaths();
m_out.parseTimePoint = Clock::now();
} else {
qWarning() << "Parsing" << m_in.filePath << "failed:"
<< errorCodeToText(m_parseErrorCode);
m_out.hasParseOrReparseFailed = true;
}
}
}
/**@brief Construct the setUp with the generic argc and argv
*
* @param argc The number of arguments
* @param argv The array of char pointers of the arguments
*
*/
programSetUp(int argc, char *argv[]) :
commands_(commandLineArguments(argc, argv)) {
commands_.arguments_["-program"] = argv[0];
// get rid of the ./ if program is being called from current dir, it can
// mess things up latter
programName_ = replaceString(argv[0], "./", "");
init();
}
/**@brief Construct the setUp with a map of string pairs converted from argc
*and argv
*
* @param inputCommands A map of string pairs object likely converted from argc
*and argv
*
*/
programSetUp(const std::map<std::string, std::string> &inputCommands) :
commands_(commandLineArguments(inputCommands)) {
auto progSearch = commands_.arguments_.find("-program");
// if no program name (though there really should be) just call it program
if (commands_.arguments_.end() == progSearch) {
programName_ = "program";
} else {
programName_ = progSearch->second;
}
init();
}
int main(int argc, char* argv[])
{
std::string compassRoot = cc::util::binaryPathToInstallDir(argv[0]);
const std::string PARSER_PLUGIN_DIR = compassRoot + "/lib/parserplugin";
const std::string SQL_DIR = compassRoot + "/share/codecompass/sql";
cc::util::initLogger();
//--- Process command line arguments ---//
po::options_description desc = commandLineArguments();
po::variables_map vm;
po::store(po::command_line_parser(argc, argv)
.options(desc).allow_unregistered().run(), vm);
//--- Skip parser list ---//
std::vector<std::string> skipParserList;
if (vm.count("skip"))
skipParserList = vm["skip"].as<std::vector<std::string>>();
//--- Load parsers ---//
cc::parser::PluginHandler pHandler(PARSER_PLUGIN_DIR);
pHandler.loadPlugins(skipParserList);
//--- Add arguments of parsers ---//
po::options_description pluginOptions = pHandler.getOptions();
desc.add(pluginOptions);
po::store(po::parse_command_line(argc, argv, desc), vm);
if (argc < 2 || vm.count("help"))
{
std::cout << desc << std::endl;
return 0;
}
if (vm.count("loglevel"))
{
trivial::severity_level loglevel
= vm["loglevel"].as<trivial::severity_level>();
boost::shared_ptr<boost::log::core> logger = boost::log::core::get();
logger->set_filter(boost::log::expressions::attr<
trivial::severity_level>("Severity") >= loglevel);
logger->add_global_attribute("Severity",
boost::log::attributes::mutable_constant<trivial::severity_level>(
loglevel));
}
if (vm.count("list"))
{
std::cout << "Available plugins:" << std::endl;
for (const std::string& pluginName : pHandler.getPluginNames())
std::cout << " - " << pluginName << std::endl;
return 0;
}
try
{
po::notify(vm);
}
catch (const po::error& e)
{
LOG(error) << "Error in command line arguments: " << e.what();
return 1;
}
//--- Check database and project directory existence ---//
bool isNewDb = cc::util::connectDatabase(
vm["database"].as<std::string>(), false) == nullptr;
bool isNewProject = !checkProjectDir(vm);
if ((isNewProject ^ isNewDb) && !vm.count("force"))
{
LOG(error) << "Database and working directory existence are inconsistent. "
"Use -f for reparsing!";
return 1;
}
if (!isNewDb && !vm.count("force"))
{
LOG(info)
<< "Project already exists, incremental parsing in action"
<< (vm.count("dry-run") ? " (DRY RUN)" : "") << ".";
}
if (isNewDb && vm.count("dry-run"))
{
LOG(error) << "Dry-run can be only used with incremental parsing, "
"no project found. Try turning --dry-run off.";
return 1;
}
//--- Prepare workspace and project directory ---//
std::string projDir = prepareProjectDir(vm);
if (projDir.empty())
return 1;
//--- Create and init database ---//
std::shared_ptr<odb::database> db = cc::util::connectDatabase(
vm["database"].as<std::string>(), true);
std::unordered_map<std::string, cc::parser::IncrementalStatus> fileStatus;
if (!db)
{
LOG(error) << "Couldn't connect to database. Check the connection string. "
"Connection string format: \"" +
cc::util::getDbDriver() + ":<opt1>=<val1>;<opt2>=<val2>...\"";
return 1;
}
if (vm.count("force"))
cc::util::removeTables(db, SQL_DIR);
if (vm.count("force") || isNewDb)
cc::util::createTables(db, SQL_DIR);
//--- Start parsers ---//
cc::parser::SourceManager srcMgr(db);
cc::parser::ParserContext ctx(db, srcMgr, compassRoot, vm);
pHandler.createPlugins(ctx);
// TODO: Handle errors returned by preparse().
std::vector<std::string> topologicalOrder = pHandler.getTopologicalOrder();
for (auto it = topologicalOrder.rbegin(); it != topologicalOrder.rend(); ++it)
{
LOG(info) << "[" << *it << "] preparse started!";
if (!pHandler.getParser(*it)->preparse(vm.count("dry-run")))
{
LOG(error) << "[" << *it << "] preparse failed!";
return 2;
}
}
if (vm.count("dry-run"))
{
incrementalList(ctx);
}
else
{
incrementalCleanup(ctx);
// TODO: Handle errors returned by parse().
for (const std::string& parserName : pHandler.getTopologicalOrder())
{
LOG(info) << "[" << parserName << "] parse started!";
pHandler.getParser(parserName)->parse();
}
}
//--- Add indexes to the database ---//
if (vm.count("force") || isNewDb)
cc::util::createIndexes(db, SQL_DIR);
//--- Create project config file ---//
boost::property_tree::ptree pt;
if (vm.count("label"))
{
boost::property_tree::ptree labels;
for (const std::string& label : vm["label"].as<std::vector<std::string>>())
{
std::size_t pos = label.find('=');
if (pos == std::string::npos)
LOG(warning)
<< "Label doesn't contain '=' for separating label and the path: "
<< label;
else
labels.put(label.substr(0, pos), label.substr(pos + 1));
}
pt.add_child("labels", labels);
}
std::string database
= cc::util::connStrComponent(vm["database"].as<std::string>(), "database");
pt.put(
"database",
database.empty() ? vm["name"].as<std::string>() : database);
if (vm.count("description"))
pt.put("description", vm["description"].as<std::string>());
boost::property_tree::write_json(projDir + "/project_info.json", pt);
// TODO: Print statistics.
return 0;
}
int main (int argc, char **argv)
{
Onyx::LinearLaRank::Classifier *classifier;
Onyx::Example::Dataset datasetTrain;
Onyx::Example::Dataset datasetTest;
std::chrono::time_point<std::chrono::system_clock> start, end; // Timings
std::string filenameTrainingData;
std::string filenameTrainingLabels;
std::string filenameTestData;
std::string filenameTestLabels;
std::string saveClassifier;
std::string loadClassifier;
bool doTraining = true; // Enable all by default
bool doTesting = true; // Enable by default
bool doOnlineTesting = false; // Disable by default
unsigned int numEpochs = 10;
double C = 1.0;
double tau = 0.0001;
// Random number generator (with random seed)
std::mt19937 random_number_generator(std::random_device{}());
// *** Print banner ***
std::cout << "Onyx v.1.0, (C) 2015 Rok Mandeljc <*****@*****.**>" << std::endl;
std::cout << std::endl;
// *** Command-line parser ***
boost::program_options::options_description commandLineArguments("Onyx - Online Classifier Application");
boost::program_options::variables_map optionsMap;
boost::program_options::positional_options_description positionalArguments;
boost::program_options::options_description argArguments("Arguments");
argArguments.add_options()
("help", "produce help message")
("training-data", boost::program_options::value<std::string>(&filenameTrainingData), "name of training .data file")
("training-labels", boost::program_options::value<std::string>(&filenameTrainingLabels), "name of training .labels file")
("test-data", boost::program_options::value<std::string>(&filenameTestData), "name of test .data file")
("test-labels", boost::program_options::value<std::string>(&filenameTestLabels), "name of test .labels file")
("save-classifier", boost::program_options::value<std::string>(&saveClassifier), "optional filename to store classifier")
("load-classifier", boost::program_options::value<std::string>(&loadClassifier), "optional filename to load classifier")
("epochs", boost::program_options::value<unsigned int>(&numEpochs)->default_value(numEpochs), "number of epochs for (re)training")
("training", boost::program_options::value<bool>(&doTraining)->default_value(doTraining), "enable training (if training data is available)")
("test", boost::program_options::value<bool>(&doTesting)->default_value(doTesting), "enable testing (if testing data is available)")
("online-test", boost::program_options::value<bool>(&doOnlineTesting)->default_value(doOnlineTesting), "enable on-line testing (if testing data is available)")
;
commandLineArguments.add(argArguments);
boost::program_options::options_description argParameters("Algorithm parameters");
argParameters.add_options()
("C", boost::program_options::value<double>(&C)->default_value(C), "SVM regularization parameter")
("tau", boost::program_options::value<double>(&tau)->default_value(tau), "tolerance for choosing new support vectors")
;
commandLineArguments.add(argParameters);
positionalArguments.add("training-data", 1);
positionalArguments.add("training-labels", 1);
positionalArguments.add("test-data", 1);
positionalArguments.add("test-labels", 1);
// Parse command-line
try {
boost::program_options::store(boost::program_options::command_line_parser(argc, argv).options(commandLineArguments).positional(positionalArguments).run(), optionsMap);
} catch (std::exception &error) {
std::cout << commandLineArguments << std::endl << std::endl;
std::cout << "Command-line error: " << error.what() << std::endl;
return -1;
}
// Display help?
if (optionsMap.count("help")) {
std::cout << commandLineArguments << std::endl;
return 1;
}
// Validate
try {
boost::program_options::notify(optionsMap);
} catch (std::exception &error) {
std::cout << commandLineArguments << std::endl << std::endl;
std::cout << "Argument error: " << error.what() << std::endl;
return -1;
}
bool trainingDataAvailable = !filenameTrainingData.empty() && !filenameTrainingLabels.empty();
bool testingDataAvailable = !filenameTestData.empty() && !filenameTestLabels.empty();
doTraining = doTraining && trainingDataAvailable;
doTesting = doTesting && testingDataAvailable;
doOnlineTesting = doOnlineTesting && testingDataAvailable;
if (!doTraining && !doTesting && !doOnlineTesting) {
std::cout << "Doing neither training nor testing; nothing to do!" << std::endl;
return 1;
}
if (!doTraining && loadClassifier.empty()) {
std::cout << "Neither training dataset nor pre-trained classifier provided!" << std::endl;
return -1;
}
// *** Load datasets ***
if (doTraining) {
std::cout << "Loading training dataset..." << std::endl;
try {
datasetTrain.load(filenameTrainingData, filenameTrainingLabels);
} catch (std::exception &error) {
std::cout << "Failed to load training dataset: " << error.what() << std::endl;
return -2;
}
std::cout << "Loaded training set:" << std::endl;
std::cout << " data file: " << filenameTrainingData << std::endl;
std::cout << " labels file: " << filenameTrainingLabels << std::endl;
std::cout << " samples: " << datasetTrain.numSamples << std::endl;
std::cout << " features: " << datasetTrain.numFeatures << std::endl;
std::cout << " classes: " << datasetTrain.numClasses << std::endl;
std::cout << std::endl;
}
if (doTesting || doOnlineTesting) {
std::cout << "Loading testing dataset..." << std::endl;
try {
datasetTest.load(filenameTestData, filenameTestLabels);
} catch (std::exception &error) {
std::cout << "Failed to load testing dataset: " << error.what() << std::endl;
return -2;
}
std::cout << "Loaded testing set:" << std::endl;
std::cout << " data file: " << filenameTestData << std::endl;
std::cout << " labels file: " << filenameTestLabels << std::endl;
std::cout << " samples: " << datasetTest.numSamples << std::endl;
std::cout << " features: " << datasetTest.numFeatures << std::endl;
std::cout << " classes: " << datasetTest.numClasses << std::endl;
std::cout << std::endl;
}
// *** Classifier ***
classifier = Onyx::LinearLaRank::create_classifier();
if (!loadClassifier.empty()) {
// Load from file
std::cout << "Loading classifier from file: " << loadClassifier << std::endl;
std::cout << std::endl;
std::ifstream stream(loadClassifier, std::ios::binary);
classifier->loadFromStream(stream);
} else {
// Create new classifier
std::cout << "Creating new classifier..." << std::endl;
std::cout << std::endl;
classifier->setC(C);
classifier->setTau(tau);
}
// *** Training (with optional testing) ***
if (doTraining) {
start = std::chrono::system_clock::now();
int sampleRatio = datasetTrain.numSamples / 10;
std::vector<float> trainError(numEpochs, 0.0);
std::vector<float> testError(numEpochs, 0.0);
std::vector<int> sampleIndices(datasetTrain.numSamples);
std::iota(sampleIndices.begin(), sampleIndices.end(), 0);
for (unsigned int epoch = 0; epoch < numEpochs; epoch++) {
std::cout << "Epoch " << epoch << std::endl;
// *** Train ***
// Randomly permute the sample indices
std::vector<std::vector<int>::iterator> shuffledSampleIndices(sampleIndices.size());
std::iota(shuffledSampleIndices.begin(), shuffledSampleIndices.end(), sampleIndices.begin());
std::shuffle(shuffledSampleIndices.begin(), shuffledSampleIndices.end(), random_number_generator);
for (unsigned int s = 0; s < shuffledSampleIndices.size(); s++) {
int idx = *shuffledSampleIndices[s];
const Eigen::VectorXf &sampleFeature = datasetTrain.features[idx];
int sampleLabel = datasetTrain.labels[idx];
// Estimate training error
int label = classifier->predict(sampleFeature);
if (label != sampleLabel) {
trainError[epoch]++;
}
// Update
classifier->update(sampleFeature, sampleLabel, 1.0f);
// Print progress
if (s && sampleRatio && s % sampleRatio == 0) {
std::cout << "Epoch: " << epoch << ": ";
std::cout << (10 * s) / sampleRatio << "%";
std::cout << " -> training error: " << trainError[epoch];
std::cout << "/" << s;
std::cout << " = " << trainError[epoch]/s*100 << "%";
std::cout << std::endl;
}
}
if (doTesting) {
// *** Test ***
for (unsigned int s = 0; s < datasetTest.numSamples; s++) {
const Eigen::VectorXf &sampleFeature = datasetTest.features[s];
int sampleLabel = datasetTest.labels[s];
if (classifier->predict(sampleFeature) != sampleLabel) {
testError[epoch]++;
}
}
std::cout << "Test error: " << testError[epoch] << "/" << datasetTest.numSamples << " = " << testError[epoch]/datasetTest.numSamples*100 << "%" << std::endl;
std::cout << std::endl;
}
}
end = std::chrono::system_clock::now();
std::cout << "Elapsed time: " << std::chrono::duration<float>(end - start).count() << " seconds." << std::endl;
}
// *** Save classifier ***
if (!saveClassifier.empty()) {
std::cout << "Saving classifier to file: " << saveClassifier << std::endl;
std::ofstream stream(saveClassifier, std::ios::binary);
classifier->saveToStream(stream);
std::cout << "Done!" << std::endl;
}
// *** Test - only if we didn't do the training ***
if (!doTraining && doTesting) {
float testError = 0.0f;
std::cout << "Performing off-line test..." << std::endl;
// Experiment
start = std::chrono::system_clock::now();
for (unsigned int s = 0; s < datasetTest.numSamples; s++) {
const Eigen::VectorXf &sampleFeature = datasetTest.features[s];
int sampleLabel = datasetTest.labels[s];
if (classifier->predict(sampleFeature) != sampleLabel) {
testError++;
}
}
end = std::chrono::system_clock::now();
std::cout << "Test error: " << testError << "/" << datasetTest.numSamples << " = " << testError/datasetTest.numSamples*100 << "%" << std::endl;
std::cout << "Elapsed time: " << std::chrono::duration<float>(end - start).count() << " seconds." << std::endl;
std::cout << std::endl;
}
// *** Online test ***
if (doOnlineTesting) {
float testError = 0.0f;
std::cout << "Performing on-line test..." << std::endl;
// Randomly permute the sample indices
std::vector<int> sampleIndices(datasetTest.numSamples);
std::iota(sampleIndices.begin(), sampleIndices.end(), 0);
std::vector<std::vector<int>::iterator> shuffledSampleIndices(sampleIndices.size());
std::iota(shuffledSampleIndices.begin(), shuffledSampleIndices.end(), sampleIndices.begin());
std::shuffle(shuffledSampleIndices.begin(), shuffledSampleIndices.end(), random_number_generator);
// Experiment
start = std::chrono::system_clock::now();
for (unsigned int s = 0; s < shuffledSampleIndices.size(); s++) {
int idx = *shuffledSampleIndices[s];
const Eigen::VectorXf &sampleFeature = datasetTest.features[idx];
int sampleLabel = datasetTest.labels[idx];
// Predict
if (classifier->predict(sampleFeature) != sampleLabel) {
testError++;
}
// Update
classifier->update(sampleFeature, sampleLabel);
}
end = std::chrono::system_clock::now();
std::cout << "Online test error: " << testError << "/" << datasetTest.numSamples << " = " << testError/datasetTest.numSamples*100 << "%" << std::endl;
std::cout << "Elapsed time: " << std::chrono::duration<float>(end - start).count() << " seconds." << std::endl;
std::cout << std::endl;
}
// Cleanup
delete classifier;
return 0;
}
