コード例 #1
0
ファイル: selector_factory.cpp プロジェクト: MGKhKhD/meta
std::unique_ptr<feature_selector>
make_selector(const cpptoml::table& config,
              std::shared_ptr<index::forward_index> idx)
{
    auto table = config.get_table("features");
    if (!table)
        throw selector_factory_exception{
            "[features] table missing from config file"};

    auto prefix = table->get_as<std::string>("prefix");
    if (!prefix)
        throw selector_factory_exception{"no prefix in [features] table"};

    auto method = table->get_as<std::string>("method");
    if (!method)
        throw selector_factory_exception{
            "feature selection method required in [features] table"};

    auto features_per_class = static_cast<uint64_t>(
        table->get_as<int64_t>("features-per-class").value_or(20));

    auto selector
        = selector_factory::get().create(*method, *table, std::move(idx));
    selector->init(features_per_class); // make_selector is a friend
    return selector;
}
コード例 #2
0
ファイル: dual_perceptron.cpp プロジェクト: MGKhKhD/meta
std::unique_ptr<classifier>
    make_classifier<dual_perceptron>(const cpptoml::table& config,
                                     multiclass_dataset_view training)
{
    auto alpha = config.get_as<double>("alpha")
                     .value_or(dual_perceptron::default_alpha);

    auto gamma = config.get_as<double>("gamma")
                     .value_or(dual_perceptron::default_gamma);

    auto bias
        = config.get_as<double>("bias").value_or(dual_perceptron::default_bias);

    auto max_iter = config.get_as<int64_t>("max-iter")
                        .value_or(dual_perceptron::default_max_iter);

    auto kernel_cfg = config.get_table("kernel");
    if (!kernel_cfg)
        return make_unique<dual_perceptron>(std::move(training),
                                            make_unique<kernel::polynomial>(),
                                            alpha, gamma, bias, max_iter);

    return make_unique<dual_perceptron>(std::move(training),
                                        kernel::make_kernel(*kernel_cfg), alpha,
                                        gamma, bias, max_iter);
}
コード例 #3
0
ファイル: one_vs_all.cpp プロジェクト: AlexOcculate/meta
std::unique_ptr<classifier>
    make_classifier<one_vs_all>(const cpptoml::table& config,
                                std::shared_ptr<index::forward_index> idx)
{
    auto base = config.get_table("base");
    if (!base)
        throw classifier_factory::exception{
            "one-vs-all missing base-classifier parameter in config file"};
    return make_unique<one_vs_all>(idx, [&](class_label positive_label)
    {
        return make_binary_classifier(*base, idx, positive_label,
                                      class_label{"negative"});
    });
}
コード例 #4
0
ファイル: analyzer.cpp プロジェクト: MGKhKhD/meta
std::unique_ptr<analyzer> load(const cpptoml::table& config)
{
    using namespace analyzers;
    std::vector<std::unique_ptr<analyzer>> toks;
    auto analyzers = config.get_table_array("analyzers");
    for (auto group : analyzers->get())
    {
        auto method = group->get_as<std::string>("method");
        if (!method)
            throw analyzer_exception{"failed to find analyzer method"};
        toks.emplace_back(
            analyzer_factory::get().create(*method, config, *group));
    }
    return make_unique<multi_analyzer>(std::move(toks));
}
コード例 #5
0
ファイル: metadata.cpp プロジェクト: eektguj/meta
metadata::schema_type metadata_schema(const cpptoml::table& config)
{
    metadata::schema_type schema;
    if (auto metadata = config.get_table_array("metadata"))
    {
        const auto& arr = metadata->get();
        schema.reserve(arr.size());
        for (const auto& table : arr)
        {
            auto name = table->get_as<std::string>("name");
            auto type = table->get_as<std::string>("type");

            if (!name)
                throw metadata_exception{"name needed for metadata field"};

            if (!type)
                throw metadata_exception{"type needed for metadata field"};

            metadata::field_type ftype;
            if (*type == "int")
            {
                ftype = metadata::field_type::SIGNED_INT;
            }
            else if (*type == "uint")
            {
                ftype = metadata::field_type::UNSIGNED_INT;
            }
            else if (*type == "double")
            {
                ftype = metadata::field_type::DOUBLE;
            }
            else if (*type == "string")
            {
                ftype = metadata::field_type::STRING;
            }
            else
            {
                throw metadata_exception{"invalid metadata type: \"" + *type
                                         + "\""};
            }
            schema.emplace_back(*name, ftype);
        }
    }
    return schema;
}
コード例 #6
0
ファイル: language_model.cpp プロジェクト: MGKhKhD/meta
language_model::language_model(const cpptoml::table& config)
{
    auto table = config.get_table("language-model");
    auto arpa_file = table->get_as<std::string>("arpa-file");
    auto binary_file = table->get_as<std::string>("binary-file-prefix");

    N_ = 0;
    if (binary_file && filesystem::file_exists(*binary_file + "0.binlm"))
    {
        LOG(info) << "Loading language model from binary files: "
                  << *binary_file << "*" << ENDLG;
        auto time = common::time(
            [&]()
            {
                prefix_ = *binary_file;
                load_vocab();
                while (filesystem::file_exists(*binary_file + std::to_string(N_)
                                               + ".binlm"))
                    lm_.emplace_back(*binary_file + std::to_string(N_++)
                                     + ".binlm");
            });
        LOG(info) << "Done. (" << time.count() << "ms)" << ENDLG;
    }
    else if (arpa_file && binary_file)
    {
        LOG(info) << "Loading language model from .arpa file: " << *arpa_file
                  << ENDLG;
        prefix_ = *binary_file;
        auto time = common::time([&]()
                                 {
                                     read_arpa_format(*arpa_file);
                                 });
        LOG(info) << "Done. (" << time.count() << "ms)" << ENDLG;
    }
    else
        throw language_model_exception{
            "arpa-file or binary-file-prefix needed in config file"};

    // cache this value
    auto unk = vocabulary_.at("<unk>");
    unk_node_ = *lm_[0].find(&unk, &unk + 1);
}
コード例 #7
0
ファイル: analyzer.cpp プロジェクト: MGKhKhD/meta
std::unique_ptr<token_stream> load_filters(const cpptoml::table& global,
                                           const cpptoml::table& config)
{

    auto check = config.get_as<std::string>("filter");
    if (check)
    {
        if (*check == "default-chain")
            return default_filter_chain(global);
        else if (*check == "default-unigram-chain")
            return default_unigram_chain(global);
        else
            throw analyzer_exception{"unknown filter option: " + *check};
    }

    auto filters = config.get_table_array("filter");
    if (!filters)
        throw analyzer_exception{"analyzer group missing filter configuration"};
    std::unique_ptr<token_stream> result;
    for (const auto filter : filters->get())
        result = load_filter(std::move(result), *filter);
    return result;
}
コード例 #8
0
ファイル: profile.cpp プロジェクト: niyikai/MOOC-Course
/**
 * Parses all sentences in a text file.
 */
void parse(const std::string& file, const cpptoml::table& config)
{
    std::cout << "Running parser" << std::endl;

    auto seq_grp = config.get_table("sequence");
    if (!seq_grp)
    {
        std::cerr << "[sequence] group needed in config file" << std::endl;
        return;
    }

    auto prefix = seq_grp->get_as<std::string>("prefix");
    if (!prefix)
    {
        std::cerr << "[sequence] group needs a prefix key" << std::endl;
        return;
    }

    auto parser_grp = config.get_table("parser");
    if (!parser_grp)
    {
        std::cerr << "[parser] group needed in config file" << std::endl;
        return;
    }

    auto parser_prefix = parser_grp->get_as<std::string>("prefix");
    if (!parser_prefix)
    {
        std::cerr << "[parser] group needs a prefix key" << std::endl;
        return;
    }

    std::cout << "Loading tagging model" << std::endl;
    // load POS-tagging model
    sequence::perceptron tagger{*prefix};

    std::cout << "Loading parser model" << std::endl;
    // load parser model
    parser::sr_parser parser{*parser_prefix};

    // construct the token filter chain
    std::unique_ptr<analyzers::token_stream> stream
        = make_unique<analyzers::tokenizers::icu_tokenizer>();
    stream = make_unique<analyzers::filters::ptb_normalizer>(std::move(stream));

    stream->set_content(filesystem::file_text(file));

    // parse each sentence in the file
    // and write its output to the output file
    auto out_name = no_ext(file) + ".parsed.txt";
    std::ofstream outfile{out_name};
    sequence::sequence seq;
    while (*stream)
    {
        auto token = stream->next();
        if (token == "<s>")
        {
            seq = {};
        }
        else if (token == "</s>")
        {
            tagger.tag(seq);
            parser.parse(seq).pretty_print(outfile);
        }
        else
        {
            seq.add_symbol(sequence::symbol_t{token});
        }
    }

    std::cout << " -> file saved as " << out_name << std::endl;
}
コード例 #9
0
ファイル: profile.cpp プロジェクト: niyikai/MOOC-Course
/**
 * Performs part-of-speech tagging on a text file.
 * @param file The input file
 * @param config Configuration settings
 * @param replace Whether or not to replace words with their POS tags
 */
void pos(const std::string& file, const cpptoml::table& config, bool replace)
{
    std::cout << "Running POS-tagging with replace = " << std::boolalpha
              << replace << std::endl;

    auto seq_grp = config.get_table("sequence");
    if (!seq_grp)
    {
        std::cerr << "[sequence] group needed in config file" << std::endl;
        return;
    }

    auto prefix = seq_grp->get_as<std::string>("prefix");
    if (!prefix)
    {
        std::cerr << "[sequence] group needs a prefix key" << std::endl;
        return;
    }

    std::cout << "Loading tagging model" << std::endl;
    sequence::perceptron tagger{*prefix};

    // construct the token filter chain
    std::unique_ptr<analyzers::token_stream> stream
        = make_unique<analyzers::tokenizers::icu_tokenizer>();
    stream = make_unique<analyzers::filters::ptb_normalizer>(std::move(stream));

    stream->set_content(filesystem::file_text(file));

    // tag each sentence in the file
    // and write its output to the output file
    auto out_name = no_ext(file)
                    + (replace ? ".pos-replace.txt" : ".pos-tagged.txt");
    std::ofstream outfile{out_name};
    sequence::sequence seq;
    while (*stream)
    {
        auto token = stream->next();
        if (token == "<s>")
        {
            seq = {};
        }
        else if (token == "</s>")
        {
            tagger.tag(seq);
            for (const auto& obs : seq)
            {
                if (replace)
                    outfile << obs.tag() << " ";
                else
                    outfile << obs.symbol() << "_" << obs.tag() << " ";
            }
            outfile << std::endl;
        }
        else
        {
            seq.add_symbol(sequence::symbol_t{token});
        }
    }

    std::cout << " -> file saved as " << out_name << std::endl;
}