Status getDatabaseValue(const std::string& domain, const std::string& key, std::string& value) { if (domain.empty()) { return Status(1, "Missing domain"); } if (RegistryFactory::get().external()) { // External registries (extensions) do not have databases active. // It is not possible to use an extension-based database. PluginRequest request = { {"action", "get"}, {"domain", domain}, {"key", key}}; PluginResponse response; auto status = Registry::call("database", request, response); if (status.ok()) { // Set value from the internally-known "v" key. if (response.size() > 0 && response[0].count("v") > 0) { value = response[0].at("v"); } } return status; } ReadLock lock(kDatabaseReset); if (!DatabasePlugin::kDBInitialized) { throw std::runtime_error("Cannot get database value: " + key); } else { auto plugin = getDatabasePlugin(); return plugin->get(domain, key, value); } }
TEST_F(TLSConfigTests, test_retrieve_config) { TLSServerRunner::start(); TLSServerRunner::setClientConfig(); // Trigger the enroll. auto endpoint = Flag::getValue("config_tls_endpoint"); Flag::updateValue("config_tls_endpoint", "/config"); Registry::setActive("config", "tls"); // Expect a POST to the /config endpoint. // A GET will return different results. Config c; c.load(); const auto& hashes = c.hash_; EXPECT_EQ("c109cd4fc0a928dba787384a89f9d03d", hashes.at("tls_plugin")); // Configure the plugin to use the node API. Flag::updateValue("tls_node_api", "1"); Registry::setActive("config", "tls"); PluginResponse response; auto status = Registry::call("config", {{"action", "genConfig"}}, response); ASSERT_TRUE(status.ok()); ASSERT_EQ(1U, response.size()); // The GET and POST results are slightly different. EXPECT_EQ("baz", response[0]["tls_plugin"]); // Clean up. Flag::updateValue("tls_node_api", "0"); Flag::updateValue("config_tls_endpoint", endpoint); TLSServerRunner::unsetClientConfig(); TLSServerRunner::stop(); }
Status Config::load() { valid_ = false; auto& config_plugin = Registry::getActive("config"); if (!Registry::exists("config", config_plugin)) { return Status(1, "Missing config plugin " + config_plugin); } PluginResponse response; auto status = Registry::call("config", {{"action", "genConfig"}}, response); if (!status.ok()) { loaded_ = true; return status; } // if there was a response, parse it and update internal state valid_ = true; if (response.size() > 0) { if (FLAGS_config_dump) { // If config checking is enabled, debug-write the raw config data. for (const auto& content : response[0]) { fprintf(stdout, "{\"%s\": %s}\n", content.first.c_str(), content.second.c_str()); } // Instead of forcing the shutdown, request one since the config plugin // may have started services. Initializer::requestShutdown(); } status = update(response[0]); } loaded_ = true; return status; }
Status Config::genPack(const std::string& name, const std::string& source, const std::string& target) { // If the pack value is a string (and not a JSON object) then it is a // resource to be handled by the config plugin. PluginResponse response; PluginRequest request = { {"action", "genPack"}, {"name", name}, {"value", target}}; Registry::call("config", request, response); if (response.size() == 0 || response[0].count(name) == 0) { return Status(1, "Invalid plugin response"); } try { pt::ptree pack_tree; std::stringstream pack_stream; pack_stream << response[0][name]; pt::read_json(pack_stream, pack_tree); addPack(name, source, pack_tree); } catch (const pt::json_parser::json_parser_error& /* e */) { LOG(WARNING) << "Error parsing the pack JSON: " << name; } return Status(0); }
Status ConfigPlugin::call(const PluginRequest& request, PluginResponse& response) { if (request.count("action") == 0) { return Status(1, "Config plugins require an action in PluginRequest"); } if (request.at("action") == "genConfig") { std::map<std::string, std::string> config; auto stat = genConfig(config); response.push_back(config); return stat; } else if (request.at("action") == "genPack") { if (request.count("name") == 0 || request.count("value") == 0) { return Status(1, "Missing name or value"); } std::string pack; auto stat = genPack(request.at("name"), request.at("value"), pack); response.push_back({{request.at("name"), pack}}); return stat; } else if (request.at("action") == "update") { if (request.count("source") == 0 || request.count("data") == 0) { return Status(1, "Missing source or data"); } return Config::getInstance().update( {{request.at("source"), request.at("data")}}); } return Status(1, "Config plugin action unknown: " + request.at("action")); }
TEST_F(RemoteEnrollmentTests, test_enroll) { // Set the enrollment URI to the server we created. FLAGS_enrollment_uri = "http://127.0.0.1:" + port_; FLAGS_enrollment_app_id = "just_a_test_id"; // Call enroll PluginRequest request = { {"enroll", "1"}, // 0 enroll if needed, 1 force re-enroll }; PluginResponse resp; Status stat = Registry::call("enrollment", "get_key", request, resp); // The enrollment server test mostly stresses workflow and code coverage. // Occasionally, like with the transports testing, the non-mocked netlib // server failed to bind. if (stat.ok()) { // Verify get key contains the string if (resp.size() == 1) { EXPECT_EQ(resp[0]["key"], "potatoes"); } else { EXPECT_EQ(resp.size(), 1); } } }
Status DatabasePlugin::call(const PluginRequest& request, PluginResponse& response) { if (request.count("action") == 0) { return Status(1, "Database plugin must include a request action"); } // Get a domain/key, which are used for most database plugin actions. auto domain = (request.count("domain") > 0) ? request.at("domain") : ""; auto key = (request.count("key") > 0) ? request.at("key") : ""; // Switch over the possible database plugin actions. if (request.at("action") == "get") { std::string value; auto status = this->get(domain, key, value); response.push_back({{"v", value}}); return status; } else if (request.at("action") == "put") { if (request.count("value") == 0) { return Status(1, "Database plugin put action requires a value"); } return this->put(domain, key, request.at("value")); } else if (request.at("action") == "remove") { return this->remove(domain, key); } else if (request.at("action") == "scan") { std::vector<std::string> keys; auto status = this->scan(domain, keys); for (const auto& key : keys) { response.push_back({{"k", key}}); } return status; } return Status(1, "Unknown database plugin action"); }
Status TablePlugin::call(const PluginRequest& request, PluginResponse& response) { response.clear(); // TablePlugin API calling requires an action. if (request.count("action") == 0) { return Status(1, "Table plugins must include a request action"); } if (request.at("action") == "generate") { // "generate" runs the table implementation using a PluginRequest with // optional serialized QueryContext and returns the QueryData results as // the PluginRequest data. QueryContext context; if (request.count("context") > 0) { setContextFromRequest(request, context); } response = generate(context); } else if (request.at("action") == "columns") { // "columns" returns a PluginRequest filled with column information // such as name and type. const auto& column_list = columns(); for (const auto& column : column_list) { response.push_back( {{"name", column.first}, {"type", columnTypeName(column.second)}}); } } else if (request.at("action") == "definition") { response.push_back({{"definition", columnDefinition()}}); } else { return Status(1, "Unknown table plugin action: " + request.at("action")); } return Status(0, "OK"); }
PluginResponse TablePlugin::routeInfo() const { // Route info consists of only the serialized column information. PluginResponse response; for (const auto& column : columns()) { response.push_back({{"name", column.first}, {"type", column.second}}); } return response; }
Status Config::genConfig() { PluginResponse response; auto status = Registry::call("config", {{"action", "genConfig"}}, response); if (!status.ok()) { return status; } if (response.size() > 0) { return update(response[0]); } return Status(0, "OK"); }
Status runEnrollment(bool force = false) { PluginResponse response; PluginRequest request = {{"enroll", (force) ? "1" : "0"}}; auto status = Registry::call("enrollment", "get_key", request, response); if (!status.ok()) { return status; } if (response.size() > 0 && response[0]["key"].size() == 0) { return Status(1, "Enrollment Error: No Key"); } return Status(0, "OK"); }
Status SpecialWidget::call(const PluginRequest& request, PluginResponse& response) { response.push_back(request); response[0]["from"] = name_; response[0]["secret_power"] = secretPower(request); return Status(0, "OK"); }
void Plugin::setResponse(const std::string& key, const boost::property_tree::ptree& tree, PluginResponse& response) { std::ostringstream output; boost::property_tree::write_json(output, tree, false); response.push_back({{key, output.str()}}); }
Status getDatabaseValue(const std::string& domain, const std::string& key, std::string& value) { PluginRequest request = {{"action", "get"}, {"domain", domain}, {"key", key}}; PluginResponse response; auto status = Registry::call("database", "rocks", request, response); if (!status.ok()) { return status; } // Set value from the internally-known "v" key. if (response.size() > 0 && response[0].count("v") > 0) { value = response[0].at("v"); } return status; }
Status callExtension(const std::string& extension_path, const std::string& registry, const std::string& item, const PluginRequest& request, PluginResponse& response) { // Make sure the extension manager path exists, and is writable. auto status = extensionPathActive(extension_path); if (!status.ok()) { return status; } ExtensionResponse ext_response; try { auto client = EXClient(extension_path); client.get()->call(ext_response, registry, item, request); } catch (const std::exception& e) { return Status(1, "Extension call failed: " + std::string(e.what())); } // Convert from Thrift-internal list type to PluginResponse type. if (ext_response.status.code == ExtensionCode::EXT_SUCCESS) { for (const auto& item : ext_response.response) { response.push_back(item); } } return Status(ext_response.status.code, ext_response.status.message); }
int xCreate(sqlite3 *db, void *pAux, int argc, const char *const *argv, sqlite3_vtab **ppVtab, char **pzErr) { auto *pVtab = new VirtualTable; if (!pVtab || argc == 0 || argv[0] == nullptr) { delete pVtab; return SQLITE_NOMEM; } memset(pVtab, 0, sizeof(VirtualTable)); pVtab->content = new VirtualTableContent; pVtab->instance = (SQLiteDBInstance *)pAux; // Create a TablePlugin Registry call, expect column details as the response. PluginResponse response; pVtab->content->name = std::string(argv[0]); // Get the table column information. auto status = Registry::call( "table", pVtab->content->name, {{"action", "columns"}}, response); if (!status.ok() || response.size() == 0) { delete pVtab->content; delete pVtab; return SQLITE_ERROR; } // Generate an SQL create table statement from the retrieved column details. auto statement = "CREATE TABLE " + pVtab->content->name + columnDefinition(response); int rc = sqlite3_declare_vtab(db, statement.c_str()); if (rc != SQLITE_OK || !status.ok() || response.size() == 0) { delete pVtab->content; delete pVtab; return (rc != SQLITE_OK) ? rc : SQLITE_ERROR; } // Keep a local copy of the column details in the VirtualTableContent struct. // This allows introspection into the column type without additional calls. for (const auto &column : response) { pVtab->content->columns.push_back( std::make_pair(column.at("name"), columnTypeName(column.at("type")))); } *ppVtab = (sqlite3_vtab *)pVtab; return rc; }
Status TablePlugin::addExternal(const std::string& name, const PluginResponse& response) { // Attach the table. if (response.size() == 0) { // Invalid table route info. return Status(1, "Invalid route info"); } // Use the SQL registry to attach the name/definition. return Registry::call("sql", "sql", {{"action", "attach"}, {"table", name}}); }
void Plugin::setResponse(const std::string& key, const boost::property_tree::ptree& tree, PluginResponse& response) { std::ostringstream output; try { boost::property_tree::write_json(output, tree, false); } catch (const pt::json_parser::json_parser_error& e) { // The plugin response could not be serialized. } response.push_back({{key, output.str()}}); }
Status DatabasePlugin::call(const PluginRequest& request, PluginResponse& response) { if (request.count("action") == 0) { return Status(1, "Database plugin must include a request action"); } // Get a domain/key, which are used for most database plugin actions. auto domain = (request.count("domain") > 0) ? request.at("domain") : ""; auto key = (request.count("key") > 0) ? request.at("key") : ""; // Switch over the possible database plugin actions. if (request.at("action") == "get") { std::string value; auto status = this->get(domain, key, value); response.push_back({{"v", value}}); return status; } else if (request.at("action") == "put") { if (request.count("value") == 0) { return Status(1, "Database plugin put action requires a value"); } return this->put(domain, key, request.at("value")); } else if (request.at("action") == "remove") { return this->remove(domain, key); } else if (request.at("action") == "scan") { // Accumulate scanned keys into a vector. std::vector<std::string> keys; // Optionally allow the caller to request a max number of keys. size_t max = 0; if (request.count("max") > 0) { max = std::stoul(request.at("max")); } auto status = this->scan(domain, keys, max); for (const auto& key : keys) { response.push_back({{"k", key}}); } return status; } return Status(1, "Unknown database plugin action"); }
Status getDatabaseValue(const std::string& domain, const std::string& key, std::string& value) { if (RegistryFactory::get().external()) { // External registries (extensions) do not have databases active. // It is not possible to use an extension-based database. PluginRequest request = { {"action", "get"}, {"domain", domain}, {"key", key}}; PluginResponse response; auto status = Registry::call("database", request, response); if (status.ok()) { // Set value from the internally-known "v" key. if (response.size() > 0 && response[0].count("v") > 0) { value = response[0].at("v"); } } return status; } else { auto plugin = getDatabasePlugin(); return plugin->get(domain, key, value); } }
Status EnrollPlugin::call(const PluginRequest& request, PluginResponse& response) { if (FLAGS_disable_enrollment) { return Status(0, "Enrollment disabled"); } // Only support the 'enroll' action. if (request.count("action") == 0 || request.at("action") != "enroll") { return Status(1, "Enroll plugins require an action"); } // The 'enroll' API should return a string and implement caching. auto node_key = this->enroll(); response.push_back({{"node_key", node_key}}); if (node_key.size() == 0) { return Status(1, "No enrollment key found/retrieved"); } else { return Status(0, "OK"); } }
Status Config::updateSource(const std::string& name, const std::string& json) { // Compute a 'synthesized' hash using the content before it is parsed. hashSource(name, json); // Remove all packs from this source. schedule_->removeAll(name); // Remove all files from this source. removeFiles(name); // load the config (source.second) into a pt::ptree pt::ptree tree; try { auto clone = json; stripConfigComments(clone); std::stringstream json_stream; json_stream << clone; pt::read_json(json_stream, tree); } catch (const pt::json_parser::json_parser_error& e) { return Status(1, "Error parsing the config JSON"); } // extract the "schedule" key and store it as the main pack if (tree.count("schedule") > 0 && !Registry::external()) { auto& schedule = tree.get_child("schedule"); pt::ptree main_pack; main_pack.add_child("queries", schedule); addPack("main", name, main_pack); } if (tree.count("scheduledQueries") > 0 && !Registry::external()) { auto& scheduled_queries = tree.get_child("scheduledQueries"); pt::ptree queries; for (const std::pair<std::string, pt::ptree>& query : scheduled_queries) { auto query_name = query.second.get<std::string>("name", ""); if (query_name.empty()) { return Status(1, "Error getting name from legacy scheduled query"); } queries.add_child(query_name, query.second); } pt::ptree legacy_pack; legacy_pack.add_child("queries", queries); addPack("legacy_main", name, legacy_pack); } // extract the "packs" key into additional pack objects if (tree.count("packs") > 0 && !Registry::external()) { auto& packs = tree.get_child("packs"); for (const auto& pack : packs) { auto value = packs.get<std::string>(pack.first, ""); if (value.empty()) { // The pack is a JSON object, treat the content as pack data. addPack(pack.first, name, pack.second); } else { // If the pack value is a string (and not a JSON object) then it is a // resource to be handled by the config plugin. PluginResponse response; PluginRequest request = { {"action", "genPack"}, {"name", pack.first}, {"value", value}}; Registry::call("config", request, response); if (response.size() == 0 || response[0].count(pack.first) == 0) { continue; } try { pt::ptree pack_tree; std::stringstream pack_stream; pack_stream << response[0][pack.first]; pt::read_json(pack_stream, pack_tree); addPack(pack.first, name, pack_tree); } catch (const pt::json_parser::json_parser_error& e) { LOG(WARNING) << "Error parsing the pack JSON: " << pack.first; } } } } applyParsers(name, tree, false); return Status(0, "OK"); }
Status Config::updateSource(const std::string& name, const std::string& json) { // Compute a 'synthesized' hash using the content before it is parsed. hashSource(name, json); // load the config (source.second) into a pt::ptree pt::ptree tree; try { auto clone = json; stripConfigComments(clone); std::stringstream json_stream; json_stream << clone; pt::read_json(json_stream, tree); } catch (const pt::json_parser::json_parser_error& e) { return Status(1, "Error parsing the config JSON"); } // extract the "schedule" key and store it as the main pack if (tree.count("schedule") > 0 && !Registry::external()) { auto& schedule = tree.get_child("schedule"); pt::ptree main_pack; main_pack.add_child("queries", schedule); addPack("main", name, main_pack); } if (tree.count("scheduledQueries") > 0 && !Registry::external()) { auto& scheduled_queries = tree.get_child("scheduledQueries"); pt::ptree queries; for (const std::pair<std::string, pt::ptree>& query : scheduled_queries) { auto query_name = query.second.get<std::string>("name", ""); if (query_name.empty()) { return Status(1, "Error getting name from legacy scheduled query"); } queries.add_child(query_name, query.second); } pt::ptree legacy_pack; legacy_pack.add_child("queries", queries); addPack("legacy_main", name, legacy_pack); } // extract the "packs" key into additional pack objects if (tree.count("packs") > 0 && !Registry::external()) { auto& packs = tree.get_child("packs"); for (const auto& pack : packs) { auto value = packs.get<std::string>(pack.first, ""); if (value.empty()) { addPack(pack.first, name, pack.second); } else { PluginResponse response; PluginRequest request = { {"action", "genPack"}, {"name", pack.first}, {"value", value}}; Registry::call("config", request, response); if (response.size() == 0 || response[0].count(pack.first) == 0) { continue; } try { pt::ptree pack_tree; std::stringstream pack_stream; pack_stream << response[0][pack.first]; pt::read_json(pack_stream, pack_tree); addPack(pack.first, name, pack_tree); } catch (const pt::json_parser::json_parser_error& e) { LOG(WARNING) << "Error parsing the pack JSON: " << pack.first; } } } } // Iterate each parser. for (const auto& plugin : Registry::all("config_parser")) { std::shared_ptr<ConfigParserPlugin> parser = nullptr; try { parser = std::dynamic_pointer_cast<ConfigParserPlugin>(plugin.second); } catch (const std::bad_cast& e) { LOG(ERROR) << "Error casting config parser plugin: " << plugin.first; } if (parser == nullptr || parser.get() == nullptr) { continue; } // For each key requested by the parser, add a property tree reference. std::map<std::string, pt::ptree> parser_config; for (const auto& key : parser->keys()) { if (tree.count(key) > 0) { parser_config[key] = tree.get_child(key); } else { parser_config[key] = pt::ptree(); } } // The config parser plugin will receive a copy of each property tree for // each top-level-config key. The parser may choose to update the config's // internal state parser->update(name, parser_config); } return Status(0, "OK"); }
TEST_F(ExtensionsTest, test_extension_broadcast) { auto status = startExtensionManager(socket_path); EXPECT_TRUE(status.ok()); EXPECT_TRUE(socketExists(socket_path)); // This time we're going to add a plugin to the extension_test registry. Registry::add<TestExtensionPlugin>("extension_test", "test_item"); // Now we create a registry alias that will be broadcasted but NOT used for // internal call lookups. Aliasing was introduced for testing such that an // EM/E could exist in the same process (the same registry) without having // duplicate registry items in the internal registry list AND extension // registry route table. Registry::addAlias("extension_test", "test_item", "test_alias"); Registry::allowDuplicates(true); // Before registering the extension there is NO route to "test_alias" since // alias resolutions are performed by the EM. EXPECT_TRUE(Registry::exists("extension_test", "test_item")); EXPECT_FALSE(Registry::exists("extension_test", "test_alias")); status = startExtension(socket_path, "test", "0.1", "0.0.0", "0.0.1"); EXPECT_TRUE(status.ok()); RouteUUID uuid; try { uuid = (RouteUUID)stoi(status.getMessage(), nullptr, 0); } catch (const std::exception& e) { EXPECT_TRUE(false); return; } auto ext_socket = socket_path + "." + std::to_string(uuid); EXPECT_TRUE(socketExists(ext_socket)); // Make sure the EM registered the extension (called in start extension). auto extensions = registeredExtensions(); // Expect two, since `getExtensions` includes the core. ASSERT_EQ(extensions.size(), 2U); EXPECT_EQ(extensions.count(uuid), 1U); EXPECT_EQ(extensions.at(uuid).name, "test"); EXPECT_EQ(extensions.at(uuid).version, "0.1"); EXPECT_EQ(extensions.at(uuid).sdk_version, "0.0.1"); // We are broadcasting to our own registry in the test, which internally has // a "test_item" aliased to "test_alias", "test_item" is internally callable // but "test_alias" can only be resolved by an EM call. EXPECT_TRUE(Registry::exists("extension_test", "test_item")); // Now "test_alias" exists since it is in the extensions route table. EXPECT_TRUE(Registry::exists("extension_test", "test_alias")); PluginResponse response; // This registry call will fail, since "test_alias" cannot be resolved using // a local registry call. status = Registry::call("extension_test", "test_alias", {{}}, response); EXPECT_FALSE(status.ok()); // The following will be the result of a: // Registry::call("extension_test", "test_alias", {{}}, response); status = callExtension(ext_socket, "extension_test", "test_alias", {{"test_key", "test_value"}}, response); EXPECT_TRUE(status.ok()); EXPECT_EQ(response.size(), 1U); EXPECT_EQ(response[0]["test_key"], "test_value"); Registry::removeBroadcast(uuid); Registry::allowDuplicates(false); }
Status call(const PluginRequest& request, PluginResponse& response) { for (const auto& request_item : request) { response.push_back({{request_item.first, request_item.second}}); } return Status(0, "Test success"); }
Status DatabasePlugin::call(const PluginRequest& request, PluginResponse& response) { if (request.count("action") == 0) { return Status(1, "Database plugin must include a request action"); } // Get a domain/key, which are used for most database plugin actions. auto domain = (request.count("domain") > 0) ? request.at("domain") : ""; auto key = (request.count("key") > 0) ? request.at("key") : ""; if (request.at("action") == "reset") { WriteLock lock(kDatabaseReset); DatabasePlugin::kDBInitialized = false; // Prevent RocksDB reentrancy by logger plugins during plugin setup. VLOG(1) << "Resetting the database plugin: " << getName(); auto status = this->reset(); if (!status.ok()) { // The active database could not be reset, fallback to an ephemeral. Registry::get().setActive("database", "ephemeral"); LOG(WARNING) << "Unable to reset database plugin: " << getName(); } DatabasePlugin::kDBInitialized = true; return status; } // Switch over the possible database plugin actions. ReadLock lock(kDatabaseReset); if (request.at("action") == "get") { std::string value; auto status = this->get(domain, key, value); response.push_back({{"v", value}}); return status; } else if (request.at("action") == "put") { if (request.count("value") == 0) { return Status(1, "Database plugin put action requires a value"); } return this->put(domain, key, request.at("value")); } else if (request.at("action") == "remove") { return this->remove(domain, key); } else if (request.at("action") == "remove_range") { auto key_high = (request.count("high") > 0) ? request.at("key_high") : ""; if (!key_high.empty() && !key.empty()) { return this->removeRange(domain, key, key_high); } return Status(1, "Missing range"); } else if (request.at("action") == "scan") { // Accumulate scanned keys into a vector. std::vector<std::string> keys; // Optionally allow the caller to request a max number of keys. size_t max = 0; if (request.count("max") > 0) { max = std::stoul(request.at("max")); } auto status = this->scan(domain, keys, request.at("prefix"), max); for (const auto& k : keys) { response.push_back({{"k", k}}); } return status; } return Status(1, "Unknown database plugin action"); }
/// The route information will usually be provided by the plugin type. /// The plugin/registry item will set some structures for the plugin /// to parse and format. BUT a plugin/registry item can also fill this /// information in if the plugin type/registry type exposes routeInfo as /// a virtual method. PluginResponse routeInfo() const { PluginResponse info; info.push_back({{"name", name_}}); return info; }