void proxy_common::get_members_from_cht_(
const std::string& name,
const std::string& id,
std::vector<std::pair<std::string, int> >& ret,
size_t n) {
ret.clear();
jubatus::util::concurrent::scoped_lock lk(mutex_);
jubatus::server::common::cht ht(zk_, a_.type, name);
ht.find(id, ret, n);
if (ret.empty()) {
throw JUBATUS_EXCEPTION(no_worker(name));
}
}
passive_aggressive::passive_aggressive(
const config& config,
storage_ptr storage)
: linear_regression(storage),
config_(config),
sum_(0.f),
sq_sum_(0.f),
count_(0.f) {
if (!(0.f <= config.sensitivity)) {
throw JUBATUS_EXCEPTION(
common::invalid_parameter("0.0 <= sensitivity"));
}
}
lru_unlearner::lru_unlearner(const config& conf)
: max_size_(conf.max_size) {
if (conf.max_size <= 0) {
throw JUBATUS_EXCEPTION(
common::config_exception() << common::exception::error_message(
"max_size must be a positive integer"));
}
entry_map_.reserve(max_size_);
if (conf.sticky_pattern) {
key_matcher_factory f;
sticky_matcher_ = f.create_matcher(*conf.sticky_pattern);
}
}
lock_service_mutex::lock_service_mutex(
lock_service& ls,
const std::string& path)
: path_(path) {
if (ls.type() == "zk" || ls.type() == "cached_zk") {
impl_ = new zkmutex(ls, path);
} else {
{
LOG(ERROR) << "unknown lock_service: " << ls.type();
}
throw JUBATUS_EXCEPTION(jubatus::core::common::exception::runtime_error(
std::string("unknown lock_service: ") + ls.type()));
}
}
// TODO(kashihara): AF_INET does not specify IPv6
void get_ip(const char* nic, string& out) {
int fd;
struct ifreq ifr;
fd = socket(AF_INET, SOCK_DGRAM, 0);
if (fd == -1) {
throw JUBATUS_EXCEPTION(jubatus::core::common::exception::runtime_error(
"Failed to create socket(AF_INET, SOCK_DGRAM)")
<< jubatus::core::common::exception::error_errno(errno));
}
ifr.ifr_addr.sa_family = AF_INET;
std::strncpy(ifr.ifr_name, nic, IFNAMSIZ - 1);
if (ioctl(fd, SIOCGIFADDR, &ifr) == -1) {
throw JUBATUS_EXCEPTION(jubatus::core::common::exception::runtime_error(
"Failed to get IP address from interface")
<< jubatus::core::common::exception::error_errno(errno));
}
close(fd);
struct sockaddr_in* sin = (struct sockaddr_in*) (&(ifr.ifr_addr));
out = inet_ntoa((struct in_addr) (sin->sin_addr));
}
inline bool check_json_type(
json_config_iarchive_cast& js,
jubatus::util::text::json::json::json_type_t t) {
if (js.get().type() != t) {
type_error e(js.get_config().path(), t, js.get().type());
if (js.trace_error()) {
js.push_error(e);
} else {
throw JUBATUS_EXCEPTION(e);
}
return false;
}
return true;
}
void async_client::writable_callback(int fd, int events)
{
if (events == EV_WRITE) {
if (sock_->is_connecting())
sock_->set_online();
int r = sock_->send_async(send_buffer_->data(), send_buffer_->size());
if (r < 0) {
if (errno == EAGAIN || errno == EWOULDBLOCK)
return;
// write error
throw JUBATUS_EXCEPTION(rpc_io_error()
<< jubatus::exception::error_api_func("write")
<< jubatus::exception::error_errno(errno));
} else if (r == 0) {
// there is no effect
} else if (sock_->sent_size() == send_buffer_->size()){
// current packet sending finished
event_del(&context_->ev_write);
sock_->reset_progress();
}
} else {
// EV_TIMEOUT or error occured
if (events == EV_TIMEOUT)
throw JUBATUS_EXCEPTION(rpc_timeout_error()
<< jubatus::exception::error_api_func("write")
<< jubatus::exception::error_errno(errno));
else
throw JUBATUS_EXCEPTION(rpc_internal_error()
<< jubatus::exception::error_api_func("write")
<< jubatus::exception::error_errno(errno));
}
}
shared_ptr<string_feature> string_feature_factory::create(
const std::string& name,
const param_t& params) const {
string_feature* p;
if (name == "ngram") {
return create_character_ngram(params);
} else if (name == "regexp") {
return create_regexp(params);
} else if (ext_ && (p = ext_(name, params))) {
return shared_ptr<string_feature>(p);
} else {
throw JUBATUS_EXCEPTION(
converter_exception(std::string("unknown splitter name: ") + name));
}
}
uint64_t zk::create_id(const std::string& path, uint32_t prefix) {
scoped_lock lk(m_);
struct Stat st;
int rc = zoo_set2(zh_, path.c_str(), "dummy", 6, -1, &st);
if(rc != ZOK) {
LOG(ERROR) << path << " failed on zoo_set2 " << zerror(rc);
throw JUBATUS_EXCEPTION(jubatus::exception::runtime_error("failed create id")
<< jubatus::exception::error_message(std::string("zerror: ") + zerror(rc))
<< jubatus::exception::error_api_func("zoo_set2"));
}
uint64_t ret = prefix;
ret <<= 32;
return (ret | st.version);
};
void set_action_on_term(jubatus::util::lang::function<void()> action) {
jubatus::util::concurrent::scoped_lock lk(mutex_on_term);
if (!blocking_sigterm) {
throw JUBATUS_EXCEPTION(
core::common::exception::runtime_error(
"prepare_signal_handling must be called before set_action_on_term."));
}
if (!handling_sigterm) {
jubatus::util::concurrent::thread(&handle_sigterm).start();
handling_sigterm = true;
}
action_on_term.swap(action);
}
bool driver_base::mixable_holder::put_diff(const diff_object& obj) {
internal_diff_object* diff_obj =
dynamic_cast<internal_diff_object*>(obj.get());
if (!diff_obj) {
throw JUBATUS_EXCEPTION(
core::common::exception::runtime_error("bad diff_object"));
}
if (count_mixable<linear_mixable>(mixables_) != diff_obj->diffs_.size()) {
throw JUBATUS_EXCEPTION(
core::common::exception::runtime_error("diff size is wrong"));
}
bool success = true;
for (size_t i = 0; i < mixables_.size(); i++) {
linear_mixable* mixable = dynamic_cast<linear_mixable*>(mixables_[i]);
if (!mixable) {
continue;
}
success = mixable->put_diff(diff_obj->diffs_[i]) && success;
}
return success;
}
void driver_base::mixable_holder::push(const msgpack::object& o) {
if (o.type != msgpack::type::ARRAY ||
o.via.array.size != count_mixable<push_mixable>(mixables_)) {
throw JUBATUS_EXCEPTION(
core::common::exception::runtime_error("push failed"));
}
for (size_t i = 0, obj_index = 0; i < mixables_.size(); i++) {
push_mixable* mixable = dynamic_cast<push_mixable*>(mixables_[i]);
if (!mixable) {
continue;
}
mixable->push(o.via.array.ptr[obj_index]);
obj_index++;
}
}
inline bool check_json_float(json_config_iarchive_cast& js) {
if (js.get().type() != pfi::text::json::json::Float
&& js.get().type() != pfi::text::json::json::Integer) {
type_error e(
js.get_config().path(),
pfi::text::json::json::Float,
js.get().type());
if (js.trace_error()) {
js.push_error(e);
} else {
throw JUBATUS_EXCEPTION(e);
}
return false;
}
return true;
}
anomaly_base* anomaly_factory::create_anomaly(
const string& name,
const config& param) {
if (name == "lof") {
anomaly_config conf = config_cast_check<anomaly_config>(param);
storage::lof_storage::config config =
config_cast_check<storage::lof_storage::config>(param);
return new lof(
config,
recommender::recommender_factory::create_recommender(
conf.method,
conf.parameter));
} else {
throw JUBATUS_EXCEPTION(common::unsupported_method(name));
}
};
//@random
double graph_serv::centrality(const std::string& id, const centrality_type& s,
const preset_query& q) const
{
if(s == 0){
jubatus::graph::preset_query q0;
return g_.get_model()->centrality(n2i(id),
jubatus::graph::EIGENSCORE,
q0);
}else{
std::stringstream msg;
msg << "unknown centrality type: " << s;
LOG(ERROR) << msg.str();
throw JUBATUS_EXCEPTION(jubatus::exception::runtime_error(msg.str()));
}
}
string real_path(const string& relative_path) {
// Resolve the given relative path to absolute path.
string absolute_path;
char *buf = ::realpath(relative_path.c_str(), NULL);
if (buf == NULL) {
throw JUBATUS_EXCEPTION(
jubatus::core::common::exception::runtime_error(
"Failed to get realpath")
<< jubatus::core::common::exception::error_api_func("realpath")
<< jubatus::core::common::exception::error_file_name(relative_path)
<< jubatus::core::common::exception::error_errno(errno));
}
absolute_path = string(buf);
free(buf);
return absolute_path;
}
// register_node :: node -> bool;
// creates /jubatus/actors/<name>/cht/<hash(ip_port)> with contents ip_port
void cht::register_node(const std::string& ip, int port) {
std::string path;
build_actor_path(path, type_, name_);
path += "/cht";
for (unsigned int i = 0; i < NUM_VSERV; ++i) {
std::string hashpath = path + "/" + make_hash(build_loc_str(ip, port, i));
if (!lock_service_->create(hashpath, build_loc_str(ip, port), true)) {
throw JUBATUS_EXCEPTION(
core::common::exception::runtime_error("Failed to register cht node")
<< core::common::exception::error_api_func("lock_service::create")
<< core::common::exception::error_message("cht hashpash: " + hashpath));
}
DLOG(INFO) << "cht node created: " << hashpath;
}
}
void init_string_filter_rules(
const std::vector<filter_rule>& filter_rules,
const std::map<std::string, string_filter_ptr>& filters,
datum_to_fv_converter& conv) {
for (size_t i = 0; i < filter_rules.size(); ++i) {
const filter_rule& rule = filter_rules[i];
std::map<std::string, string_filter_ptr>::const_iterator it =
filters.find(rule.type);
if (it == filters.end()) {
throw JUBATUS_EXCEPTION(
converter_exception("unknown type: " + rule.type));
}
matcher_ptr m(create_key_matcher(rule.key, rule.except));
conv.register_string_filter(m, it->second, rule.suffix);
}
}
void init_num_rules(
const std::vector<num_rule>& num_rules,
const std::map<std::string, num_feature_ptr>& num_features,
datum_to_fv_converter& conv) {
for (size_t i = 0; i < num_rules.size(); ++i) {
const num_rule& rule = num_rules[i];
matcher_ptr m(create_key_matcher(rule.key, rule.except));
std::map<std::string, num_feature_ptr>::const_iterator it =
num_features.find(rule.type);
if (it == num_features.end()) {
throw JUBATUS_EXCEPTION(
converter_exception("unknown type: " + rule.type));
}
conv.register_num_rule(rule.type, m, it->second);
}
}
int64_t gmm::get_nearest_center_index(const eigen_svec_t& p) const {
if (means_.empty()) {
throw JUBATUS_EXCEPTION(common::exception::runtime_error(
"clustering is not performed yet"));
}
double max_prob = 0;
int64_t max_idx = 0;
eigen_svec_t cps = cluster_probs(p, means_, covs_, cov_solvers_);
for (int c = 0; c < k_; ++c) {
double cp = cps.coeff(c);
if (cp > max_prob) {
max_prob = cp;
max_idx = c;
}
}
return max_idx;
}
void euclid_lsh::update_row(const string& id, const sfv_diff_t& diff) {
if (unlearner_ && !unlearner_->can_touch(id)) {
throw JUBATUS_EXCEPTION(common::exception::runtime_error(
"cannot add new row as number of sticky rows reached "
"the maximum size of unlearner: " + id));
}
storage::lsh_index_storage& lsh_index = *mixable_storage_->get_model();
orig_.set_row(id, diff);
common::sfv_t row;
orig_.get_row(id, row);
const vector<float> hash = calculate_lsh(row);
const float norm = calc_norm(row);
lsh_index.set_row(id, hash, norm);
if (unlearner_) {
unlearner_->touch(id);
}
}
explicit server_helper(const server_argv& a, bool use_cht = false)
: impl_(a),
start_time_(get_clock_time()),
use_cht_(use_cht) {
impl_.prepare_for_start(a, use_cht);
server_.reset(new Server(a, impl_.zk()));
impl_.get_config_lock(a, 3);
try {
server_->set_config(get_conf(a));
} catch (const core::common::jsonconfig::cast_check_error& e) {
core::common::config_exception config_error;
const core::common::jsonconfig::config_error_list& errors = e.errors();
for (core::common::jsonconfig::config_error_list::const_iterator
it = errors.begin(), end = errors.end(); it != end; ++it) {
config_error << core::common::exception::error_message((*it)->what());
}
// send error message to caller
throw JUBATUS_EXCEPTION(config_error);
} catch (const jubatus::util::lang::parse_error& e) {
// exit immediately on JSON parse error with exit-code 1
std::string msg =
std::string("syntax error in configuration: ") +
(a.is_standalone() ?
a.configpath :
std::string("<zookeeper>")) + ":" +
jubatus::util::lang::lexical_cast<std::string>(e.lineno()) + ":" +
jubatus::util::lang::lexical_cast<std::string>(e.pos()) + " " +
e.msg();
LOG(ERROR) << msg;
exit(1);
}
try {
// standalone only, is it desirable?
if (a.is_standalone() && !a.modelpath.empty()) {
server_->load_file(a.modelpath);
}
} catch (const std::runtime_error& e) {
exit(1);
}
}
void init_string_rules(
const std::vector<string_rule>& string_rules,
const std::map<std::string, splitter_ptr>& splitters,
datum_to_fv_converter& conv) {
for (size_t i = 0; i < string_rules.size(); ++i) {
const string_rule& rule = string_rules[i];
matcher_ptr m(create_key_matcher(rule.key, rule.except));
std::map<std::string, splitter_ptr>::const_iterator it =
splitters.find(rule.type);
if (it == splitters.end()) {
throw JUBATUS_EXCEPTION(
converter_exception("unknown type: " + rule.type));
}
std::vector<splitter_weight_type> ws;
ws.push_back(make_weight_type(rule.sample_weight, rule.global_weight));
conv.register_string_rule(rule.type, m, it->second, ws);
}
}
shared_ptr<num_filter> num_filter_factory::create(
const std::string& name,
const param_t& params) const {
num_filter* p;
if (name == "add") {
return create_add_filter(params);
} else if (name == "linear_normalization") {
return create_linear_normalization_filter(params);
} else if (name == "gaussian_normalization") {
return create_gaussian_normalization_filter(params);
} else if (name == "sigmoid_normalization") {
return create_sigmoid_normalization_filter(params);
} else if (ext_ && (p = ext_(name, params))) {
return shared_ptr<num_filter>(p);
} else {
throw JUBATUS_EXCEPTION(
converter_exception("unknonw num filter name: " + name));
}
}
classifier_base* classifier_factory::create_classifier(const std::string& name, storage::storage_base* storage) {
if (name == "perceptron"){
return static_cast<classifier_base*>(new perceptron(storage));
} else if (name == "PA"){
return static_cast<classifier_base*>(new PA(storage));
} else if (name == "PA1"){
return static_cast<classifier_base*>(new PA1(storage));
} else if (name == "PA2"){
return static_cast<classifier_base*>(new PA2(storage));
} else if (name == "CW"){
return static_cast<classifier_base*>(new CW(storage));
} else if (name == "AROW"){
return static_cast<classifier_base*>(new AROW(storage));
} else if (name == "NHERD"){
return static_cast<classifier_base*>(new NHERD(storage));
} else {
throw JUBATUS_EXCEPTION(unsupported_method(name));
}
}
key_matcher* key_matcher_factory::create_matcher(const std::string& matcher) {
if (matcher == "" || matcher == "*") {
return new match_all();
} else if (matcher[0] == '*') {
return new suffix_match(matcher.substr(1));
} else if (matcher[matcher.size() - 1] == '*') {
return new prefix_match(matcher.substr(0, matcher.size() - 1));
} else if (matcher.size() >= 2 && matcher[0] == '/'
&& matcher[matcher.size() - 1] == '/') {
#ifdef HAVE_RE2
return new re2_match(matcher.substr(1, matcher.size() - 2));
#else
throw JUBATUS_EXCEPTION(
converter_exception("cannot use regexp rule: " + matcher));
#endif
} else {
return new exact_match(matcher);
}
}
dynamic_loader::dynamic_loader(const std::string& path)
: handle_(0) {
void* handle = NULL;
std::string loaded_path;
if (is_absolute_or_relative_path(path)) {
// If the path contains "/", load the plugin with the given path.
handle = ::dlopen(path.c_str(), RTLD_LAZY);
loaded_path = path;
} else {
// Try to load the plugin from the plugin path environment.
const char* plugin_dir = get_plugin_path();
if (plugin_dir) {
const std::string plugin_path =
std::string(plugin_dir) + "/" + path;
handle = ::dlopen(plugin_path.c_str(), RTLD_LAZY);
loaded_path = plugin_path;
}
// If failed, try to load it from the plugin directory specified on
// configure.
if (!handle) {
const std::string plugin_path =
std::string(JUBATUS_PLUGIN_DIR) + "/" + path;
handle = ::dlopen(plugin_path.c_str(), RTLD_LAZY);
loaded_path = plugin_path;
}
}
if (!handle) {
char* error = dlerror();
throw JUBATUS_EXCEPTION(
converter_exception(
"cannot load dynamic library: " + path + ": " + error)
<< jubatus::core::common::exception::error_api_func("dlopen")
<< jubatus::core::common::exception::error_file_name(path)
<< jubatus::core::common::exception::error_message(error));
}
handle_ = handle;
}
void cht::setup_cht_dir(
lock_service& ls,
const std::string& type,
const std::string& name) {
bool success = true;
std::string path;
build_actor_path(path, type, name);
success = ls.create(path) && success;
path += "/cht";
success = ls.create(path) && success;
if (!success) {
throw JUBATUS_EXCEPTION(
core::common::exception::runtime_error("Failed to create cht directory")
<< core::common::exception::error_api_func("lock_service::create")
<< core::common::exception::error_message("cht path: " + path));
}
}
diff_object driver_base::mixable_holder::convert_diff_object(
const msgpack::object& o) const {
if (o.type != msgpack::type::ARRAY ||
o.via.array.size != count_mixable<linear_mixable>(mixables_)) {
throw JUBATUS_EXCEPTION(
core::common::exception::runtime_error("conversion failed"));
}
vector<diff_object> diffs;
for (size_t i = 0; i < mixables_.size(); i++) {
const linear_mixable* mixable =
dynamic_cast<const linear_mixable*>(mixables_[i]);
if (!mixable) {
continue;
}
diffs.push_back(mixable->convert_diff_object(o.via.array.ptr[i]));
}
return diff_object(new internal_diff_object(diffs));
}
void zk::create(const std::string& path, const std::string& payload, bool ephemeral) {
scoped_lock lk(m_);
int rc = zoo_create(zh_, path.c_str(), payload.c_str(), payload.length(),
&ZOO_OPEN_ACL_UNSAFE,
((ephemeral)?ZOO_EPHEMERAL:0), // | ZOO_SEQUENCE
NULL, 0);
if(ephemeral) {
if(rc != ZOK) {
LOG(ERROR) << path << " failed in creation:" << zerror(rc);
throw JUBATUS_EXCEPTION(jubatus::exception::runtime_error("failed to create zk empheral node")
<< jubatus::exception::error_message(std::string("zerror: ") + zerror(rc))
<< jubatus::exception::error_api_func("zoo_create")
<< jubatus::exception::error_file_name(path));
}
} else {
if(rc != ZOK && rc != ZNODEEXISTS) {
LOG(ERROR) << path << " failed in creation " << rc << " " << zerror(rc);
}
}
};
