void QContentHubServer::pop_queue_nowait(msgpack::rpc::request &req, const std::string &name)
{
std::string ret;
queue_map_t &qmap = q_map.unsafe_ref();
queue_map_it_t it = qmap.find(name);
if (it == qmap.end()) {
req.result(QCONTENTHUB_STRERROR);
} else {
queue_t *q = it->second;
if (q->stop) {
req.result(QCONTENTHUB_STRAGAIN);
return;
}
pthread_mutex_lock(&(q->lock));
if (q->str_q.size() == 0) {
ret = QCONTENTHUB_STRAGAIN;
} else {
ret = q->str_q.front();
q->str_q.pop();
pthread_cond_signal(&q->not_full);
}
pthread_mutex_unlock(&(q->lock));
req.result(ret);
}
}
void WorkerServer::dispatch(msgpack::rpc::request req)
{
try
{
std::string method;
req.method().convert(&method);
WorkerHandlerBase* handler = router_.find(method);
if (handler)
{
//handler->invoke(req);
worker_pool_.schedule_task(boost::bind(handler_wrapper, handler, req));
}
else
{
std::cerr << "[WorkerServer] Method not found: " << method << std::endl;
req.error(msgpack::rpc::NO_METHOD_ERROR);
}
}
catch (const msgpack::type_error& e)
{
std::cerr << "[WorkerServer] " << e.what() << std::endl;
req.error(msgpack::rpc::ARGUMENT_ERROR);
}
catch (const std::exception& e)
{
std::cerr << "[WorkerServer] " << e.what() << std::endl;
req.error(std::string(e.what()));
}
}
void QContentHubServer::push_queue_nowait(msgpack::rpc::request &req, const std::string &name, const std::string &obj)
{
queue_map_t &qmap = q_map.unsafe_ref();
queue_map_it_t it = qmap.find(name);
if (it == qmap.end()) {
int ret = add_queue(name, DEFAULT_QUEUE_CAPACITY);
if (ret == QCONTENTHUB_ERROR) {
req.result(ret);
} else {
push_queue(req, name, obj);
}
} else {
queue_t *q = it->second;
pthread_mutex_lock(&q->lock);
if ((int)q->str_q.size() > q->capacity) {
pthread_mutex_unlock(&q->lock);
req.result(QCONTENTHUB_AGAIN);
} else {
q->str_q.push(obj);
pthread_cond_signal(&q->not_empty);
pthread_mutex_unlock(&q->lock);
req.result(QCONTENTHUB_OK);
}
}
}
void QContentHubServer::stat_queue(msgpack::rpc::request &req, const std::string &name)
{
char buf[64];
std::string ret;
int current = get_current_time();
ret.append("STAT uptime ");
sprintf(buf, "%d", current - m_start_time);
ret.append(buf);
ret.append("\n");
ret.append("STAT time ");
sprintf(buf, "%d", current);
ret.append(buf);
ret.append("\n");
queue_map_t &qmap = q_map.unsafe_ref();
queue_map_it_t it = qmap.find(name);
if (it == qmap.end()) {
req.result(ret);
} else {
ret.append("STAT name ");
ret.append(name);
ret.append("\n");
ret.append("STAT size ");
sprintf(buf, "%ld", it->second->str_q.size());
ret.append(buf);
ret.append("\n");
req.result(ret);
}
}
void QUrlQueueServer::start_dump_all(msgpack::rpc::request &req)
{
if (m_dump_all_dumping) {
req.result(QCONTENTHUB_ERROR);
} else {
m_dump_all_dumping = true;
m_dump_all_it = m_site_map.unsafe_ref().begin();
req.result(QCONTENTHUB_OK);
}
}
void QContentHubServer::set_queue_capacity(msgpack::rpc::request &req, const std::string &name, int capacity)
{
queue_map_t &qmap = q_map.unsafe_ref();
queue_map_it_t it = qmap.find(name);
if (it == qmap.end()) {
req.result(QCONTENTHUB_WARN);
} else {
queue_t *q = it->second;
pthread_mutex_lock(&q->lock);
q->capacity = capacity;
pthread_mutex_unlock(&q->lock);
}
req.result(QCONTENTHUB_OK);
}
void QContentHubServer::stats(msgpack::rpc::request &req)
{
char buf[64];
std::string ret;
int current = get_current_time();
ret.append("STAT uptime ");
sprintf(buf, "%d", current - m_start_time);
ret.append(buf);
ret.append("\n");
ret.append("STAT time ");
sprintf(buf, "%d", current);
ret.append(buf);
ret.append("\n");
queue_map_t &qmap = q_map.unsafe_ref();
for (queue_map_it_t it = qmap.begin(); it != qmap.end(); it++) {
ret.append("STAT name ");
ret.append(it->first);
ret.append("\n");
ret.append("STAT size ");
sprintf(buf, "%ld", it->second->str_q.size());
ret.append(buf);
ret.append("\n");
}
req.result(ret);
}
void QUrlQueueServer::dump_all(msgpack::rpc::request &req)
{
std::string content;
if (!m_dump_all_dumping) {
content = QCONTENTHUB_STRERROR;
} else {
mp::sync<SiteMap>::ref ref(m_site_map);
while (m_dump_all_it != ref->end()) {
Site *s = m_dump_all_it->second;
while (!s->url_queue.empty()) {
content = s->url_queue.top().value;
s->url_queue.pop();
return;
}
m_dump_all_it++;
}
if (m_dump_all_it == ref->end()) {
content = QCONTENTHUB_STREND;
m_dump_all_dumping = false;
}
}
req.result(content);
}
void QUrlQueueServer::stats(msgpack::rpc::request &req)
{
char buf[64];
std::string ret;
uint64_t current_time = get_current_time();
uint64_t current = current_time / 1000;
ret.append("STAT uptime ");
sprintf(buf, "%ld", current - m_start_time);
ret.append(buf);
ret.append("\nSTAT time ");
sprintf(buf, "%ld", current);
ret.append(buf);
ret.append("\nSTAT default_interval ");
sprintf(buf, "%d", m_default_interval);
ret.append(buf);
ret.append("\nSTAT stop_all ");
sprintf(buf, "%d", m_stop_all);
ret.append(buf);
ret.append("\nSTAT site_items ");
sprintf(buf, "%ld", m_site_map.unsafe_ref().size());
ret.append(buf);
int ordered_site_size = 0;
for (int i = 0; i < MAX_PRIORITY; i++) {
sprintf(buf, "\nSTAT ordered_site_%d items ", i);
ret.append(buf);
sprintf(buf, "%ld", site_queues[i].size());
ret.append(buf);
ordered_site_size += site_queues[i].size();
}
ret.append("\nSTAT ordered_site_total items ");
sprintf(buf, "%d", ordered_site_size);
ret.append(buf);
ret.append("\nSTAT enqueue_items ");
sprintf(buf, "%ld", m_enqueue_items);
ret.append(buf);
ret.append("\nSTAT dequeue_items ");
sprintf(buf, "%ld", m_dequeue_items);
ret.append(buf);
ret.append("\nSTAT clear_items ");
sprintf(buf, "%ld", m_clear_items);
ret.append(buf);
// TODO:
// STAT curr_connections 141
ret.append("\nEND\r\n");
req.result(ret);
}
void handler_wrapper(WorkerHandlerBase* handler, msgpack::rpc::request& req)
{
try
{
handler->invoke(req);
}
catch (const msgpack::type_error& e)
{
std::cerr << "[WorkerServer] " << e.what() << std::endl;
req.error(msgpack::rpc::ARGUMENT_ERROR);
}
catch (const std::exception& e)
{
std::cerr << "[WorkerServer] " << e.what() << std::endl;
req.error(std::string(e.what()));
}
}
void QUrlQueueServer::set_site_interval(msgpack::rpc::request &req, const std::string &site, int interval)
{
{
mp::sync<SiteMap>::ref ref(m_site_map);
m_site_interval_map[site] = interval;
}
req.result(QCONTENTHUB_OK);
}
void QUrlQueueServer::set_default_interval(msgpack::rpc::request &req, int interval)
{
{
mp::sync<SiteMap>::ref ref(m_site_map);
m_default_interval = interval;
}
req.result(QCONTENTHUB_OK);
}
void QUrlQueueServer::stop_site(msgpack::rpc::request &req, const std::string &site)
{
{
mp::sync<SiteMap>::ref ref(m_site_map);
SiteMapIter it = ref->find(site);
if (it != ref->end()) {
it->second->stop = true;
}
}
req.result(QCONTENTHUB_OK);
}
void server::dispatch(msgpack::rpc::request req)
try {
std::string method;
req.method().convert(&method);
const table_type* table((table_type*)TABLE);
table_type::const_iterator m = table->find(method);
if(m == table->end()) {
req.error(msgpack::rpc::NO_METHOD_ERROR);
return;
}
(*m->second)(this, &req);
} catch (msgpack::type_error& e) {
req.error(msgpack::rpc::ARGUMENT_ERROR);
return;
} catch (std::exception& e) {
req.error(std::string(e.what()));
return;
}
void QContentHubServer::push_queue(msgpack::rpc::request &req, const std::string &name, const std::string &obj)
{
queue_map_t &qmap = q_map.unsafe_ref();
queue_map_it_t it = qmap.find(name);
if (it == qmap.end()) {
int ret = add_queue(name, DEFAULT_QUEUE_CAPACITY);
if (ret == QCONTENTHUB_ERROR) {
req.result(ret);
} else {
push_queue(req, name, obj);
}
} else {
queue_t *q = it->second;
pthread_mutex_lock(&q->lock);
while ((int)q->str_q.size() > q->capacity) {
struct timespec ts;
clock_gettime(CLOCK_REALTIME, &ts);
ts.tv_sec += 60;
int rc = pthread_cond_timedwait(&q->not_full, &q->lock, &ts);
if (rc == ETIMEDOUT) {
pthread_mutex_unlock(&q->lock);
req.result(QCONTENTHUB_STRAGAIN);
return;
} else if (rc != 0) {
pthread_mutex_unlock(&q->lock);
req.result(QCONTENTHUB_STRERROR);
return;
}
}
q->str_q.push(obj);
pthread_cond_signal(&q->not_empty);
pthread_mutex_unlock(&q->lock);
req.result(QCONTENTHUB_OK);
}
}
void QContentHubServer::pop_queue(msgpack::rpc::request &req, const std::string &name)
{
queue_map_t &qmap = q_map.unsafe_ref();
queue_map_it_t it = qmap.find(name);
if (it == qmap.end()) {
req.result(QCONTENTHUB_STRAGAIN);
} else {
queue_t *q = it->second;
if (q->stop) {
req.result(QCONTENTHUB_STRAGAIN);
return;
}
pthread_mutex_lock(&(q->lock));
while (q->str_q.size() == 0) {
struct timespec ts;
clock_gettime(CLOCK_REALTIME, &ts);
ts.tv_sec += 60;
int rc = pthread_cond_timedwait(&q->not_empty, &q->lock, &ts);
if (rc == ETIMEDOUT) {
pthread_mutex_unlock(&(q->lock));
req.result(QCONTENTHUB_STRAGAIN);
return;
} else if (rc != 0) {
pthread_mutex_unlock(&(q->lock));
req.result(QCONTENTHUB_STRERROR);
return;
}
}
//assert(q->str_q.size() > 0);
std::string content = q->str_q.front();
q->str_q.pop();
pthread_cond_signal(&q->not_full);
pthread_mutex_unlock(&(q->lock));
req.result(content);
}
}
void QContentHubServer::clear_queue(msgpack::rpc::request &req, const std::string &name)
{
queue_map_t &qmap = q_map.unsafe_ref();
queue_map_it_t it = qmap.find(name);
if (it == qmap.end()) {
req.result(QCONTENTHUB_WARN);
} else {
queue_t *q = it->second;
pthread_mutex_lock(&q->lock);
while (!q->str_q.empty()) {
q->str_q.pop();
}
pthread_mutex_unlock(&q->lock);
}
}
// rpc_server
// Msgpack-RPC based server with 'hashed' dispatcher.
// rpc_server can add RPC method on-the-fly.
//
void rpc_server::dispatch(msgpack::rpc::request req) {
std::string method;
req.method().convert(&method);
func_map::iterator fun = funcs_.find(method);
if (fun == funcs_.end()) {
req.error(msgpack::rpc::NO_METHOD_ERROR, method);
return;
}
try {
fun->second->invoke(req);
} catch(const msgpack::type_error& e) {
req.error(msgpack::rpc::ARGUMENT_ERROR, std::string(e.what()));
} catch(const jubatus::core::common::exception::jubatus_exception& e) {
LOG(WARNING) << "exception in RPC thread: "
<< e.diagnostic_information(true);
req.error(std::string(e.what()));
} catch(const std::exception& e) {
LOG(ERROR) << "error in RPC thread: "
<< e.what();
req.error(std::string(e.what()));
}
}
void QUrlQueueServer::stat_site(msgpack::rpc::request &req, const std::string &site)
{
char buf[64];
std::string ret;
mp::sync<SiteMap>::ref ref(m_site_map);
ret.append("STAT site ");
ret.append(site);
SiteIntervalMapIter interval_it = m_site_interval_map.find(site);
ret.append("\nSTAT interval ");
if (interval_it == m_site_interval_map.end()) {
ret.append("default ");
sprintf(buf, "%d", m_default_interval);
ret.append(buf);
} else {
sprintf(buf, "%d", interval_it->second);
ret.append(buf);
}
SiteMapIter it = ref->find(site);
if (it != ref->end()) {
ret.append("\nSTAT stop ");
if (it->second->stop) {
ret.append("1");
} else {
ret.append("0");
}
ret.append("\nSTAT enqueue_items ");
sprintf(buf, "%ld", it->second->enqueue_items);
ret.append(buf);
ret.append("\nSTAT dequeue_items ");
sprintf(buf, "%ld", it->second->dequeue_items);
ret.append(buf);
ret.append("\nSTAT clear_items ");
sprintf(buf, "%ld", it->second->clear_items);
ret.append(buf);
ret.append("\nSTAT items_size ");
sprintf(buf, "%ld", it->second->url_queue.size());
ret.append(buf);
}
ret.append("\nEND\r\n");
req.result(ret);
}
void QUrlQueueServer::start_site(msgpack::rpc::request &req, const std::string &site)
{
{
mp::sync<SiteMap>::ref ref(m_site_map);
SiteMapIter it = ref->find(site);
if (it != ref->end()) {
Site * s = it->second;
it->second->stop = false;
s->ref_cnt++;
if (!s->url_queue.empty()) {
push_ordered_site(s->url_queue.top().priority, s);
}
}
}
req.result(QCONTENTHUB_OK);
}
void QUrlQueueServer::clear_site(msgpack::rpc::request &req, const std::string &site)
{
{
mp::sync<SiteMap>::ref ref(m_site_map);
SiteMapIter it = ref->find(site);
if (it != ref->end()) {
int size = it->second->url_queue.size();
it->second->clear_items += size;
m_clear_items += size;
SiteUrlQueue &url_queue = it->second->url_queue;
while (!url_queue.empty()) {
url_queue.pop();
}
}
}
req.result(QCONTENTHUB_OK);
}
void QUrlQueueServer::clear_all(msgpack::rpc::request &req)
{
{
mp::sync<SiteMap>::ref ref(m_site_map);
for (SiteMapIter it = ref->begin(); it != ref->end(); ++it) {
int size = it->second->url_queue.size();
it->second->clear_items += size;
m_clear_items += size;
// clear
SiteUrlQueue &url_queue = it->second->url_queue;
while (!url_queue.empty()) {
url_queue.pop();
}
}
}
req.result(QCONTENTHUB_OK);
}
void QUrlQueueServer::dump_ordered_site(msgpack::rpc::request &req, int limit)
{
int cnt = 0;
char buf[64];
std::string ret;
{
mp::sync<SiteMap>::ref ref(m_site_map);
for (SiteMapIter it = ref->begin(); it != ref->end(); ++it) {
Site * s = it->second;
if (!s->url_queue.empty()) {
ret.append("STAT site ");
ret.append(s->name);
ret.append("\nSTAT enqueue_items ");
sprintf(buf, "%ld", s->enqueue_items);
ret.append(buf);
ret.append("\nSTAT dequeue_items ");
sprintf(buf, "%ld", s->dequeue_items);
ret.append(buf);
ret.append("\nSTAT clear_items ");
sprintf(buf, "%ld", s->clear_items);
ret.append(buf);
ret.append("\nSTAT items_size ");
sprintf(buf, "%ld", s->url_queue.size());
ret.append(buf);
ret.append("\n\n");
++cnt;
}
if (cnt >= limit) {
break;
}
}
}
ret.append("\nEND\r\n");
req.result(ret);
}
void QUrlQueueServer::dump_site(msgpack::rpc::request &req, const std::string &site)
{
std::string content;
{
mp::sync<SiteMap>::ref ref(m_site_map);
SiteMapIter it = ref->find(site);
if (it == ref->end()) {
content = QCONTENTHUB_STREND;
} else {
Site *s = it->second;
if (!s->url_queue.empty()) {
content = s->url_queue.top().value;
s->url_queue.pop();
} else {
content = QCONTENTHUB_STREND;
}
}
}
req.result(content);
}
void AdStreamReceiveServer::dispatch(msgpack::rpc::request req)
{
try
{
std::string method;
req.method().convert(&method);
if (method == AdStreamReceiveServerRequest::method_names[AdStreamReceiveServerRequest::METHOD_TEST])
{
LOG(INFO) << "got heart check msg.";
//msgpack::type::tuple<bool> params;
//req.params().convert(¶ms);
req.result(true);
}
else if (method == AdStreamReceiveServerRequest::method_names[AdStreamReceiveServerRequest::METHOD_PUSH_ADMESSAGE])
{
//LOG(INFO) << "got pushed ad msg";
//LOG(INFO) << "data:" << req.params();
msgpack::type::tuple<AdMessageListData> params;
req.params().convert(¶ms);
AdMessageListData& data_list = params.get<0>();
AdStreamSubscriber::get()->onAdMessage(data_list.msg_list);
req.result(true);
}
else
{
req.error(msgpack::rpc::NO_METHOD_ERROR);
}
}
catch (const msgpack::type_error& e)
{
req.error(msgpack::rpc::ARGUMENT_ERROR);
LOG(WARNING) << "type error in rpc server." << e.what();
}
catch (const std::exception& e)
{
req.error(std::string(e.what()));
LOG(WARNING) << "exception in rpc server." << e.what();
}
}
void QUrlQueueServer::push_url(msgpack::rpc::request &req, int priority, const std::string &site, const std::string &record)
{
int ret = push_url(priority, site, record, false);
req.result(ret);
}
void QUrlQueueServer::dispatch(msgpack::rpc::request req)
{
try {
std::string method;
req.method().convert(&method);
if(method == "push") {
msgpack::type::tuple<int, std::string, std::string> params;
req.params().convert(¶ms);
push_url(req, params.get<0>(), params.get<1>(), params.get<2>());
} else if(method == "pop") {
pop_url(req);
} else if(method == "push_url_front") {
msgpack::type::tuple<int, std::string, std::string> params;
req.params().convert(¶ms);
push_url_front(req, params.get<0>(), params.get<1>(), params.get<2>());
} else if(method == "start_dump_all") {
start_dump_all(req);
} else if(method == "dump_all") {
dump_all(req);
} else if(method == "stats") {
stats(req);
} else if(method == "set_default_interval") {
msgpack::type::tuple<int> params;
req.params().convert(¶ms);
set_default_interval(req, params.get<0>());
} else if(method == "set_site_interval") {
msgpack::type::tuple<std::string, int> params;
req.params().convert(¶ms);
set_site_interval(req, params.get<0>(), params.get<1>());
} else if(method == "stat_site") {
msgpack::type::tuple<std::string> params;
req.params().convert(¶ms);
stat_site(req, params.get<0>());
} else if(method == "dump_ordered_site") {
msgpack::type::tuple<int> params;
req.params().convert(¶ms);
dump_ordered_site(req, params.get<0>());
} else if(method == "start_all") {
start_all(req);
} else if(method == "stop_all") {
stop_all(req);
} else if(method == "start_site") {
msgpack::type::tuple<std::string> params;
req.params().convert(¶ms);
start_site(req, params.get<0>());
} else if(method == "stop_site") {
msgpack::type::tuple<std::string> params;
req.params().convert(¶ms);
stop_site(req, params.get<0>());
} else if(method == "clear_site") {
msgpack::type::tuple<std::string> params;
req.params().convert(¶ms);
clear_site(req, params.get<0>());
} else if(method == "start_dump_site") {
msgpack::type::tuple<std::string> params;
req.params().convert(¶ms);
start_dump_site(req, params.get<0>());
} else if(method == "dump_site") {
msgpack::type::tuple<std::string> params;
req.params().convert(¶ms);
dump_site(req, params.get<0>());
} else if(method == "clear_empty_site") {
clear_empty_site(req);
} else {
req.error(msgpack::rpc::NO_METHOD_ERROR);
}
} catch (msgpack::type_error& e) {
req.error(msgpack::rpc::ARGUMENT_ERROR);
return;
} catch (std::exception& e) {
req.error(std::string(e.what()));
return;
}
}
void QContentHubServer::dispatch(msgpack::rpc::request req) {
try {
std::string method;
req.method().convert(&method);
if(method == "push") {
msgpack::type::tuple<std::string, std::string> params;
req.params().convert(¶ms);
push_queue(req, params.get<0>(), params.get<1>());
} else if(method == "pop") {
msgpack::type::tuple<std::string> params;
req.params().convert(¶ms);
pop_queue(req, params.get<0>());
} else if(method == "push_nowait") {
msgpack::type::tuple<std::string, std::string> params;
req.params().convert(¶ms);
push_queue_nowait(req, params.get<0>(), params.get<1>());
} else if(method == "pop_nowait") {
msgpack::type::tuple<std::string> params;
req.params().convert(¶ms);
pop_queue_nowait(req, params.get<0>());
} else if(method == "add") {
msgpack::type::tuple<std::string, int> params;
req.params().convert(¶ms);
add_queue(req, params.get<0>(), params.get<1>());
/*
} else if(method == "del") {
msgpack::type::tuple<std::string> params;
req.params().convert(¶ms);
del_queue(req, params.get<0>());
} else if(method == "fdel") {
msgpack::type::tuple<std::string> params;
req.params().convert(¶ms);
force_del_queue(req, params.get<0>());
*/
} else if(method == "set_capacity") {
msgpack::type::tuple<std::string, int> params;
req.params().convert(¶ms);
set_queue_capacity(req, params.get<0>(), params.get<1>());
} else if(method == "start") {
msgpack::type::tuple<std::string> params;
req.params().convert(¶ms);
start_queue(req, params.get<0>());
} else if(method == "stop") {
msgpack::type::tuple<std::string> params;
req.params().convert(¶ms);
stop_queue(req, params.get<0>());
} else if(method == "clear") {
msgpack::type::tuple<std::string> params;
req.params().convert(¶ms);
clear_queue(req, params.get<0>());
} else if(method == "stats") {
stats(req);
} else if(method == "stat_queue") {
msgpack::type::tuple<std::string> params;
req.params().convert(¶ms);
stat_queue(req, params.get<0>());
} else {
req.error(msgpack::rpc::NO_METHOD_ERROR);
}
} catch (msgpack::type_error& e) {
req.error(msgpack::rpc::ARGUMENT_ERROR);
return;
} catch (std::exception& e) {
req.error(std::string(e.what()));
return;
}
}
void QContentHubServer::add_queue(msgpack::rpc::request &req, const std::string &name, int capacity)
{
req.result(add_queue(name, capacity));
}
void QUrlQueueServer::push_url_front(msgpack::rpc::request &req, int /*priority*/, const std::string &site, const std::string &record)
{
int ret = push_url(MAX_PRIORITY, site, record, true);
req.result(ret);
}