/* start clc main work socket */
int ly_epoll_work_start(int port)
{
if (g_efd == -1)
return -1;
int listener, ret = 0;
char portstr[10];
sprintf(portstr, "%d", port);
listener = lyutil_create_and_bind(portstr);
if (listener < 0) {
logerror(_("failed create/bind port %d\n"), port);
return -1;
}
ret = lyutil_make_socket_nonblocking(listener);
if (ret < 0) {
logerror(_("failed make port %d nonblocking\n"), port);
goto out;
}
ret = listen(listener, SOMAXCONN);
if (ret < 0) {
logerror(_("error in %s(%d)\n"), __func__, __LINE__);
goto out;
}
int id = ly_entity_new(listener);
if (id < 0) {
logerror(_("error in %s(%d)\n"), __func__, __LINE__);
goto out;
}
struct epoll_event ev;
ev.events = EPOLLIN;
ev.data.fd = id;
ret = epoll_ctl(g_efd, EPOLL_CTL_ADD, listener, &ev);
if (ret < 0) {
logerror(_("add socket to epoll error in %s.\n"), __func__);
ly_entity_release(id);
/* close(listener); closed in ly_entity_release */
return -1;
}
ly_entity_init(id, LY_ENTITY_CLC);
out:
if (ret != 0)
close(listener);
return ret;
}
int main(int argc, char *argv[])
{
int ret, keeppidfile = 1;
setlocale(LC_ALL, "");
bindtextdomain(PACKAGE, LOCALEDIR);
textdomain(PACKAGE);
lyxml_init();
lyauth_init();
/* start initializeing g_c */
CLCConfig *c = malloc(sizeof(CLCConfig));
if (c == NULL) {
printf(_("malloc for g_c have a error.\n"));
return -255;
}
g_c = c;
/* parse command line option and configuration file */
ret = clc_config(argc, argv, c);
if (ret == CLC_CONFIG_RET_HELP)
usage();
else if (ret == CLC_CONFIG_RET_VER)
printf(_("%s : Version %s\n"), PROGRAM_NAME, PROGRAM_VERSION);
else if (ret == CLC_CONFIG_RET_ERR_CMD)
printf(_
("command line parsing error, use -h option to display usage\n"));
else if (ret == CLC_CONFIG_RET_ERR_NOCONF) {
printf(_
("missing lyclc config file, default build-in settings are used.\n"));
ret = 0;
}
else if (ret == CLC_CONFIG_RET_ERR_ERRCONF)
printf(_("can not find %s.\n"), c->conf_path);
else if (ret == CLC_CONFIG_RET_ERR_CONF)
printf(_("reading config file %s returned error\n"), c->conf_path);
else if (ret == CLC_CONFIG_RET_ERR_UNKNOWN)
printf(_("internal error\n"));
/* exit if ret is not zero */
if (ret != 0)
goto out;
/* node cpu/mem factors */
if (c->node_cpu_factor == 0)
c->node_cpu_factor = DEFAULT_NODE_CPU_FACTOR;
if (c->node_mem_factor == 0)
c->node_mem_factor = DEFAULT_NODE_MEM_FACTOR;
/* for debuuging */
if (c->debug)
__print_config(c);
/* make sure data directory exists */
if (lyutil_create_dir(c->clc_data_dir)) {
printf(_("%s is not accessible\n"), c->clc_data_dir);
ret = -255;
goto out;
}
/* check whether program is started already */
ret = lyutil_check_pid_file(c->pid_path, PROGRAM_NAME);
if (ret == 1) {
printf(_("%s is running already.\n"), PROGRAM_NAME);
goto out;
}
else if (ret != 0) {
printf(_("error checking pid file.\n"));
goto out;
}
/* check DB */
if (ly_db_check() < 0) {
printf(_("failed connecting DB\n"));
ret = -255;
goto out;
}
/* get clc ip */
if (c->clc_ip == NULL && ly_clc_ip_get() < 0) {
logerror(_("CLC no proper network interface to use.\n"));
goto out;
}
/* Daemonize the progress */
if (c->daemon) {
if (c->debug == LYDEBUG)
printf(_("Run as daemon, log to %s.\n"), c->log_path);
lyutil_daemonize(__main_clean, keeppidfile);
logfile(c->log_path, c->debug ? LYDEBUG : c->verbose ? LYINFO : LYWARN);
}
else
logfile(NULL, c->debug ? LYDEBUG : c->verbose ? LYINFO : LYWARN);
/* create lock file */
ret = lyutil_create_pid_file(c->pid_path, PROGRAM_NAME);
if (ret == 1) {
logsimple(_("%s is running already.\n"), PROGRAM_NAME);
ret = 0;
goto out;
}
else if (ret != 0) {
logsimple(_("error creating pid file.\n"));
goto out;
}
keeppidfile = 0;
/* init db connection */
if (ly_db_init() < 0) {
logsimple(_("ly_db_init failed.\n"));
ret = -255;
goto out;
}
/* initialize entity store */
if (ly_entity_store_init() < 0) {
logsimple(_("ly_entity_init failed.\n"));
ret = -255;
goto out;
}
/* init job queue */
if (job_init() < 0 || job_internal_init() < 0) {
logsimple(_("job_init failed.\n"));
ret = -255;
goto out;
}
if (c->debug)
job_print_queue();
/* set up signal handler */
lyutil_signal_init();
/* handle specific signal */
struct sigaction sa;
sa.sa_flags = 0;
sigemptyset(&sa.sa_mask);
sa.sa_sigaction = __sig_handler;
if (sigaction(SIGTERM, &sa, NULL)) {
logsimple(_("Setting signal handler error.\n"));
ret = -255;
goto out;
}
/* initialize g_c->efd */
if (ly_epoll_init(EPOLL_EVENTS_MAX) != 0) {
logsimple(_("ly_epoll_init failed.\n"));
ret = -255;
goto out;
}
if (ly_epoll_work_start(g_c->clc_port) != 0) {
ret = -1;
logsimple(_("ly_epoll_init failed.\n"));
goto out;
}
/* init timeout values */
time_t mcast_join_time, job_dispatch_time, job_internal_time;
mcast_join_time = 0;
time(&job_dispatch_time);
job_internal_time = job_dispatch_time + (CLC_MCAST_JOIN_INTERVAL<<1);
job_dispatch_time = job_dispatch_time + (CLC_MCAST_JOIN_INTERVAL<<2);
/* start main event driven loop */
if (c->clc_ip)
loginfo(_("clc uses IP %s\n"), c->clc_ip);
else
loginfo(_("clc uses IP automatically detected\n"));
loginfo(_("start event loop, waiting for events ...\n"));
int i, n = 0;
struct epoll_event events[EPOLL_EVENTS_MAX];
while (1) {
time_t time_now;
time(&time_now);
/* send mcast request */
if (time_now - mcast_join_time > CLC_MCAST_JOIN_INTERVAL) {
if (ly_mcast_send_join() < 0)
logerror(_("failed sending mcast request.\n"));
mcast_join_time = time_now;
}
else if (time_now < mcast_join_time)
mcast_join_time = time_now;
/* job dispatch */
if (time_now - job_dispatch_time > CLC_JOB_DISPATCH_INTERVAL) {
if (job_dispatch() < 0)
logerror(_("job_dispatch failed.\n"));
job_dispatch_time = time_now;
}
else if (time_now < job_dispatch_time)
job_dispatch_time = time_now;
/* internal job dispatch */
if (time_now - job_internal_time > CLC_JOB_INTERNAL_INTERVAL) {
if (job_internal_dispatch() < 0)
logerror(_("job_internal failed.\n"));
job_internal_time = time_now;
}
else if (time_now < job_internal_time)
job_internal_time = time_now;
n = epoll_wait(g_efd, events, EPOLL_EVENTS_MAX,
CLC_EPOLL_TIMEOUT);
if (n != 0)
logdebug(_("waiting ... got %d events\n"), n);
for (i = 0; i < n; i++) {
int id = events[i].data.fd;
if (events[i].events & EPOLLIN) {
ret = ly_epoll_entity_recv(id);
if (ret < 0) {
logerror(_("epoll_data_recv error\n"));
}
else if (ret > 0) {
loginfo(_("release entity %d\n"), id);
ly_entity_release(id);
}
}
else if (events[i].events & EPOLLRDHUP) {
loginfo(_("epoll entity(%d) got rdhup. close.\n"), id);
ly_entity_release(id);
}
else if (events[i].events & EPOLLHUP) {
loginfo(_("epoll entity(%d) got hup. close.\n"), id);
ly_entity_release(id);
}
else {
logerror(_("unexpected event(%d, %d). ignore.\n"),
events[i].events, id);
}
}
}
out:
__main_clean(keeppidfile);
if (ret <= 0)
loginfo(_("%s exits\n"), PROGRAM_NAME);
return ret;
}
/* process node register request */
static int __node_xml_register(xmlDoc * doc, xmlNode * node, int ent_id)
{
if (ly_entity_is_registered(ent_id)) {
logwarn(_("received node register request again, ignored\n"));
return -1;
}
LYNodeData * nd = ly_entity_data(ent_id);
if (nd == NULL ) {
logerror(_("error in %s(%d)\n"), __func__, __LINE__);
return -1;
}
NodeInfo * nf = &nd->node;
/* Create xpath evaluation context */
xmlXPathContextPtr xpathCtx = xmlXPathNewContext(doc);
if (xpathCtx == NULL) {
logerror(_("unable to create new XPath context %s, %d\n"),
__func__, __LINE__);
return -1;
}
int ret = -1;
char *str;
str = xml_xpath_text_from_ctx(xpathCtx,
"/" LYXML_ROOT "/request/parameters/status");
if (str == NULL)
goto xml_err;
nf->status = atoi(str);
free(str);
str = xml_xpath_text_from_ctx(xpathCtx,
"/" LYXML_ROOT "/request/parameters/hypervisor");
if (str == NULL)
goto xml_err;
nf->hypervisor = atoi(str);
free(str);
str = xml_xpath_text_from_ctx(xpathCtx,
"/" LYXML_ROOT "/request/parameters/host/tag");
int tag = -1;
if (str) {
tag = atoi(str); /* NULL str is allowed for new node */
free(str);
}
str = xml_xpath_text_from_ctx(xpathCtx,
"/" LYXML_ROOT "/request/parameters/host/name");
if (str == NULL)
goto xml_err;
nf->host_name = str;
str = xml_xpath_text_from_ctx(xpathCtx,
"/" LYXML_ROOT "/request/parameters/host/ip");
if (str == NULL)
goto xml_err;
nf->host_ip = str;
str = xml_xpath_text_from_ctx(xpathCtx,
"/" LYXML_ROOT "/request/parameters/cpu/arch");
if (str == NULL)
goto xml_err;
nf->cpu_arch = atoi(str);
free(str);
str = xml_xpath_text_from_ctx(xpathCtx,
"/" LYXML_ROOT "/request/parameters/cpu/model");
if (str == NULL)
goto xml_err;
nf->cpu_model = str;
str = xml_xpath_text_from_ctx(xpathCtx,
"/" LYXML_ROOT "/request/parameters/cpu/mhz");
if (str == NULL)
goto xml_err;
nf->cpu_mhz = atoi(str);
free(str);
str = xml_xpath_text_from_ctx(xpathCtx,
"/" LYXML_ROOT "/request/parameters/cpu/max");
if (str == NULL)
goto xml_err;
nf->cpu_max = atoi(str);
nf->cpu_vlimit = NODE_SCHEDULE_CPU_LIMIT(nf->cpu_max);
free(str);
str = xml_xpath_text_from_ctx(xpathCtx,
"/" LYXML_ROOT "/request/parameters/cpu/commit");
if (str == NULL)
goto xml_err;
nf->cpu_commit = atoi(str);
free(str);
str = xml_xpath_text_from_ctx(xpathCtx,
"/" LYXML_ROOT "/request/parameters/memory/total");
if (str == NULL)
goto xml_err;
nf->mem_max = atoi(str);
nf->mem_vlimit = NODE_SCHEDULE_MEM_LIMIT(nf->mem_max);
free(str);
str = xml_xpath_text_from_ctx(xpathCtx,
"/" LYXML_ROOT "/request/parameters/memory/free");
if (str == NULL)
goto xml_err;
nf->mem_free = atoi(str);
free(str);
str = xml_xpath_text_from_ctx(xpathCtx,
"/" LYXML_ROOT "/request/parameters/memory/commit");
if (str == NULL)
goto xml_err;
nf->mem_commit = atoi(str);
free(str);
str = xml_xpath_text_from_ctx(xpathCtx,
"/" LYXML_ROOT "/request/parameters/storage/total");
if (str == NULL)
goto xml_err;
nf->storage_total = atoi(str);
free(str);
str = xml_xpath_text_from_ctx(xpathCtx,
"/" LYXML_ROOT "/request/parameters/storage/free");
if (str == NULL)
goto xml_err;
nf->storage_free = atoi(str);
free(str);
str = xml_xpath_text_from_ctx(xpathCtx,
"/" LYXML_ROOT "/request/parameters/load/average");
if (str == NULL)
goto xml_err;
nf->load_average = atoi(str);
free(str);
if (nf->status >= NODE_STATUS_REGISTERED) {
logwarn(_("node(%d, %s) tries to register from wrong status(%d)\n"),
nf->host_tag, nf->host_ip, nf->status);
ly_entity_release(ent_id);
ret = 0; /* no need to continue node registration */
goto done;
}
if (ly_entity_node_active(nf->host_ip)) {
logwarn(_("duplicate node ip received. something is wrong...\n"));
ly_entity_release(ent_id);
ret = 0; /* no need to continue node registration */
goto done;
}
if (tag > 0) {
/* check authentication result */
if (nf->host_tag > 0 && tag != nf->host_tag) {
logerror(_("node tag changed, %d -> %d. something is wrong.\n"),
nf->host_tag, tag);
goto done;
}
nf->host_tag = tag;
ret = __node_register_auth(nf, ent_id);
if (ret == LY_S_REGISTERING_DONE_SUCCESS) {
AuthConfig * ac = ly_entity_auth(ent_id);
if (db_node_update_secret(DB_NODE_FIND_BY_ID, &tag,
ac->secret) < 0 ||
db_node_update_status(DB_NODE_FIND_BY_ID, &tag,
NODE_STATUS_REGISTERED) < 0) {
logerror(_("error in %s(%d)\n"), __func__, __LINE__);
ret = -1;
goto done;
}
loginfo(_("node(tag:%d) registered\n"), tag);
ly_entity_update(ent_id, tag, LY_ENTITY_FLAG_STATUS_REGISTERED);
}
goto done;
}
/* new node */
DBNodeRegInfo db_nf;
bzero(&db_nf, sizeof(DBNodeRegInfo));
ret = db_node_find(DB_NODE_FIND_BY_IP, nf->host_ip, &db_nf);
if (ret < 0 || ret > 1) {
logerror(_("error in %s(%d)\n"), __func__, __LINE__);
ret = -1;
goto done;
}
if (ret == 0 || db_nf.secret) {
/* new node */
logdebug(_("new node\n"));
if (nf->status != NODE_STATUS_UNINITIALIZED) {
logwarn(_("node(%d, %s) tries to register from unexpected status(%d)\n"),
nf->host_tag, nf->host_ip, nf->status);
nf->status = NODE_STATUS_UNINITIALIZED;
}
if (db_nf.secret) {
logwarn(_("new node takes ip(%s) used by tagged node\n"), nf->host_ip);
db_node_reginfo_free(&db_nf);
bzero(&db_nf, sizeof(DBNodeRegInfo));
}
ret = db_node_insert(nf);
if (ret < 0) {
logerror(_("error in %s(%d)\n"), __func__, __LINE__);
goto new_done;
}
db_nf.id = ret;
loginfo(_("new node %s added in db(%d)\n"), nf->host_ip, ret);
/* enable node if node is control server */
if (ly_is_clc_ip(nf->host_ip)) {
if (db_node_enable(ret, 1) != 0 ||
db_node_find(DB_NODE_FIND_BY_ID, &ret, &db_nf) != 1) {
logerror(_("error in %s(%d)\n"), __func__, __LINE__);
ret = -1;
goto new_done;
}
}
}
else
logdebug(_("untagged node for ip(%s) found in db\n"), nf->host_ip);
ly_entity_update(ent_id, db_nf.id, LY_ENTITY_FLAG_STATUS_ONLINE);
if (db_nf.enabled) {
AuthConfig * ac = ly_entity_auth(ent_id);
ret = -1;
if (ac->secret) {
logerror(_("error in %s(%d)\n"), __func__, __LINE__);
goto new_done;
}
ac->secret = lyauth_secret();
if (ac->secret == NULL) {
logerror(_("error in %s(%d)\n"), __func__, __LINE__);
goto new_done;
}
nf->host_tag = db_nf.id;
ret = LY_S_REGISTERING_CONFIG;
}
else {
logdebug(_("register request done, node %s not enabled\n"), nf->host_ip);
ret = LY_S_REGISTERING_INIT;
}
nf->status = NODE_STATUS_ONLINE;
if (db_node_update(DB_NODE_FIND_BY_ID, &db_nf.id, nf) < 0) {
logerror(_("error in %s(%d)\n"), __func__, __LINE__);
ret = -1;
}
new_done:
db_node_reginfo_free(&db_nf);
goto done;
xml_err:
logerror(_("invalid node xml register request\n"));
done:
xmlXPathFreeContext(xpathCtx);
logdebug(_("end of %s, node status %d\n"), __func__, nf->status);
return ret;
}
/* process xml response */
static int __process_node_xml_response(xmlDoc * doc, xmlNode * node,
int ent_id)
{
logdebug(_("node response for entity %d\n"), ent_id);
char * str = (char *) xmlGetProp(node, (const xmlChar *) "id");
if (str)
logdebug("node response id(job id) = %s\n", str);
else {
logerror(_("error in %s(%d)\n"), __func__, __LINE__);
return -1;
}
int id = atoi(str);
free(str);
str = (char *) xmlGetProp(node, (const xmlChar *) "status");
if (str)
logdebug("node response status = %s\n", str);
else {
logerror(_("error in %s(%d)\n"), __func__, __LINE__);
return -1;
}
int status = atoi(str);
free(str);
int data_type = -1;
node = node->children;
for (; node; node = node->next) {
if (node->type == XML_ELEMENT_NODE) {
if (strcmp((char *)node->name, "result") == 0) {
xmlNode * n = node->children;
if (n && n->type == XML_TEXT_NODE && n->content)
loginfo(_("response result: %s\n"), n->content);
else
loginfo(_("response no result message\n"));
}
else if (strcmp((char *)node->name, "data") == 0) {
str = (char *) xmlGetProp(node, (const xmlChar *) "type");
if (str) {
logdebug("node response data type = %s\n", str);
data_type = atoi(str);
free(str);
}
else
logwarn(_("response data no type\n"));
}
}
}
if (id != 0) {
LYJobInfo * job = job_find(id);
if (job == NULL) {
logwarn(_("job(%d) not found waiting for node reply\n"), id);
return 0;
}
if (job_update_status(job, status)) {
logerror(_("error in %s(%d)\n"), __func__, __LINE__);
return -1;
}
/* hack to release entity during instance shutting down */
if (((job->j_action == LY_A_NODE_STOP_INSTANCE) ||
(job->j_action == LY_A_NODE_FULLREBOOT_INSTANCE) ||
(job->j_action == LY_A_NODE_ACPIREBOOT_INSTANCE)) &&
status == LY_S_RUNNING_STOPPED) {
int ins_id = ly_entity_find_by_db(LY_ENTITY_OSM, job->j_target_id);
if (ins_id > 0)
ly_entity_release(ins_id);
}
/* continue processing for data */
}
int ent_type = ly_entity_type(ent_id);
if (ent_type == LY_ENTITY_NODE && data_type == DATA_INSTANCE_INFO) {
if ( __instance_info_update(doc, node)) {
logerror(_("error in %s(%d)\n"), __func__, __LINE__);
return -1;
}
}
if (ent_type == LY_ENTITY_NODE && data_type == DATA_NODE_INFO) {
if ( __node_info_update(doc, node, ent_id, &status)) {
logerror(_("error in %s(%d)\n"), __func__, __LINE__);
return -1;
}
}
return 0;
}
/* clc work socket receives connection */
static int __epoll_work_recv(int ent_id)
{
int fd = ly_entity_fd(ent_id);
if (fd == -1)
return -255;
struct sockaddr in_addr;
socklen_t in_len;
int infd;
in_len = sizeof(in_addr);
infd = accept(fd, &in_addr, &in_len);
if (infd == -1) {
if ((errno == EAGAIN) || (errno == EWOULDBLOCK)) {
/* We have processed all incoming connections. */
logdebug(_("accept error(%d) ignored\n"), errno);
return 0;
}
else {
logerror(_("unexpected accept error(%d) in %s"), errno,
__func__);
return -1;
}
}
char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];
int ret = getnameinfo(&in_addr, in_len, hbuf, sizeof(hbuf),
sbuf, sizeof(sbuf),
NI_NUMERICHOST | NI_NUMERICSERV);
if (ret)
logerror(_("error in %s(%d)\n"), __func__, __LINE__);
else
loginfo(_("accepted connection from %s:%s. open socket %d\n"),
hbuf, sbuf, infd);
/* keep alive */
if (lyutil_set_keepalive(infd, CLC_SOCKET_KEEPALIVE_INTVL,
CLC_SOCKET_KEEPALIVE_INTVL,
CLC_SOCKET_KEEPALIVE_PROBES) < 0) {
logerror(_("error in %s(%d)\n"), __func__, __LINE__);
close(infd);
return -1;
}
int id = ly_entity_new(infd);
if (id < 0) {
logerror(_("error in %s(%d)\n"), __func__, __LINE__);
close(infd);
return -1;
}
struct epoll_event ev;
ev.data.fd = id;
ev.events = EPOLLIN;
ret = epoll_ctl(g_efd, EPOLL_CTL_ADD, infd, &ev);
if (ret == -1) {
logerror(_("add socket to epoll error in %s.\n"), __func__);
ly_entity_release(id);
/* close(infd); closed in ly_entity_release */
return -1;
}
loginfo(_("entity %d registered in epoll.\n"), id);
return 0;
}
