int main(int argc, char *argv[])
{
int ret, keeppidfile=1;
setlocale(LC_ALL, "");
bindtextdomain(PACKAGE, LOCALEDIR);
textdomain(PACKAGE);
lyxml_init();
lyauth_init();
/* start initializeing g_c */
g_c = malloc(sizeof(NodeControl));
if (g_c == NULL) {
printf(_("malloc for g_c have a error.\n"));
return -255;
}
bzero(g_c, sizeof(NodeControl));
g_c->node = NULL;
g_c->mfd_cmsg = NULL;
g_c->efd = -1;
g_c->mfd = -1;
g_c->wfd = -1;
NodeConfig *c = &g_c->config;
NodeSysConfig *s = &g_c->config_sys;
/* parse command line option and configuration file */
ret = node_config(argc, argv, c, s);
if (ret == NODE_CONFIG_RET_HELP)
usage();
else if (ret == NODE_CONFIG_RET_VER)
printf(_("%s : Version %s\n"), PROGRAM_NAME, PROGRAM_VERSION);
else if (ret == NODE_CONFIG_RET_ERR_CMD)
printf(_("command line parsing error, use -h option to display usage\n"));
else if (ret == NODE_CONFIG_RET_ERR_NOCONF){
printf(_("missing lynode config file, default build-in settings are used.\n"));
ret = 0;
}
else if (ret == NODE_CONFIG_RET_ERR_ERRCONF)
printf(_("can not find %s.\n"), c->conf_path);
else if (ret == NODE_CONFIG_RET_ERR_CONF)
printf(_("reading config file %s returned error\n"), c->conf_path);
else if (ret == NODE_CONFIG_RET_ERR_UNKNOWN)
printf(_("internal error\n"));
else if (ret != 0)
printf(_("undefined error\n"));
/* exit if ret is not zero */
if (ret != 0)
goto out;
/* identify the clc info to use */
if (c->auto_connect == DISABLE) {
g_c->clc_ip = strdup(c->clc_ip);
g_c->clc_port = c->clc_port;
}
else if (s->clc_ip) {
g_c->clc_ip = strdup(s->clc_ip);
g_c->clc_port = s->clc_port;
}
/* init node state */
if (g_c->clc_ip)
g_c->state = NODE_STATUS_UNINITIALIZED;
else
g_c->state = NODE_STATUS_INITIALIZED;
/* get secret */
if (s->node_secret)
g_c->auth.secret = strdup(s->node_secret);
/* for debuuging */
if (c->debug)
__print_config(c);
/* make sure data directory exists */
if (lyutil_create_dir(c->node_data_dir)) {
printf(_("%s is not accessible\n"), c->node_data_dir);
ret = -255;
goto out;
}
/* check whether program is started already */
ret = lyutil_check_pid_file(c->pid_path, PROGRAM_NAME);
if (ret == 1) {
printf(_("%s is running already.\n"), PROGRAM_NAME);
ret = 0;
goto out;
}
else if (ret != 0) {
printf(_("error checking pid file.\n"));
goto out;
}
/* Connect to libvirt daemon */
if (libvirt_check(c->driver) < 0) {
logsimple(_("error connecting hypervisor.\n"));
ret = -255;
goto out;
}
/* Daemonize the progress */
if (c->daemon) {
if (c->debug == LYDEBUG)
printf(_("Run as daemon, log to %s.\n"), c->log_path);
lyutil_daemonize(__main_clean, keeppidfile);
logfile(c->log_path, c->debug ? LYDEBUG : c->verbose ? LYINFO : LYWARN);
}
else
logfile(NULL, c->debug ? LYDEBUG : c->verbose ? LYINFO : LYWARN);
logcallback(ly_node_send_report, 0);
/* create lock file */
ret = lyutil_create_pid_file(c->pid_path, PROGRAM_NAME);
if (ret == 1) {
logsimple(_("%s is running already.\n"), PROGRAM_NAME);
goto out;
}
else if (ret != 0) {
logsimple(_("error creating pid file.\n"));
goto out;
}
keeppidfile = 0;
/* Connect to libvirt daemon */
if (libvirt_connect(c->driver) < 0) {
logsimple(_("error connecting hypervisor.\n"));
ret = -255;
goto out;
}
/* Init node info */
g_c->node = ly_node_info_init();
if (g_c->node == NULL) {
logsimple(_("error initializing node info.\n"));
ret = -255;
goto out;
}
NodeInfo * nf = g_c->node;
nf->host_tag = s->node_tag;
if (c->debug)
luoyun_node_info_print(nf);
/* set up signal handler */
lyutil_signal_init();
/* handle specific signal */
struct sigaction sa;
sa.sa_flags = 0;
sigemptyset(&sa.sa_mask);
sa.sa_sigaction = __sig_handler;
if (sigaction(SIGTERM, &sa, NULL)) {
logsimple(_("Setting signal handler error.\n"));
ret = -255;
goto out;
}
/* block SIGCHLD */
sigset_t sig;
sigemptyset(&sig);
sigaddset(&sig, SIGCHLD);
pthread_sigmask(SIG_BLOCK, &sig, NULL);
/* initialize g_c->efd */
if (ly_epoll_init(MAX_EVENTS) != 0) {
logsimple(_("ly_epoll_init failed.\n"));
ret = -255;
goto out;
}
/* start main event driven loop */
int i, n;
int wait = -1;
struct epoll_event events[MAX_EVENTS];
while (1) {
if (g_c->clc_ip == NULL) {
/* start listening on clc mcast */
if (g_c->mfd < 0 && ly_epoll_mcast_register() != 0) {
logerror(_("listening on clc mcast error.\n"));
ret = -1;
break;
}
loginfo(_("wait for clc mcast join request...\n"));
}
else if (g_c->wfd < 0 && wait < 0) {
/* init node state */
if (nf->host_tag > 0)
g_c->state = NODE_STATUS_INITIALIZED;
else
g_c->state = NODE_STATUS_UNINITIALIZED;
/* start node registration */
if ((ly_epoll_work_register() != 0 || ly_register_node() != 0)) {
/* close work socket */
ly_epoll_work_close();
logerror(_("failed registering node. will try again\n"));
if (c->auto_connect == ALWAYS) {
free(g_c->clc_ip);
g_c->clc_ip = NULL;
continue;
}
else {
wait = LY_NODE_EPOLL_WAIT;
loginfo(_("wait before retry...\n"));
}
}
}
logdebug(_("waiting for events ...\n"));
n = epoll_wait(g_c->efd, events, MAX_EVENTS, wait);
if (wait > 0)
wait = -1;
loginfo(_("waiting ... got %d events\n"), n);
for (i = 0; i < n; i++) {
if (LY_EVENT_MCAST_DATAIN(events[i])) {
/* mcast data received */
ret = ly_epoll_mcast_recv();
if (ret < 0) {
logwarn(_("unexpected clc mcast data recevied.\n"));
}
else if (ret == 0) {
logdebug(_("ly_epoll_mcast_recv returns 0. do nothing.\n"));
}
else {
/* the clc ip/port are obtained from mcast */
loginfo(_("new clc mcast data received. "
"re-registering....\n"));
ly_epoll_mcast_close();
}
}
else if (LY_EVENT_WORK_DATAIN(events[i])) {
/* clc data received */
ret = ly_epoll_work_recv();
if (ret == 0) {
logdebug(_("ly_epoll_work_recv return 0. continue reading...\n"));
continue;
}
/* in all other cases, close work socket */
ly_epoll_work_close();
if (ret < 0)
logwarn(_("unexpected work process error. "
"will reopen work socket ...\n"));
else
logwarn(_("node work socket closed. "
"will reopen ...\n"));
if (c->auto_connect == ALWAYS) {
free(g_c->clc_ip);
g_c->clc_ip = NULL;
}
}
else if (events[i].events & EPOLLRDHUP) {
/* work closed by clc */
logdebug(_("close by remote\n"));
ly_epoll_work_close();
}
else {
logwarn(_("unexpected epoll event(%d) for %d\n"),
events[i].events, events[i].data.fd);
}
}
}
out:
__main_clean(keeppidfile);
if (ret <= 0)
loginfo(_("%s exits\n"), PROGRAM_NAME);
return ret;
}
int main(int argc, char *argv[])
{
int ret, keeppidfile = 1;
setlocale(LC_ALL, "");
bindtextdomain(PACKAGE, LOCALEDIR);
textdomain(PACKAGE);
lyxml_init();
lyauth_init();
/* start initializeing g_c */
CLCConfig *c = malloc(sizeof(CLCConfig));
if (c == NULL) {
printf(_("malloc for g_c have a error.\n"));
return -255;
}
g_c = c;
/* parse command line option and configuration file */
ret = clc_config(argc, argv, c);
if (ret == CLC_CONFIG_RET_HELP)
usage();
else if (ret == CLC_CONFIG_RET_VER)
printf(_("%s : Version %s\n"), PROGRAM_NAME, PROGRAM_VERSION);
else if (ret == CLC_CONFIG_RET_ERR_CMD)
printf(_
("command line parsing error, use -h option to display usage\n"));
else if (ret == CLC_CONFIG_RET_ERR_NOCONF) {
printf(_
("missing lyclc config file, default build-in settings are used.\n"));
ret = 0;
}
else if (ret == CLC_CONFIG_RET_ERR_ERRCONF)
printf(_("can not find %s.\n"), c->conf_path);
else if (ret == CLC_CONFIG_RET_ERR_CONF)
printf(_("reading config file %s returned error\n"), c->conf_path);
else if (ret == CLC_CONFIG_RET_ERR_UNKNOWN)
printf(_("internal error\n"));
/* exit if ret is not zero */
if (ret != 0)
goto out;
/* node cpu/mem factors */
if (c->node_cpu_factor == 0)
c->node_cpu_factor = DEFAULT_NODE_CPU_FACTOR;
if (c->node_mem_factor == 0)
c->node_mem_factor = DEFAULT_NODE_MEM_FACTOR;
/* for debuuging */
if (c->debug)
__print_config(c);
/* make sure data directory exists */
if (lyutil_create_dir(c->clc_data_dir)) {
printf(_("%s is not accessible\n"), c->clc_data_dir);
ret = -255;
goto out;
}
/* check whether program is started already */
ret = lyutil_check_pid_file(c->pid_path, PROGRAM_NAME);
if (ret == 1) {
printf(_("%s is running already.\n"), PROGRAM_NAME);
goto out;
}
else if (ret != 0) {
printf(_("error checking pid file.\n"));
goto out;
}
/* check DB */
if (ly_db_check() < 0) {
printf(_("failed connecting DB\n"));
ret = -255;
goto out;
}
/* get clc ip */
if (c->clc_ip == NULL && ly_clc_ip_get() < 0) {
logerror(_("CLC no proper network interface to use.\n"));
goto out;
}
/* Daemonize the progress */
if (c->daemon) {
if (c->debug == LYDEBUG)
printf(_("Run as daemon, log to %s.\n"), c->log_path);
lyutil_daemonize(__main_clean, keeppidfile);
logfile(c->log_path, c->debug ? LYDEBUG : c->verbose ? LYINFO : LYWARN);
}
else
logfile(NULL, c->debug ? LYDEBUG : c->verbose ? LYINFO : LYWARN);
/* create lock file */
ret = lyutil_create_pid_file(c->pid_path, PROGRAM_NAME);
if (ret == 1) {
logsimple(_("%s is running already.\n"), PROGRAM_NAME);
ret = 0;
goto out;
}
else if (ret != 0) {
logsimple(_("error creating pid file.\n"));
goto out;
}
keeppidfile = 0;
/* init db connection */
if (ly_db_init() < 0) {
logsimple(_("ly_db_init failed.\n"));
ret = -255;
goto out;
}
/* initialize entity store */
if (ly_entity_store_init() < 0) {
logsimple(_("ly_entity_init failed.\n"));
ret = -255;
goto out;
}
/* init job queue */
if (job_init() < 0 || job_internal_init() < 0) {
logsimple(_("job_init failed.\n"));
ret = -255;
goto out;
}
if (c->debug)
job_print_queue();
/* set up signal handler */
lyutil_signal_init();
/* handle specific signal */
struct sigaction sa;
sa.sa_flags = 0;
sigemptyset(&sa.sa_mask);
sa.sa_sigaction = __sig_handler;
if (sigaction(SIGTERM, &sa, NULL)) {
logsimple(_("Setting signal handler error.\n"));
ret = -255;
goto out;
}
/* initialize g_c->efd */
if (ly_epoll_init(EPOLL_EVENTS_MAX) != 0) {
logsimple(_("ly_epoll_init failed.\n"));
ret = -255;
goto out;
}
if (ly_epoll_work_start(g_c->clc_port) != 0) {
ret = -1;
logsimple(_("ly_epoll_init failed.\n"));
goto out;
}
/* init timeout values */
time_t mcast_join_time, job_dispatch_time, job_internal_time;
mcast_join_time = 0;
time(&job_dispatch_time);
job_internal_time = job_dispatch_time + (CLC_MCAST_JOIN_INTERVAL<<1);
job_dispatch_time = job_dispatch_time + (CLC_MCAST_JOIN_INTERVAL<<2);
/* start main event driven loop */
if (c->clc_ip)
loginfo(_("clc uses IP %s\n"), c->clc_ip);
else
loginfo(_("clc uses IP automatically detected\n"));
loginfo(_("start event loop, waiting for events ...\n"));
int i, n = 0;
struct epoll_event events[EPOLL_EVENTS_MAX];
while (1) {
time_t time_now;
time(&time_now);
/* send mcast request */
if (time_now - mcast_join_time > CLC_MCAST_JOIN_INTERVAL) {
if (ly_mcast_send_join() < 0)
logerror(_("failed sending mcast request.\n"));
mcast_join_time = time_now;
}
else if (time_now < mcast_join_time)
mcast_join_time = time_now;
/* job dispatch */
if (time_now - job_dispatch_time > CLC_JOB_DISPATCH_INTERVAL) {
if (job_dispatch() < 0)
logerror(_("job_dispatch failed.\n"));
job_dispatch_time = time_now;
}
else if (time_now < job_dispatch_time)
job_dispatch_time = time_now;
/* internal job dispatch */
if (time_now - job_internal_time > CLC_JOB_INTERNAL_INTERVAL) {
if (job_internal_dispatch() < 0)
logerror(_("job_internal failed.\n"));
job_internal_time = time_now;
}
else if (time_now < job_internal_time)
job_internal_time = time_now;
n = epoll_wait(g_efd, events, EPOLL_EVENTS_MAX,
CLC_EPOLL_TIMEOUT);
if (n != 0)
logdebug(_("waiting ... got %d events\n"), n);
for (i = 0; i < n; i++) {
int id = events[i].data.fd;
if (events[i].events & EPOLLIN) {
ret = ly_epoll_entity_recv(id);
if (ret < 0) {
logerror(_("epoll_data_recv error\n"));
}
else if (ret > 0) {
loginfo(_("release entity %d\n"), id);
ly_entity_release(id);
}
}
else if (events[i].events & EPOLLRDHUP) {
loginfo(_("epoll entity(%d) got rdhup. close.\n"), id);
ly_entity_release(id);
}
else if (events[i].events & EPOLLHUP) {
loginfo(_("epoll entity(%d) got hup. close.\n"), id);
ly_entity_release(id);
}
else {
logerror(_("unexpected event(%d, %d). ignore.\n"),
events[i].events, id);
}
}
}
out:
__main_clean(keeppidfile);
if (ret <= 0)
loginfo(_("%s exits\n"), PROGRAM_NAME);
return ret;
}
int node_config(int argc, char *argv[], NodeConfig *c, NodeSysConfig *s)
{
int ret;
if (c == NULL)
return NODE_CONFIG_RET_ERR_UNKNOWN; /* internal error */
/* initialize NodeConfig with undefined values */
bzero(c, sizeof(NodeConfig));
c->auto_connect = UNKNOWN;
c->verbose = UNDEFINED_CFG_INT;
c->daemon = UNDEFINED_CFG_INT;
c->debug = UNDEFINED_CFG_INT;
c->driver = HYPERVISOR_IS_KVM;
/* parse command line options */
ret = __parse_opt(argc, argv, c);
if (ret)
return ret; /* to exit program */
/* check conf_path before reading it */
if (c->conf_path == NULL) {
c->conf_path = strdup(DEFAULT_LYNODE_CONF_PATH);
if (c->conf_path == NULL)
return NODE_CONFIG_RET_ERR_NOMEM;
}
/* parse config file */
if (access(c->conf_path, R_OK) == 0) {
ret = __parse_config(c);
if (ret && ret != NODE_CONFIG_RET_ERR_NOCONF)
return ret; /* to exit programe */
}
/* set default values for auto_connect */
if (c->auto_connect == UNKNOWN)
c->auto_connect = ALWAYS;
/* read sysconf settings */
bzero(s, sizeof(NodeSysConfig));
s->node_tag = -1;
if (c->sysconf_path == NULL) {
c->sysconf_path = strdup(DEFAULT_LYNODE_SYSCONF_PATH);
if (c->conf_path == NULL)
return NODE_CONFIG_RET_ERR_NOMEM;
}
if (access(c->sysconf_path, W_OK) != 0) {
if (lyutil_create_file(c->sysconf_path, 0) < 0) {
logsimple(_("not able to write to %s\n"), c->sysconf_path);
return NODE_CONFIG_RET_ERR_CMD;
}
}
else {
ret = __parse_sysconf(c->sysconf_path, s);
if (ret)
return ret; /* to exit program */
}
/* read xml template */
if (c->vm_template_path) {
c->vm_xml = file2str(c->vm_template_path, LIBVIRT_XML_MAX);
if (c->vm_xml == NULL) {
logsimple(_("error while reading %s\n"), c->vm_template_path);
return NODE_CONFIG_RET_ERR_CMD;
}
}
if (c->vm_template_net_nat_path) {
c->vm_xml_net_nat = file2str(c->vm_template_net_nat_path, LIBVIRT_XML_MAX);
if (c->vm_xml_net_nat == NULL) {
logsimple(_("error while reading %s\n"), c->vm_template_net_nat_path);
return NODE_CONFIG_RET_ERR_CMD;
}
}
if (c->vm_template_net_br_path) {
c->vm_xml_net_br = file2str(c->vm_template_net_br_path, LIBVIRT_XML_MAX);
if (c->vm_xml_net_br == NULL) {
logsimple(_("error while reading %s\n"), c->vm_template_net_br_path);
return NODE_CONFIG_RET_ERR_CMD;
}
}
if (c->vm_template_disk_path) {
c->vm_xml_disk = file2str(c->vm_template_disk_path, LIBVIRT_XML_MAX);
if (c->vm_xml_disk == NULL) {
logsimple(_("error while reading %s\n"), c->vm_template_disk_path);
return NODE_CONFIG_RET_ERR_CMD;
}
}
/* set default values for unconfigured settings */
if (c->verbose == UNDEFINED_CFG_INT)
c->verbose = 0;
if (c->daemon == UNDEFINED_CFG_INT)
c->daemon = 1;
if (c->debug == UNDEFINED_CFG_INT)
c->debug = 0;
if (c->clc_port == 0)
c->clc_port = DEFAULT_LYCLC_PORT;
if (c->clc_mcast_ip == NULL)
c->clc_mcast_ip = strdup(DEFAULT_LYCLC_MCAST_IP);
if (c->clc_mcast_port == 0)
c->clc_mcast_port = DEFAULT_LYCLC_MCAST_PORT;
if (c->log_path == NULL) {
c->log_path = strdup(DEFAULT_LYNODE_LOG_PATH);
if (c->log_path == NULL)
return NODE_CONFIG_RET_ERR_NOMEM;
}
if (c->pid_path == NULL) {
c->pid_path = strdup(DEFAULT_LYNODE_PID_PATH);
if (c->pid_path == NULL)
return NODE_CONFIG_RET_ERR_NOMEM;
}
if (c->osm_conf_path == NULL) {
c->osm_conf_path = strdup(DEFAULT_LYOSM_CONF_PATH);
if (c->osm_conf_path == NULL)
return NODE_CONFIG_RET_ERR_NOMEM;
}
if (c->osm_key_path == NULL) {
c->osm_key_path = strdup(DEFAULT_LYOSM_KEY_PATH);
if (c->osm_key_path == NULL)
return NODE_CONFIG_RET_ERR_NOMEM;
}
if (c->node_data_dir == NULL) {
c->node_data_dir = strdup(DEFAULT_LYNODE_DATA_DIR);
if (c->node_data_dir == NULL)
return NODE_CONFIG_RET_ERR_NOMEM;
}
if (c->net_primary == NULL) {
c->net_primary = strdup(LUOYUN_INSTANCE_NET_DEFAULT);
if (c->net_primary == NULL)
return NODE_CONFIG_RET_ERR_NOMEM;
}
/* create necessary sub-directories for normal operation */
int len = strlen(c->node_data_dir);
c->app_data_dir = malloc(len + 15);
c->ins_data_dir = malloc(len + 15);
c->trash_data_dir = malloc(len + 15);
if (c->app_data_dir == NULL || c->ins_data_dir == NULL)
return NODE_CONFIG_RET_ERR_NOMEM;
sprintf(c->app_data_dir, "%s/appliances", c->node_data_dir);
if (lyutil_create_dir(c->app_data_dir) != 0) {
logsimple(_("failed creating directory of %s\n"), c->app_data_dir);
return NODE_CONFIG_RET_ERR_CMD;
}
if (__clean_lockfile(c->app_data_dir) != 0) {
logsimple(_("failed cleaning directory of %s\n"), c->app_data_dir);
return NODE_CONFIG_RET_ERR_CMD;
}
sprintf(c->ins_data_dir, "%s/instances", c->node_data_dir);
if (lyutil_create_dir(c->ins_data_dir) != 0) {
logsimple(_("failed creating directory of %s\n"), c->ins_data_dir);
return NODE_CONFIG_RET_ERR_CMD;
}
if (__clean_lockfile(c->ins_data_dir) != 0) {
logsimple(_("failed cleaning directory of %s\n"), c->ins_data_dir);
return NODE_CONFIG_RET_ERR_CMD;
}
sprintf(c->trash_data_dir, "%s/trash", c->node_data_dir);
if (lyutil_create_dir(c->trash_data_dir) != 0) {
logsimple(_("failed creating directory of %s\n"), c->trash_data_dir);
return NODE_CONFIG_RET_ERR_CMD;
}
/* simple configuration validity checking */
if (__is_IP_valid(c->clc_mcast_ip, 1) == 0) {
logsimple(_("cloud controller mcast ip is invalid\n"));
return NODE_CONFIG_RET_ERR_CONF;
}
if (__is_port_valid(c->clc_mcast_port) == 0 ) {
logsimple(_("cloud controller port is invalid\n"));
return NODE_CONFIG_RET_ERR_CONF;
}
if (__is_port_valid(c->clc_port) == 0 ) {
logsimple(_("cloud controller port is invalid\n"));
return NODE_CONFIG_RET_ERR_CONF;
}
if ((c->auto_connect == DISABLE || c->clc_ip)
&& __is_IP_valid(c->clc_ip, 0) == 0) {
logsimple(_("cloud controller ip is invalid<%s>\n"), c->clc_ip);
return NODE_CONFIG_RET_ERR_CONF;
}
if (c->log_path) {
if (access(c->log_path, F_OK) && lyutil_create_file(c->log_path, 0) < 0) {
logsimple(_("not able to create %s\n"), c->log_path);
return NODE_CONFIG_RET_ERR_CMD;
}
if (access(c->log_path, W_OK)) {
logsimple(_("not able to write to %s\n"), c->log_path);
return NODE_CONFIG_RET_ERR_CMD;
}
}
return ret;
}
