static int doInitialize (struct nc_state_t *nc)
{
char *s = NULL;
virNodeInfo ni;
long long dom0_min_mem;
logprintfl(EUCADEBUG, "doInitialized() invoked\n");
/* set up paths of Eucalyptus commands NC relies on */
snprintf (nc->gen_libvirt_cmd_path, MAX_PATH, EUCALYPTUS_GEN_LIBVIRT_XML, nc->home, nc->home);
snprintf (nc->get_info_cmd_path, MAX_PATH, EUCALYPTUS_GET_XEN_INFO, nc->home, nc->home);
snprintf (nc->virsh_cmd_path, MAX_PATH, EUCALYPTUS_VIRSH, nc->home);
snprintf (nc->xm_cmd_path, MAX_PATH, EUCALYPTUS_XM);
snprintf (nc->detach_cmd_path, MAX_PATH, EUCALYPTUS_DETACH, nc->home, nc->home);
snprintf (nc->connect_storage_cmd_path, MAX_PATH, EUCALYPTUS_CONNECT_ISCSI, nc->home, nc->home);
snprintf (nc->disconnect_storage_cmd_path, MAX_PATH, EUCALYPTUS_DISCONNECT_ISCSI, nc->home, nc->home);
snprintf (nc->get_storage_cmd_path, MAX_PATH, EUCALYPTUS_GET_ISCSI, nc->home, nc->home);
strcpy(nc->uri, HYPERVISOR_URI);
nc->convert_to_disk = 0;
/* check connection is fresh */
if (!check_hypervisor_conn()) {
return ERROR_FATAL;
}
/* get resources */
if (virNodeGetInfo(nc->conn, &ni)) {
logprintfl (EUCAFATAL, "error: failed to discover resources\n");
return ERROR_FATAL;
}
/* dom0-min-mem has to come from xend config file */
s = system_output (nc->get_info_cmd_path);
if (get_value (s, "dom0-min-mem", &dom0_min_mem)) {
logprintfl (EUCAFATAL, "error: did not find dom0-min-mem in output from %s\n", nc->get_info_cmd_path);
free (s);
return ERROR_FATAL;
}
free (s);
/* calculate the available memory */
nc->mem_max = ni.memory/1024 - 32 - dom0_min_mem;
/* calculate the available cores */
nc->cores_max = ni.cpus;
/* let's adjust the values based on the config values */
if (nc->config_max_mem && nc->config_max_mem < nc->mem_max)
nc->mem_max = nc->config_max_mem;
if (nc->config_max_cores)
nc->cores_max = nc->config_max_cores;
logprintfl(EUCAINFO, "Using %lld cores\n", nc->cores_max);
logprintfl(EUCAINFO, "Using %lld memory\n", nc->mem_max);
return OK;
}
//!
//!
//!
//! @param[in] ptr
//!
//! @return Always returns NULL
//!
static void *checker_thread(void *ptr)
{
int iter = 0;
for (iter = 0; iter < IITERS; iter++) {
printf("checker thread starting iteration %d\n", iter);
check_hypervisor_conn();
}
return (NULL);
}
static int
doTerminateInstance( struct nc_state_t *nc,
ncMetadata *meta,
char *instanceId,
int *shutdownState,
int *previousState)
{
ncInstance *instance, *vninstance;
virConnectPtr *conn;
int err;
sem_p (inst_sem);
instance = find_instance(&global_instances, instanceId);
sem_v (inst_sem);
if (instance == NULL)
return NOT_FOUND;
/* try stopping the KVM domain */
conn = check_hypervisor_conn();
if (conn) {
sem_p(hyp_sem);
virDomainPtr dom = virDomainLookupByName(*conn, instanceId);
sem_v(hyp_sem);
if (dom) {
/* also protect 'destroy' commands, just in case */
sem_p (hyp_sem);
err = virDomainDestroy (dom);
sem_v (hyp_sem);
if (err==0) {
logprintfl (EUCAINFO, "destroyed domain for instance %s\n", instanceId);
}
sem_p(hyp_sem);
virDomainFree(dom); /* necessary? */
sem_v(hyp_sem);
} else {
if (instance->state != BOOTING)
logprintfl (EUCAWARN, "warning: domain %s to be terminated not running on hypervisor\n", instanceId);
}
}
/* change the state and let the monitoring_thread clean up state */
sem_p (inst_sem);
if (instance->state==BOOTING) {
change_state (instance, CANCELED);
} else {
change_state (instance, SHUTOFF);
}
sem_v (inst_sem);
*previousState = instance->stateCode;
*shutdownState = instance->stateCode;
return OK;
}
static int doInitialize (struct nc_state_t *nc)
{
char *s = NULL;
virNodeInfo ni;
long long dom0_min_mem;
// set up paths of Eucalyptus commands NC relies on
snprintf (nc->get_info_cmd_path, MAX_PATH, EUCALYPTUS_GET_XEN_INFO, nc->home, nc->home);
snprintf (nc->virsh_cmd_path, MAX_PATH, EUCALYPTUS_VIRSH, nc->home);
snprintf (nc->xm_cmd_path, MAX_PATH, EUCALYPTUS_XM);
snprintf (nc->detach_cmd_path, MAX_PATH, EUCALYPTUS_DETACH, nc->home, nc->home);
strcpy(nc->uri, HYPERVISOR_URI);
nc->convert_to_disk = 0;
nc->capability = HYPERVISOR_XEN_AND_HARDWARE; // TODO: set to XEN_PARAVIRTUALIZED if on older Xen kernel
// check connection is fresh
if (!check_hypervisor_conn()) {
return ERROR_FATAL;
}
// get resources
if (virNodeGetInfo(nc->conn, &ni)) {
logprintfl (EUCAFATAL, "failed to discover resources\n");
return ERROR_FATAL;
}
// dom0-min-mem has to come from xend config file
s = system_output (nc->get_info_cmd_path);
if (get_value (s, "dom0-min-mem", &dom0_min_mem)) {
logprintfl (EUCAFATAL, "did not find dom0-min-mem in output from %s\n", nc->get_info_cmd_path);
free (s);
return ERROR_FATAL;
}
free (s);
// calculate the available memory
nc->mem_max = ni.memory/1024 - 32 - dom0_min_mem;
// calculate the available cores
nc->cores_max = ni.cpus;
// let's adjust the values based on the config values
if (nc->config_max_mem && nc->config_max_mem < nc->mem_max)
nc->mem_max = nc->config_max_mem;
if (nc->config_max_cores)
nc->cores_max = nc->config_max_cores;
return OK;
}
static int
doAttachSystemDisk ( struct nc_state_t *nc,
ncMetadata *meta,
char *instanceId,
char *isoPath)
{
ncInstance *instance;
virConnectPtr *conn;
virDomainPtr dom = NULL;
char xml[MAX_PATH]={'\0'};
int err;
snprintf (xml, sizeof(xml), "<disk type='block' device='cdrom'><source dev='%s'/><target dev='hdc' bus='ide'/><readonly/></disk>", isoPath);
//logprintfl(EUCAINFO, "doAttachSystemDisk(): founded %s [%s---%s---%d]\n", instanceId,__FILE__, __FUNCTION__, __LINE__);
sem_p(inst_sem);
instance = find_instance(&global_instances, instanceId);
sem_v(inst_sem);
if (instance == NULL || (instance->state != RUNNING&& instance->state != BLOCKED&& instance->state !=PAUSED)) {
logprintfl(EUCAINFO, "doAttachSystemDisk(): found %s failed or instance->state!=RUNNING||BLOCKED||PAUSED [file:%s---line:%d]\n", instanceId, __FILE__, __LINE__);
return NOT_FOUND;
}
conn = check_hypervisor_conn();
if (conn) {
/*
snprintf(cmd, sizeof(cmd), "virsh attach-disk %s %s hdc --type cdrom --mode readonly",instanceId, isoPath);
logprintfl(EUCAINFO, "xiongm: cmd=%s [%s---%s---%d]\n",cmd, __FILE__, __FUNCTION__, __LINE__);
system(cmd);
*/
dom = NULL;
sem_p(hyp_sem);
dom = virDomainLookupByName(*conn, instanceId);
sem_v(hyp_sem);
if (dom) {
sem_p (hyp_sem);
err = virDomainAttachDevice (dom, xml);
sem_v (hyp_sem);
if(err != 0) {
logprintfl(EUCAINFO,"virDomainAttachDevice failed. err=%d\n",err);
return 1;
}
}
logprintfl(EUCAINFO,"doAttachSystemDisk success.\n");
return 0;
}
else {
logprintfl(EUCAINFO, "doAttachSystemDisk(): no connect hypervisior [file:%s---line:%d]\n",__FILE__, __LINE__);
return 1;
}
}
static int doRebootInstance( struct nc_state_t *nc,
ncMetadata *meta,
char *instanceId)
{
ncInstance *instance;
virConnectPtr *conn;
sem_p (inst_sem);
instance = find_instance(&global_instances, instanceId);
sem_v (inst_sem);
if ( instance == NULL ) return NOT_FOUND;
/* reboot the Xen domain */
conn = check_hypervisor_conn();
if (conn) {
sem_p(hyp_sem);
virDomainPtr dom = virDomainLookupByName(*conn, instanceId);
sem_v(hyp_sem);
if (dom) {
/* also protect 'reboot', just in case */
sem_p (hyp_sem);
int err=virDomainReboot (dom, 0);
sem_v (hyp_sem);
if (err==0) {
logprintfl (EUCAINFO, "[%s] rebooting Xen domain for instance\n", instanceId);
}
sem_p(hyp_sem);
virDomainFree(dom); /* necessary? */
sem_v(hyp_sem);
} else {
if (instance->state != BOOTING && instance->state != STAGING) {
logprintfl (EUCAWARN, "[%s] domain to be rebooted not running on hypervisor\n", instanceId);
}
}
}
return 0;
}
static int doDescribeHardware ( struct nc_state_t *nc,
ncMetadata *meta,
ncHardwareInfo *hwinfo)
{
virNodeInfo info;
virConnectPtr *con = check_hypervisor_conn();
sem_p (hyp_sem);
if (virNodeGetInfo (*con, &info) != 0)
return (-1);
sem_v (hyp_sem);
strcpy (hwinfo->model, info.model);
hwinfo->memory = info.memory;
hwinfo->cpus = info.cpus;
hwinfo->mhz = info.mhz;
hwinfo->nodes = info.nodes;
hwinfo->sockets = info.sockets;
hwinfo->cores = info.cores;
hwinfo->threads = info.threads;
return (0);
}
//!
//!
//!
//! @param[in] ptr
//!
//! @return Always returns NULL
//!
static void *tortura_thread(void *ptr)
{
int i = 0;
int iter = 0;
int error = 0;
int num_doms = 0;
int dom_ids[MAXDOMS] = { 0 };
long long tid = (*(long long *)ptr);
virDomainPtr dom[MAXDOMS] = { NULL };
virDomainInfo info[MAXDOMS] = { 0 };
check_hypervisor_conn();
for (iter = 0; iter < ITERS; iter++) {
printf("thread %lld starting iteration %d\n", tid, iter);
pthread_mutex_lock(&check_mutex);
{
num_doms = virConnectListDomains(conn, dom_ids, MAXDOMS);
}
pthread_mutex_unlock(&check_mutex);
for (i = 0; i < num_doms; i++) {
pthread_mutex_lock(&check_mutex);
{
dom[i] = virDomainLookupByID(conn, dom_ids[i]);
}
pthread_mutex_unlock(&check_mutex);
if (!dom[i]) {
printf("failed to look up domain %d\n", dom_ids[i]);
continue;
}
}
for (i = 0; i < num_doms; i++) {
pthread_mutex_lock(&check_mutex);
{
error = virDomainGetInfo(dom[i], &info[i]);
}
pthread_mutex_unlock(&check_mutex);
if (error < 0) {
printf("failed to get info on domain %d\n", dom_ids[i]);
continue;
}
}
for (i = 0; i < num_doms; i++) {
pthread_mutex_lock(&check_mutex);
{
error = virDomainFree(dom[i]);
}
pthread_mutex_unlock(&check_mutex);
if (error < 0) {
printf("failed to close domain %d\n", dom_ids[i]);
continue;
}
}
}
return (NULL);
}
void adopt_instances()
{
int dom_ids[MAXDOMS];
int num_doms = 0;
int i;
virDomainPtr dom = NULL;
if (! check_hypervisor_conn())
return;
logprintfl (EUCAINFO, "looking for existing domains\n");
virSetErrorFunc (NULL, libvirt_error_handler);
num_doms = virConnectListDomains(nc_state.conn, dom_ids, MAXDOMS);
if (num_doms == 0) {
logprintfl (EUCAINFO, "no currently running domains to adopt\n");
return;
} if (num_doms < 0) {
logprintfl (EUCAWARN, "WARNING: failed to find out about running domains\n");
return;
}
for ( i=0; i<num_doms; i++) {
int error;
virDomainInfo info;
const char * dom_name;
ncInstance * instance;
sem_p(hyp_sem);
dom = virDomainLookupByID(nc_state.conn, dom_ids[i]);
sem_v(hyp_sem);
if (!dom) {
logprintfl (EUCAWARN, "WARNING: failed to lookup running domain #%d, ignoring it\n", dom_ids[i]);
continue;
}
sem_p(hyp_sem);
error = virDomainGetInfo(dom, &info);
sem_v(hyp_sem);
if (error < 0 || info.state == VIR_DOMAIN_NOSTATE) {
logprintfl (EUCAWARN, "WARNING: failed to get info on running domain #%d, ignoring it\n", dom_ids[i]);
continue;
}
if (info.state == VIR_DOMAIN_SHUTDOWN ||
info.state == VIR_DOMAIN_SHUTOFF ||
info.state == VIR_DOMAIN_CRASHED ) {
logprintfl (EUCADEBUG, "ignoring non-running domain #%d\n", dom_ids[i]);
continue;
}
sem_p(hyp_sem);
if ((dom_name = virDomainGetName(dom))==NULL) {
sem_v(hyp_sem);
logprintfl (EUCAWARN, "WARNING: failed to get name of running domain #%d, ignoring it\n", dom_ids[i]);
continue;
}
sem_v(hyp_sem);
if (!strcmp(dom_name, "Domain-0"))
continue;
if ((instance = scRecoverInstanceInfo (dom_name))==NULL) {
logprintfl (EUCAWARN, "WARNING: failed to recover Eucalyptus metadata of running domain %s, ignoring it\n", dom_name);
continue;
}
change_state (instance, info.state);
sem_p (inst_sem);
int err = add_instance (&global_instances, instance);
sem_v (inst_sem);
if (err) {
free_instance (&instance);
continue;
}
logprintfl (EUCAINFO, "- adopted running domain %s from user %s\n", instance->instanceId, instance->userId);
/* TODO: try to look up IPs? */
sem_p(hyp_sem);
virDomainFree (dom);
sem_v(hyp_sem);
}
}
void *startup_thread (void * arg)
{
ncInstance * instance = (ncInstance *)arg;
virDomainPtr dom = NULL;
char *disk_path=NULL, *xml=NULL;
char *brname=NULL;
int error, i;
if (! check_hypervisor_conn ()) {
logprintfl (EUCAFATAL, "could not start instance %s, abandoning it\n", instance->instanceId);
change_state (instance, SHUTOFF);
return NULL;
}
error = vnetStartNetwork (nc_state.vnetconfig, instance->ncnet.vlan, NULL, NULL, &brname);
if ( error ) {
logprintfl (EUCAFATAL, "start network failed for instance %s, terminating it\n", instance->instanceId);
change_state (instance, SHUTOFF);
return NULL;
}
logprintfl (EUCAINFO, "network started for instance %s\n", instance->instanceId);
error = scMakeInstanceImage (nc_state.home,
instance->userId,
instance->imageId, instance->imageURL,
instance->kernelId, instance->kernelURL,
instance->ramdiskId, instance->ramdiskURL,
instance->instanceId, instance->keyName,
&disk_path,
addkey_sem, nc_state.convert_to_disk,
instance->params.disk*1024);
if (error) {
logprintfl (EUCAFATAL, "Failed to prepare images for instance %s (error=%d)\n", instance->instanceId, error);
change_state (instance, SHUTOFF);
if (brname) free(brname);
if (disk_path) free(disk_path);
return NULL;
}
if (instance->state==TEARDOWN) { // timed out in STAGING
if (disk_path) free(disk_path);
if (brname) free(brname);
return NULL;
}
if (instance->state==CANCELED) {
logprintfl (EUCAFATAL, "Startup of instance %s was cancelled\n", instance->instanceId);
change_state (instance, SHUTOFF);
if (brname) free(brname);
if (disk_path) free(disk_path);
return NULL;
}
error = get_instance_xml (nc_state.gen_libvirt_cmd_path,
instance->userId, instance->instanceId,
instance->ramdiskId,
instance->kernelId,
disk_path,
&(instance->params),
instance->ncnet.privateMac,
brname,
nc_state.config_use_virtio_net,
nc_state.config_use_virtio_root,
&xml);
if (brname) free(brname);
if (disk_path) free(disk_path);
if (xml) logprintfl (EUCADEBUG2, "libvirt XML config:\n%s\n", xml);
if (error) {
logprintfl (EUCAFATAL, "Failed to create libvirt XML config for instance %s\n", instance->instanceId);
change_state (instance, SHUTOFF);
return NULL;
}
scStoreStringToInstanceFile (instance->userId, instance->instanceId, "libvirt.xml", xml); /* for debugging */
scSaveInstanceInfo(instance); /* to enable NC recovery */
/* we serialize domain creation as hypervisors can get confused with
* too many simultaneous create requests */
logprintfl (EUCADEBUG2, "about to start domain %s\n", instance->instanceId);
print_running_domains ();
for (i=0; i<5 && dom == NULL; i++) {
sem_p (hyp_sem);
dom = virDomainCreateLinux (nc_state.conn, xml, 0);
sem_v (hyp_sem);
}
if (xml) free(xml);
if (dom == NULL) {
logprintfl (EUCAFATAL, "hypervisor failed to start domain\n");
change_state (instance, SHUTOFF);
return NULL;
}
eventlog("NC", instance->userId, "", "instanceBoot", "begin"); // TODO: bring back correlationId
sem_p(hyp_sem);
virDomainFree(dom);
sem_v(hyp_sem);
sem_p (inst_sem);
// check one more time for cancellation
if (instance->state==TEARDOWN) {
// timed out in BOOTING
} else if (instance->state==CANCELED || instance->state==SHUTOFF) {
logprintfl (EUCAFATAL, "startup of instance %s was cancelled\n", instance->instanceId);
change_state (instance, SHUTOFF);
} else {
logprintfl (EUCAINFO, "booting VM instance %s\n", instance->instanceId);
instance->bootTime = time (NULL);
change_state (instance, BOOTING);
}
sem_v (inst_sem);
return NULL;
}
static int
doDetachVolume ( struct nc_state_t *nc,
ncMetadata *meta,
char *instanceId,
char *volumeId,
char *remoteDev,
char *localDev,
int force)
{
int ret = OK;
ncInstance * instance;
char localDevReal[32];
virConnectPtr *conn;
// fix up format of incoming local dev name, if we need to
ret = convert_dev_names (localDev, localDevReal, NULL);
if (ret)
return ret;
sem_p (inst_sem);
instance = find_instance(&global_instances, instanceId);
sem_v (inst_sem);
if ( instance == NULL )
return NOT_FOUND;
/* try attaching to the Xen domain */
conn = check_hypervisor_conn();
if (conn) {
sem_p(hyp_sem);
virDomainPtr dom = virDomainLookupByName(*conn, instanceId);
sem_v(hyp_sem);
if (dom) {
int err = 0, fd, rc, pid, status;
char xml [1024], tmpfile[32], cmd[MAX_PATH];
FILE *FH;
int is_iscsi_target = 0;
char *local_iscsi_dev;
if(check_iscsi(remoteDev)) {
is_iscsi_target = 1;
/*get credentials, decrypt them*/
//parse_target(remoteDev);
/*logout from target*/
if((local_iscsi_dev = get_iscsi_target(nc->get_storage_cmd_path, remoteDev)) == NULL)
return ERROR;
snprintf (xml, 1024, "<disk type='block'><driver name='phy'/><source dev='%s'/><target dev='%s'/></disk>", local_iscsi_dev, localDevReal);
} else {
snprintf (xml, 1024, "<disk type='block'><driver name='phy'/><source dev='%s'/><target dev='%s'/></disk>", remoteDev, localDevReal);
}
/* protect Xen calls, just in case */
sem_p (hyp_sem);
pid = fork();
if (!pid) {
char cmd[MAX_PATH];
snprintf(tmpfile, 32, "/tmp/detachxml.XXXXXX");
fd = mkstemp(tmpfile);
if (fd > 0) {
write(fd, xml, strlen(xml));
close(fd);
snprintf(cmd, MAX_PATH, "%s %s `which virsh` %s %s %s", nc->detach_cmd_path, nc->rootwrap_cmd_path, instanceId, localDevReal, tmpfile);
rc = system(cmd);
rc = rc>>8;
unlink(tmpfile);
} else {
logprintfl(EUCAERROR, "could not write to tmpfile for detach XML: %s\n", tmpfile);
rc = 1;
}
exit(rc);
} else {
static int
doAttachVolume ( struct nc_state_t *nc,
ncMetadata *meta,
char *instanceId,
char *volumeId,
char *remoteDev,
char *localDev)
{
int ret = OK, rc;
ncInstance * instance;
char localDevReal[32];
virConnectPtr *conn;
struct stat statbuf;
// fix up format of incoming local dev name, if we need to
ret = convert_dev_names (localDev, localDevReal, NULL);
if (ret)
return ret;
sem_p (inst_sem);
instance = find_instance(&global_instances, instanceId);
sem_v (inst_sem);
if ( instance == NULL )
return NOT_FOUND;
/* try attaching to the Xen domain */
conn = check_hypervisor_conn();
if (conn) {
sem_p(hyp_sem);
virDomainPtr dom = virDomainLookupByName(*conn, instanceId);
sem_v(hyp_sem);
if (dom) {
int err = 0;
char xml [1024];
int is_iscsi_target = 0;
char *local_iscsi_dev;
rc = 0;
if(check_iscsi(remoteDev)) {
is_iscsi_target = 1;
/*get credentials, decrypt them*/
//parse_target(remoteDev);
/*login to target*/
local_iscsi_dev = connect_iscsi_target(nc->connect_storage_cmd_path, remoteDev);
if (!local_iscsi_dev || !strstr(local_iscsi_dev, "/dev")) {
logprintfl(EUCAERROR, "AttachVolume(): failed to connect to iscsi target\n");
rc = 1;
} else {
snprintf (xml, 1024, "<disk type='block'><driver name='phy'/><source dev='%s'/><target dev='%s'/></disk>", local_iscsi_dev, localDevReal);
}
} else {
snprintf (xml, 1024, "<disk type='block'><driver name='phy'/><source dev='%s'/><target dev='%s'/></disk>", remoteDev, localDevReal);
rc = stat(remoteDev, &statbuf);
if (rc) {
logprintfl(EUCAERROR, "AttachVolume(): cannot locate local block device file '%s'\n", remoteDev);
rc = 1;
}
}
if (!rc) {
/* protect Xen calls, just in case */
sem_p (hyp_sem);
err = virDomainAttachDevice (dom, xml);
sem_v (hyp_sem);
if (err) {
logprintfl (EUCAERROR, "AttachVolume() failed (err=%d) XML=%s\n", err, xml);
// rc = doDetachVolume(nc, meta, instanceId, volumeId, remoteDev, localDev, 1);
ret = ERROR;
} else {
logprintfl (EUCAINFO, "attached %s to %s in domain %s\n", remoteDev, localDevReal, instanceId);
}
} else {
ret = ERROR;
}
sem_p(hyp_sem);
virDomainFree(dom);
sem_v(hyp_sem);
if(is_iscsi_target) {
if (local_iscsi_dev) free(local_iscsi_dev);
}
} else {
if (instance->state != BOOTING && instance->state != STAGING) {
logprintfl (EUCAWARN, "warning: domain %s not running on hypervisor, cannot attach device\n", instanceId);
}
ret = ERROR;
}
} else {
ret = ERROR;
}
if (ret==OK) {
ncVolume * volume;
sem_p (inst_sem);
volume = add_volume (instance, volumeId, remoteDev, localDevReal, localDevReal, "attached");
scSaveInstanceInfo(instance); /* to enable NC recovery */
sem_v (inst_sem);
if ( volume == NULL ) {
logprintfl (EUCAFATAL, "ERROR: Failed to save the volume record, aborting volume attachment\n");
return ERROR;
}
}
return ret;
}
static int doInstanceMigrate(struct nc_state_t *nc, ncMetadata *meta, char *instanceId,
char *targetNc, char *taskId, int optType)
{
pthread_t tcb;
virDomainPtr dom_ptr;
ncInstance *instance = NULL;
virConnectPtr *conn = NULL;
virDomainPtr dom;
struct instance_migrate *instmig = calloc(1, sizeof(struct instance_migrate));
if (!instmig) return -1;
if (!instanceId || !targetNc || !taskId)
return -1;
logprintfl(EUCAINFO, HANF_DEBUG"%s: instId [%s] targetIP[%s] taskId[%s] optType[%d]\n",
__func__, instanceId, targetNc, taskId, optType);
sem_p (inst_sem);
instance = find_instance (&global_instances, instanceId);
sem_v (inst_sem);
if ( instance == NULL )
return NOT_FOUND;
change_task_state(instance, MIGRATING);
conn = check_hypervisor_conn();
if (conn==NULL) {
logprintfl(EUCAERROR, HANF_DEBUG"%s: cannot get connection to hypervisor\n", __func__);
change_task_state(instance, FAILED_MIGRATE);
return ERROR;
}
sem_p(hyp_sem);
dom = virDomainLookupByName(*conn, instanceId);
sem_v(hyp_sem);
if (dom == NULL) {
//if (instance->state != BOOTING && instance->state != STAGING) {
if (instance->state == SHUTOFF || instance->state == SHUTDOWN) { // and some other status ?
logprintfl (EUCAWARN, HANF_DEBUG"%s: domain %s not running on hypervisor, cannot live migrate\n", instanceId);
}
change_task_state(instance, FAILED_MIGRATE);
return ERROR;
}
instmig->dom = dom;
instmig->targetNc = strdup(targetNc);
instmig->taskId = strdup(taskId);
instmig->instanceId = strdup(instanceId);
instmig->instance = instance;
if(pthread_create(&tcb, NULL, instance_migrate_thread, (void *)instmig)) {
logprintfl (EUCAERROR, HANF_DEBUG"%s: failed to spawn a instance_migrate_thread thread\n", __func__);
sem_p(hyp_sem);
virDomainFree(instmig->dom);
sem_v(hyp_sem);
free(instmig->targetNc);
free(instmig->instanceId);
free(instmig->taskId);
free(instmig);
return ERROR_FATAL;
}
return 0;
}
/* thread that does the actual reboot */
static void * rebooting_thread (void *arg)
{
virConnectPtr *conn;
ncInstance * instance = (ncInstance *)arg;
struct stat statbuf;
int rc = 0;
// RESTARTING,
// FAILED_RESTART,
// SUCCESS_RESTART
logprintfl (EUCADEBUG, "{%u} spawning rebooting thread\n", (unsigned int)pthread_self());
char * xml = file2str (instance->libvirtFilePath);
if (xml == NULL) {
logprintfl (EUCAERROR, "cannot obtain XML file %s\n", instance->libvirtFilePath);
return NULL;
}
conn = check_hypervisor_conn();
if (! conn) {
logprintfl (EUCAERROR, "cannot restart instance %s, abandoning it\n", instance->instanceId);
change_state (instance, SHUTOFF);
free (xml);
return NULL;
}
sem_p(inst_sem);
change_task_state (instance, REBOOTING);
copy_instances ();
sem_v(inst_sem);
sem_p (hyp_sem);
virDomainPtr dom = virDomainLookupByName(*conn, instance->instanceId);
sem_v (hyp_sem);
if (dom == NULL) {
free (xml);
sem_p(inst_sem);
change_task_state (instance, FAILED_REBOOT);
sem_v(inst_sem);
return NULL;
}
sem_p (hyp_sem);
// for KVM, must stop and restart the instance
int error = virDomainDestroy (dom); // TODO: change to Shutdown? TODO: is this synchronous?
virDomainFree(dom);
sem_v (hyp_sem);
if (error) {
free (xml);
sem_p(inst_sem);
change_task_state (instance, FAILED_REBOOT);
sem_v(inst_sem);
return NULL;
}
#if 0
// domain is now shut down, create a new one with the same XML
sem_p (hyp_sem);
dom = virDomainCreateLinux (*conn, xml, 0);
sem_v (hyp_sem);
free (xml);
char *remoteDevStr=NULL;
// re-attach each volume previously attached
for (int i=0; i < EUCA_MAX_VOLUMES; ++i) {
ncVolume * volume = &instance->volumes[i];
if (strcmp (volume->stateName, VOL_STATE_ATTACHED) &&
strcmp (volume->stateName, VOL_STATE_ATTACHING))
continue; // skip the entry unless attached or attaching
char attach_xml[1024];
int rc;
// get credentials, decrypt them
remoteDevStr = get_iscsi_target (volume->remoteDev);
if (!remoteDevStr || !strstr(remoteDevStr, "/dev")) {
logprintfl(EUCAERROR, "Reattach-volume: failed to get local name of host iscsi device\n");
rc = 1;
} else {
rc = gen_libvirt_attach_xml (volume->volumeId,
instance,
volume->localDevReal,
remoteDevStr,
attach_xml,
sizeof(attach_xml));
}
if (remoteDevStr)
free (remoteDevStr);
if (!rc) {
int err;
sem_p (hyp_sem);
err = virDomainAttachDevice (dom, attach_xml);
sem_v (hyp_sem);
if (err) {
logprintfl (EUCAERROR, "virDomainAttachDevice() failed (err=%d) XML=%s\n", err, attach_xml);
} else {
logprintfl (EUCAINFO, "reattached '%s' to '%s' in domain %s\n", volume->remoteDev, volume->localDevReal, instance->instanceId);
}
}
}
#endif
int ret=restore_and_volume_instance(instance->instanceId); //20130122-th
if (ret==1) {
logprintfl (EUCAERROR, "Failed to REBOOT instance %s\n", instance->instanceId);
change_state (instance, SHUTOFF);
sem_p(inst_sem);
change_task_state (instance, FAILED_REBOOT);
sem_v(inst_sem);
return NULL;
}
sem_p(inst_sem);
change_task_state (instance, SUCCESS_REBOOT);
save_instance_struct (instance);
copy_instances();
sem_v (inst_sem);
return NULL;
}
static int doMigrateInstance(struct nc_state_t *nc, ncMetadata *meta, char *instanceId, char *target)
{
int ret=OK;
char remoteURI[CHAR_BUFFER_SIZE];
virConnectPtr *conn, dst;
ncInstance *instance;
logprintfl(EUCADEBUG, "doMigrateInstance() in default handler invoked\n");
conn = check_hypervisor_conn();
if (!conn) {
logprintfl(EUCAERROR, "doMigrateInstance() cannot connect to hypervisor\n");
}
if (target || !strcmp(target, "")) {
getRemoteURI(nc, target, remoteURI, CHAR_BUFFER_SIZE);
logprintfl(EUCADEBUG, "doMigrateInstance(): connecting to remote hypervisor\n");
dst = virConnectOpen(remoteURI);
if (!dst) {
logprintfl(EUCAERROR, "doMigrateInstance(): Connection to remote Hypervisor failed (URI: %s)\n", remoteURI);
} else {
logprintfl(EUCADEBUG, "doMigrateInstance(): Connected to %s\n", remoteURI);
}
} else {
logprintfl(EUCAERROR, "doMigrateInstance(): no migration target\n");
return (ERROR);
}
sem_p (inst_sem);
instance = find_instance(&global_instances, instanceId);
sem_v (inst_sem);
if (instance == NULL) {
logprintfl(EUCAERROR, "doMigrateInstance(): instance not found\n");
return (NOT_FOUND);
}
if (conn && dst) {
sem_p(hyp_sem);
virDomainPtr dom = virDomainLookupByName(*conn, instanceId);
sem_v(hyp_sem);
if (dom) {
sem_p (hyp_sem);
if (virDomainMigrate (dom, dst, VIR_MIGRATE_LIVE, NULL, NULL, 0))
logprintfl (EUCAINFO, "doMigrateInstance(): migrated instance %s\n", instanceId);
else
ret = ERROR;
sem_v (hyp_sem);
}
else {
logprintfl (EUCAWARN, "warning: domain %s to be migrated not running on hypervisor\n", instanceId);
ret = ERROR;
}
}
else {
logprintfl(EUCAERROR, "doMigrateInstance(): Migrating %s failed\n", instanceId);
ret = ERROR;
}
if (ret == OK) {
sem_p (inst_sem);
instance = find_instance(&global_instances, instanceId);
logprintfl(EUCADEBUG, "doMigrateInstance(): removing instance from global_instances\n");
if (remove_instance (&global_instances, instance) != OK) {
logprintfl(EUCAERROR, "doMigrateInstance(): cannot remove instance from global_instances\n");
ret = ERROR;
}
sem_v (inst_sem);
}
return (ret);
}
//!
//!
//!
//! @param[in] arg
//!
//! @return Always returns NULL
//!
static void *startup_thread(void *arg)
{
int fd = 0;
int iter = 0;
long long tid = (*(long long *)arg);
virDomainPtr dom = NULL;
char file_name[1024] = { 0 };
char xml[4096] = { 0 };
char *xml_template =
"<?xml version='1.0' encoding='UTF-8'?>"
"<domain type='kvm'>"
" <name>tortura-%04lld</name>"
" <description>Eucalyptus instance i-XXX</description>"
" <os>"
" <type>hvm</type>"
" </os>"
" <features>"
" <acpi/>"
" </features>"
" <clock offset='localtime'/>"
" <on_poweroff>destroy</on_poweroff>"
" <on_reboot>restart</on_reboot>"
" <on_crash>destroy</on_crash>"
" <vcpu>1</vcpu>"
" <memory>52428</memory>"
" <devices>" " <disk device='disk'>" " <source file='%s'/>" " <target bus='virtio' dev='vda'/>" " </disk>" " </devices>" "</domain>";
snprintf(file_name, sizeof(file_name), "%s/%s-%lld", get_current_dir_name(), DUMMY_DISK_FILE, tid);
umask(0000);
if ((fd = open(file_name, O_RDWR | O_CREAT | O_TRUNC, 0666)) < 0) {
printf("failed to create %s\n", DUMMY_DISK_FILE);
perror("libvirt_tortura");
exit(1);
}
if (lseek(fd, DUMMY_DISK_SIZE_BYTES, SEEK_SET) == ((off_t) - 1)) {
printf("failed to seek in %s\n", DUMMY_DISK_FILE);
perror("libvirt_tortura");
exit(1);
}
if (write(fd, "x", 1) != 1) {
printf("failed to write to %s\n", DUMMY_DISK_FILE);
perror("libvirt_tortura");
exit(1);
}
close(fd);
sync();
check_hypervisor_conn();
for (iter = 0; iter < SITERS; iter++) {
printf("startup thread %lld starting iteration %d\n", tid, iter);
snprintf(xml, sizeof(xml), xml_template, tid, file_name);
pthread_mutex_lock(&check_mutex);
{
dom = virDomainCreateLinux(conn, xml, 0);
}
pthread_mutex_unlock(&check_mutex);
sleep(3);
if (dom == NULL) {
printf("ERROR: failed to start domain\n");
continue;
}
pthread_mutex_lock(&check_mutex);
{
virDomainDestroy(dom);
virDomainFree(dom);
dom = NULL;
}
pthread_mutex_unlock(&check_mutex);
}
unlink(file_name);
return (NULL);
}
//!
//! Defines the thread that does the actual reboot of an instance.
//!
//! @param[in] arg a transparent pointer to the argument passed to this thread handler
//!
//! @return Always return NULL
//!
static void *rebooting_thread(void *arg)
{
#define REATTACH_RETRIES 3
int i = 0;
int err = 0;
int error = 0;
int rc = 0;
int log_level_for_devstring = EUCATRACE;
char *xml = NULL;
char *remoteDevStr = NULL;
char path[MAX_PATH] = "";
char lpath[MAX_PATH] = "";
char resourceName[1][MAX_SENSOR_NAME_LEN] = { {0} };
char resourceAlias[1][MAX_SENSOR_NAME_LEN] = { {0} };
ncVolume *volume = NULL;
ncInstance *instance = ((ncInstance *) arg);
virDomainPtr dom = NULL;
virConnectPtr *conn = NULL;
logprintfl(EUCADEBUG, "[%s] spawning rebooting thread\n", instance->instanceId);
if ((xml = file2str(instance->libvirtFilePath)) == NULL) {
logprintfl(EUCAERROR, "[%s] cannot obtain instance XML file %s\n", instance->instanceId, instance->libvirtFilePath);
return NULL;
}
if ((conn = check_hypervisor_conn()) == NULL) {
logprintfl(EUCAERROR, "[%s] cannot restart instance %s, abandoning it\n", instance->instanceId, instance->instanceId);
change_state(instance, SHUTOFF);
EUCA_FREE(xml);
return NULL;
}
sem_p(hyp_sem);
{
dom = virDomainLookupByName(*conn, instance->instanceId);
}
sem_v(hyp_sem);
if (dom == NULL) {
EUCA_FREE(xml);
return NULL;
}
sem_p(hyp_sem);
{
// for KVM, must stop and restart the instance
logprintfl(EUCADEBUG, "[%s] destroying domain\n", instance->instanceId);
error = virDomainDestroy(dom); // @todo change to Shutdown? is this synchronous?
virDomainFree(dom);
}
sem_v(hyp_sem);
if (error) {
EUCA_FREE(xml);
return NULL;
}
// Add a shift to values of three of the metrics: ones that
// drop back to zero after a reboot. The shift, which is based
// on the latest value, ensures that values sent upstream do
// not go backwards .
sensor_shift_metric(instance->instanceId, "CPUUtilization");
sensor_shift_metric(instance->instanceId, "NetworkIn");
sensor_shift_metric(instance->instanceId, "NetworkOut");
// domain is now shut down, create a new one with the same XML
sem_p(hyp_sem);
{
logprintfl(EUCAINFO, "[%s] rebooting\n", instance->instanceId);
dom = virDomainCreateLinux(*conn, xml, 0);
}
sem_v(hyp_sem);
EUCA_FREE(xml);
euca_strncpy(resourceName[0], instance->instanceId, MAX_SENSOR_NAME_LEN);
sensor_refresh_resources(resourceName, resourceAlias, 1); // refresh stats so we set base value accurately
// re-attach each volume previously attached
for (i = 0; i < EUCA_MAX_VOLUMES; ++i) {
volume = &instance->volumes[i];
if (strcmp(volume->stateName, VOL_STATE_ATTACHED) && strcmp(volume->stateName, VOL_STATE_ATTACHING))
continue; // skip the entry unless attached or attaching
logprintfl(EUCADEBUG, "[%s] volumes [%d] = '%s'\n", instance->instanceId, i, volume->stateName);
// get credentials, decrypt them
remoteDevStr = get_iscsi_target(volume->remoteDev);
if (!remoteDevStr || !strstr(remoteDevStr, "/dev")) {
logprintfl(EUCAERROR, "[%s] failed to get local name of host iscsi device when re-attaching\n", instance->instanceId);
rc = 1;
} else {
// set the path
snprintf(path, sizeof(path), EUCALYPTUS_VOLUME_XML_PATH_FORMAT, instance->instancePath, volume->volumeId); // vol-XXX.xml
snprintf(lpath, sizeof(lpath), EUCALYPTUS_VOLUME_LIBVIRT_XML_PATH_FORMAT, instance->instancePath, volume->volumeId); // vol-XXX-libvirt.xml
// read in libvirt XML, which may have been modified by the hook above
if ((xml = file2str(lpath)) == NULL) {
logprintfl(EUCAERROR, "[%s][%s] failed to read volume XML from %s\n", instance->instanceId, volume->volumeId, lpath);
rc = 1;
}
}
EUCA_FREE(remoteDevStr);
if (!rc) {
// zhill - wrap with retry in case libvirt is dumb.
err = 0;
for (i = 1; i < REATTACH_RETRIES; i++) {
// protect libvirt calls because we've seen problems during concurrent libvirt invocations
sem_p(hyp_sem);
{
err = virDomainAttachDevice(dom, xml);
}
sem_v(hyp_sem);
if (err) {
logprintfl(EUCAERROR, "[%s][%s] failed to reattach volume (attempt %d of %d)\n", instance->instanceId, volume->volumeId, i,
REATTACH_RETRIES);
logprintfl(EUCADEBUG, "[%s][%s] error from virDomainAttachDevice: %d xml: %s\n", instance->instanceId, volume->volumeId, err,
xml);
sleep(3); // sleep a bit and retry
} else {
logprintfl(EUCAINFO, "[%s][%s] volume reattached as '%s'\n", instance->instanceId, volume->volumeId, volume->localDevReal);
break;
}
}
log_level_for_devstring = EUCATRACE;
if (err)
log_level_for_devstring = EUCADEBUG;
logprintfl(log_level_for_devstring, "[%s][%s] remote device string: %s\n", instance->instanceId, volume->volumeId, volume->remoteDev);
}
EUCA_FREE(xml);
}
if (dom == NULL) {
logprintfl(EUCAERROR, "[%s] failed to restart instance\n", instance->instanceId);
change_state(instance, SHUTOFF);
return NULL;
}
sem_p(hyp_sem);
{
virDomainFree(dom);
}
sem_v(hyp_sem);
return NULL;
#undef REATTACH_RETRIES
}
static void
refresh_instance_info( struct nc_state_t *nc,
ncInstance *instance)
{
int now = instance->state;
if (! check_hypervisor_conn ())
return;
/* no need to bug for domains without state on Hypervisor */
if (now==TEARDOWN || now==STAGING)
return;
sem_p(hyp_sem);
virDomainPtr dom = virDomainLookupByName (nc_state.conn, instance->instanceId);
sem_v(hyp_sem);
if (dom == NULL) { /* hypervisor doesn't know about it */
if (now==RUNNING ||
now==BLOCKED ||
now==PAUSED ||
now==SHUTDOWN) {
/* Most likely the user has shut it down from the inside */
if (instance->retries) {
instance->retries--;
logprintfl (EUCAWARN, "warning: hypervisor failed to find domain %s, will retry %d more times\n", instance->instanceId, instance->retries);
} else {
logprintfl (EUCAWARN, "warning: hypervisor failed to find domain %s, assuming it was shut off\n", instance->instanceId);
change_state (instance, SHUTOFF);
}
}
/* else 'now' stays in SHUTFOFF, BOOTING, CANCELED, or CRASHED */
return;
}
virDomainInfo info;
sem_p(hyp_sem);
int error = virDomainGetInfo(dom, &info);
sem_v(hyp_sem);
if (error < 0 || info.state == VIR_DOMAIN_NOSTATE) {
logprintfl (EUCAWARN, "warning: failed to get informations for domain %s\n", instance->instanceId);
/* what to do? hopefully we'll find out more later */
sem_p(hyp_sem);
virDomainFree (dom);
sem_v(hyp_sem);
return;
}
int xen = info.state;
switch (now) {
case BOOTING:
case RUNNING:
case BLOCKED:
case PAUSED:
/* change to state, whatever it happens to be */
change_state (instance, xen);
break;
case SHUTDOWN:
case SHUTOFF:
case CRASHED:
if (xen==RUNNING ||
xen==BLOCKED ||
xen==PAUSED) {
/* cannot go back! */
logprintfl (EUCAWARN, "warning: detected prodigal domain %s, terminating it\n", instance->instanceId);
sem_p (hyp_sem);
virDomainDestroy (dom);
sem_v (hyp_sem);
} else {
change_state (instance, xen);
}
break;
default:
logprintfl (EUCAERROR, "error: refresh...(): unexpected state (%d) for instance %s\n", now, instance->instanceId);
return;
}
sem_p(hyp_sem);
virDomainFree(dom);
sem_v(hyp_sem);
/* if instance is running, try to find out its IP address */
if (instance->state==RUNNING ||
instance->state==BLOCKED ||
instance->state==PAUSED) {
char *ip=NULL;
int rc;
if (!strncmp(instance->ncnet.publicIp, "0.0.0.0", 24)) {
if (!strcmp(nc_state.vnetconfig->mode, "SYSTEM") || !strcmp(nc_state.vnetconfig->mode, "STATIC")) {
rc = mac2ip(nc_state.vnetconfig, instance->ncnet.privateMac, &ip);
if (!rc) {
if(ip) {
logprintfl (EUCAINFO, "discovered public IP %s for instance %s\n", ip, instance->instanceId);
strncpy(instance->ncnet.publicIp, ip, 24);
free(ip);
}
}
}
}
if (!strncmp(instance->ncnet.privateIp, "0.0.0.0", 24)) {
rc = mac2ip(nc_state.vnetconfig, instance->ncnet.privateMac, &ip);
if (!rc) {
if(ip) {
logprintfl (EUCAINFO, "discovered private IP %s for instance %s\n", ip, instance->instanceId);
strncpy(instance->ncnet.privateIp, ip, 24);
free(ip);
}
}
}
}
}
