//! //! prints contents of an arbitrary file (at file_path) using logprintfl, thus dumping //! its contents into a log //! //! @param[in] debug_level the log level //! @param[in] file_path a path to the file to dump info into //! //! @return the number of lines written into the file //! //! @pre The file_path field must not be null and should be a valid file //! //! @post the content of the file is added to the current log at the given log level. //! int logcat(int debug_level, const char *file_path) { int l = -0; int got = 0; char buf[LOGLINEBUF] = ""; FILE *fp = NULL; if ((fp = fopen(file_path, "r")) == NULL) return (got); while (fgets(buf, LOGLINEBUF, fp)) { if ((l = strlen(buf)) < 0) break; if (((l + 1) < LOGLINEBUF) && (buf[l - 1] != '\n')) { buf[l++] = '\n'; buf[l] = '\0'; } EUCALOG(debug_level, buf); got += l; } fclose(fp); return (got); }
//! //! Defines the thread that does the actual reboot of an instance. //! //! @param[in] arg a transparent pointer to the argument passed to this thread handler //! //! @return Always return NULL //! static void *rebooting_thread(void *arg) { #define REATTACH_RETRIES 3 int i = 0; int err = 0; int error = 0; int rc = 0; int log_level_for_devstring = EUCA_LOG_TRACE; char *xml = NULL; char *remoteDevStr = NULL; char path[MAX_PATH] = ""; char lpath[MAX_PATH] = ""; char resourceName[1][MAX_SENSOR_NAME_LEN] = { {0} }; char resourceAlias[1][MAX_SENSOR_NAME_LEN] = { {0} }; ncVolume *volume = NULL; ncInstance *instance = ((ncInstance *) arg); virDomainPtr dom = NULL; virConnectPtr *conn = NULL; LOGDEBUG("[%s] spawning rebooting thread\n", instance->instanceId); if ((xml = file2str(instance->libvirtFilePath)) == NULL) { LOGERROR("[%s] cannot obtain instance XML file %s\n", instance->instanceId, instance->libvirtFilePath); return NULL; } if ((conn = check_hypervisor_conn()) == NULL) { LOGERROR("[%s] cannot restart instance %s, abandoning it\n", instance->instanceId, instance->instanceId); change_state(instance, SHUTOFF); EUCA_FREE(xml); return NULL; } sem_p(hyp_sem); { dom = virDomainLookupByName(*conn, instance->instanceId); } sem_v(hyp_sem); if (dom == NULL) { EUCA_FREE(xml); return NULL; } sem_p(hyp_sem); { // for KVM, must stop and restart the instance LOGDEBUG("[%s] destroying domain\n", instance->instanceId); error = virDomainDestroy(dom); // @todo change to Shutdown? is this synchronous? virDomainFree(dom); } sem_v(hyp_sem); if (error) { EUCA_FREE(xml); return NULL; } // Add a shift to values of three of the metrics: ones that // drop back to zero after a reboot. The shift, which is based // on the latest value, ensures that values sent upstream do // not go backwards . sensor_shift_metric(instance->instanceId, "CPUUtilization"); sensor_shift_metric(instance->instanceId, "NetworkIn"); sensor_shift_metric(instance->instanceId, "NetworkOut"); // domain is now shut down, create a new one with the same XML sem_p(hyp_sem); { LOGINFO("[%s] rebooting\n", instance->instanceId); dom = virDomainCreateLinux(*conn, xml, 0); } sem_v(hyp_sem); EUCA_FREE(xml); euca_strncpy(resourceName[0], instance->instanceId, MAX_SENSOR_NAME_LEN); sensor_refresh_resources(resourceName, resourceAlias, 1); // refresh stats so we set base value accurately // re-attach each volume previously attached for (i = 0; i < EUCA_MAX_VOLUMES; ++i) { volume = &instance->volumes[i]; if (strcmp(volume->stateName, VOL_STATE_ATTACHED) && strcmp(volume->stateName, VOL_STATE_ATTACHING)) continue; // skip the entry unless attached or attaching LOGDEBUG("[%s] volumes [%d] = '%s'\n", instance->instanceId, i, volume->stateName); // get credentials, decrypt them remoteDevStr = get_iscsi_target(volume->remoteDev); if (!remoteDevStr || !strstr(remoteDevStr, "/dev")) { LOGERROR("[%s] failed to get local name of host iscsi device when re-attaching\n", instance->instanceId); rc = 1; } else { // set the path snprintf(path, sizeof(path), EUCALYPTUS_VOLUME_XML_PATH_FORMAT, instance->instancePath, volume->volumeId); // vol-XXX.xml snprintf(lpath, sizeof(lpath), EUCALYPTUS_VOLUME_LIBVIRT_XML_PATH_FORMAT, instance->instancePath, volume->volumeId); // vol-XXX-libvirt.xml // read in libvirt XML, which may have been modified by the hook above if ((xml = file2str(lpath)) == NULL) { LOGERROR("[%s][%s] failed to read volume XML from %s\n", instance->instanceId, volume->volumeId, lpath); rc = 1; } } EUCA_FREE(remoteDevStr); if (!rc) { // zhill - wrap with retry in case libvirt is dumb. err = 0; for (i = 1; i < REATTACH_RETRIES; i++) { // protect libvirt calls because we've seen problems during concurrent libvirt invocations sem_p(hyp_sem); { err = virDomainAttachDevice(dom, xml); } sem_v(hyp_sem); if (err) { LOGERROR("[%s][%s] failed to reattach volume (attempt %d of %d)\n", instance->instanceId, volume->volumeId, i, REATTACH_RETRIES); LOGDEBUG("[%s][%s] error from virDomainAttachDevice: %d xml: %s\n", instance->instanceId, volume->volumeId, err, xml); sleep(3); // sleep a bit and retry } else { LOGINFO("[%s][%s] volume reattached as '%s'\n", instance->instanceId, volume->volumeId, volume->localDevReal); break; } } log_level_for_devstring = EUCA_LOG_TRACE; if (err) log_level_for_devstring = EUCA_LOG_DEBUG; EUCALOG(log_level_for_devstring, "[%s][%s] remote device string: %s\n", instance->instanceId, volume->volumeId, volume->remoteDev); } EUCA_FREE(xml); } if (dom == NULL) { LOGERROR("[%s] failed to restart instance\n", instance->instanceId); change_state(instance, SHUTOFF); return NULL; } sem_p(hyp_sem); { virDomainFree(dom); } sem_v(hyp_sem); return NULL; #undef REATTACH_RETRIES }