Esempio n. 1
0
//!
//! Handles the instance migration request.
//!
//! @param[in]  nc a pointer to the node controller (NC) state
//! @param[in]  pMeta a pointer to the node controller (NC) metadata structure
//! @param[in]  instances metadata for the instance to migrate to destination
//! @param[in]  instancesLen number of instances in the instance list
//! @param[in]  action IP of the destination Node Controller
//! @param[in]  credentials credentials that enable the migration
//!
//! @return EUCA_OK on success or EUCA_*ERROR on failure
//!
//! @pre
//!
//! @post
static int doMigrateInstances(struct nc_state_t *nc, ncMetadata * pMeta, ncInstance ** instances, int instancesLen, char *action, char *credentials, char ** resourceLocations, int resourceLocationsLen)
{
    int ret = EUCA_OK;
    int credentials_prepared = 0;
    char *libvirt_xml_modified = NULL;

    if (instancesLen <= 0) {
        LOGERROR("called with invalid instancesLen (%d)\n", instancesLen);
        pMeta->replyString = strdup("internal error (invalid instancesLen)");
        return (EUCA_INVALID_ERROR);
    }

    LOGDEBUG("verifying %d instance[s] for migration...\n", instancesLen);
    for (int inst_idx = 0; inst_idx < instancesLen; inst_idx++) {
        LOGDEBUG("verifying instance # %d...\n", inst_idx);
        if (instances[inst_idx]) {
            ncInstance *instance_idx = instances[inst_idx];
            LOGDEBUG("[%s] proposed migration action '%s' (%s > %s) [creds=%s]\n", SP(instance_idx->instanceId), SP(action), SP(instance_idx->migration_src),
                     SP(instance_idx->migration_dst), (instance_idx->migration_credentials == NULL) ? "UNSET" : "present");
        } else {
            pMeta->replyString = strdup("internal error (instance count mismatch)");
            LOGERROR("Mismatch between migration instance count (%d) and length of instance list\n", instancesLen);
            return (EUCA_ERROR);
        }
    }

    // TO-DO: Optimize the location of this loop, placing it inside various conditionals below?
    for (int inst_idx = 0; inst_idx < instancesLen; inst_idx++) {
        ncInstance *instance_req = instances[inst_idx];
        char *sourceNodeName = instance_req->migration_src;
        char *destNodeName = instance_req->migration_dst;

        LOGDEBUG("[%s] processing instance # %d (%s > %s)\n", instance_req->instanceId, inst_idx, instance_req->migration_src, instance_req->migration_dst);

        // this is a call to the source of migration
        if (!strcmp(pMeta->nodeName, sourceNodeName)) {

            // locate the instance structure
            ncInstance *instance;
            sem_p(inst_sem);
            {
                instance = find_instance(&global_instances, instance_req->instanceId);
            }
            sem_v(inst_sem);
            if (instance == NULL) {
                LOGERROR("[%s] cannot find instance\n", instance_req->instanceId);
                pMeta->replyString = strdup("failed to locate instance to migrate");
                return (EUCA_NOT_FOUND_ERROR);
            }

            if (strcmp(action, "prepare") == 0) {
                sem_p(inst_sem);
                instance->migration_state = MIGRATION_PREPARING;
                euca_strncpy(instance->migration_src, sourceNodeName, HOSTNAME_SIZE);
                euca_strncpy(instance->migration_dst, destNodeName, HOSTNAME_SIZE);
                euca_strncpy(instance->migration_credentials, credentials, CREDENTIAL_SIZE);
                instance->migrationTime = time(NULL);
                update_resource_locations(&(instance->params), resourceLocations, resourceLocationsLen);
                save_instance_struct(instance);
                copy_instances();
                sem_v(inst_sem);

                // Establish migration-credential keys if this is the first instance preparation for this host.
                LOGINFO("[%s] migration source preparing %s > %s [creds=%s]\n", SP(instance->instanceId), SP(instance->migration_src), SP(instance->migration_dst),
                        (instance->migration_credentials == NULL) ? "UNSET" : "present");
                if (!credentials_prepared) {
                    if (generate_migration_keys(sourceNodeName, credentials, TRUE, instance) != EUCA_OK) {
                        pMeta->replyString = strdup("internal error (migration credentials generation failed)");
                        return (EUCA_SYSTEM_ERROR);
                    } else {
                        credentials_prepared++;
                    }
                }
                sem_p(inst_sem);
                instance->migration_state = MIGRATION_READY;
                save_instance_struct(instance);
                copy_instances();
                sem_v(inst_sem);

            } else if (strcmp(action, "commit") == 0) {

                sem_p(inst_sem);
                if (instance->migration_state == MIGRATION_IN_PROGRESS) {
                    LOGWARN("[%s] duplicate request to migration source to initiate %s > %s (already migrating)\n", instance->instanceId,
                            instance->migration_src, instance->migration_dst);
                    sem_v(inst_sem);
                    return (EUCA_DUPLICATE_ERROR);
                } else if (instance->migration_state != MIGRATION_READY) {
                    LOGERROR("[%s] request to commit migration %s > %s when source migration_state='%s' (not 'ready')\n", instance->instanceId,
                             SP(sourceNodeName), SP(destNodeName), migration_state_names[instance->migration_state]);
                    sem_v(inst_sem);
                    return (EUCA_UNSUPPORTED_ERROR);
                }
                instance->migration_state = MIGRATION_IN_PROGRESS;
                outgoing_migrations_in_progress++;
                LOGINFO("[%s] migration source initiating %s > %s [creds=%s] (1 of %d active outgoing migrations)\n", instance->instanceId, instance->migration_src,
                        instance->migration_dst, (instance->migration_credentials == NULL) ? "UNSET" : "present", outgoing_migrations_in_progress);
                save_instance_struct(instance);
                copy_instances();
                sem_v(inst_sem);

                // since migration may take a while, we do them in a thread
                pthread_t tcb = { 0 };
                if (pthread_create(&tcb, NULL, migrating_thread, (void *)instance)) {
                    LOGERROR("[%s] failed to spawn a migration thread\n", instance->instanceId);
                    return (EUCA_THREAD_ERROR);
                }
                set_corrid_pthread(get_corrid() != NULL ? get_corrid()->correlation_id : NULL, tcb);
                if (pthread_detach(tcb)) {
                    LOGERROR("[%s] failed to detach the migration thread\n", instance->instanceId);
                    return (EUCA_THREAD_ERROR);
                }
            } else if (strcmp(action, "rollback") == 0) {
                if ((instance->migration_state == MIGRATION_READY) || (instance->migration_state == MIGRATION_PREPARING)) {
                    LOGINFO("[%s] rolling back migration (%s > %s) on source\n", instance->instanceId, instance->migration_src, instance->migration_dst);
                    sem_p(inst_sem);
                    migration_rollback(instance);
                    sem_v(inst_sem);
                } else {
                    LOGINFO("[%s] ignoring request to roll back migration on source with instance in state %s(%s) -- duplicate rollback request?\n", instance->instanceId,
                            instance->stateName, migration_state_names[instance->migration_state]);
                }
            } else {
                LOGERROR("[%s] action '%s' is not valid\n", instance->instanceId, action);
                return (EUCA_INVALID_ERROR);
            }

        } else if (!strcmp(pMeta->nodeName, destNodeName)) {    // this is a migrate request to destination

            if (!strcmp(action, "commit")) {
                LOGERROR("[%s] action '%s' for migration (%s > %s) is not valid on destination node\n", instance_req->instanceId, action, SP(sourceNodeName), SP(destNodeName));
                return (EUCA_UNSUPPORTED_ERROR);
            } else if (!strcmp(action, "rollback")) {
                LOGINFO("[%s] rolling back migration (%s > %s) on destination\n", instance_req->instanceId, SP(sourceNodeName), SP(destNodeName));
                sem_p(inst_sem);
                {
                    ncInstance *instance = find_instance(&global_instances, instance_req->instanceId);
                    if (instance != NULL) {
                        LOGDEBUG("[%s] marked for cleanup\n", instance->instanceId);
                        change_state(instance, SHUTOFF);
                        instance->migration_state = MIGRATION_CLEANING;
                        save_instance_struct(instance);
                    }
                }
                sem_v(inst_sem);
                return EUCA_OK;
            } else if (strcmp(action, "prepare") != 0) {
                LOGERROR("[%s] action '%s' is not valid or not implemented\n", instance_req->instanceId, action);
                return (EUCA_INVALID_ERROR);
            }
            // Everything from here on is specific to "prepare" on a destination.

            // allocate a new instance struct
            ncInstance *instance = clone_instance(instance_req);
            if (instance == NULL) {
                LOGERROR("[%s] could not allocate instance struct\n", instance_req->instanceId);
                goto failed_dest;
            }

            sem_p(inst_sem);
            instance->migration_state = MIGRATION_PREPARING;
            instance->migrationTime = time(NULL); //In preparing state, so set migrationTime.
            euca_strncpy(instance->migration_src, sourceNodeName, HOSTNAME_SIZE);
            euca_strncpy(instance->migration_dst, destNodeName, HOSTNAME_SIZE);
            euca_strncpy(instance->migration_credentials, credentials, CREDENTIAL_SIZE);
            update_resource_locations(&(instance->params), resourceLocations, resourceLocationsLen);
            sem_v(inst_sem);

            // Establish migration-credential keys.
            LOGINFO("[%s] migration destination preparing %s > %s [creds=%s]\n", instance->instanceId, SP(instance->migration_src), SP(instance->migration_dst),
                    (instance->migration_credentials == NULL) ? "UNSET" : "present");
            // First, call config-file modification script to authorize source node.
            LOGDEBUG("[%s] authorizing migration source node %s\n", instance->instanceId, instance->migration_src);
            if (authorize_migration_keys("-a", instance->migration_src, instance->migration_credentials, instance, TRUE) != EUCA_OK) {
                goto failed_dest;
            }
            // Then, generate keys and restart libvirtd.
            if (generate_migration_keys(instance->migration_dst, instance->migration_credentials, TRUE, instance) != EUCA_OK) {
                goto failed_dest;
            }
            int error;

            //Fix for EUCA-10433, need instance struct in global_instances prior to doing volume ops
            //The monitor thread will now pick up the instance, so the migrationTime must be set
            sem_p(inst_sem);
            save_instance_struct(instance);
            error = add_instance(&global_instances, instance);
            copy_instances(); 
            sem_v(inst_sem);
            if (error) {
                if (error == EUCA_DUPLICATE_ERROR) {
                    LOGINFO("[%s] instance struct already exists (from previous migration?), deleting and re-adding...\n", instance->instanceId);
                    error = remove_instance(&global_instances, instance);
                    if (error) {
                        LOGERROR("[%s] could not replace (remove) instance struct, failing...\n", instance->instanceId);
                        goto failed_dest;
                    }
                    error = add_instance(&global_instances, instance);
                    if (error) {
                        LOGERROR("[%s] could not replace (add) instance struct, failing...\n", instance->instanceId);
                        goto failed_dest;
                    }
                } else {
                    LOGERROR("[%s] could not add instance struct, failing...\n", instance->instanceId);
                    goto failed_dest;
                }
            }
            
            if (vbr_parse(&(instance->params), pMeta) != EUCA_OK) {
                goto failed_dest;
            }
            // set up networking
            char brname[IF_NAME_LEN] = "";
            if (!strcmp(nc->pEucaNet->sMode, NETMODE_MANAGED)) {
                snprintf(brname, IF_NAME_LEN, "%s", instance->groupIds[0]);
            } else {
                snprintf(brname, IF_NAME_LEN, "%s", nc->pEucaNet->sBridgeDevice);
            }
            euca_strncpy(instance->params.guestNicDeviceName, brname, sizeof(instance->params.guestNicDeviceName));
            // TODO: move stuff in startup_thread() into a function?
            
            set_instance_params(instance);

            if ((error = create_instance_backing(instance, TRUE))   // create files that back the disks
                || (error = gen_instance_xml(instance)) // create euca-specific instance XML file
                || (error = gen_libvirt_instance_xml(instance))) {  // transform euca-specific XML into libvirt XML
                LOGERROR("[%s] failed to prepare images for migrating instance (error=%d)\n", instance->instanceId, error);
                goto failed_dest;
            }

            // attach any volumes
            for (int v = 0; v < EUCA_MAX_VOLUMES; v++) {
                ncVolume *volume = &instance->volumes[v];
                if (strcmp(volume->stateName, VOL_STATE_ATTACHED) && strcmp(volume->stateName, VOL_STATE_ATTACHING))
                    continue;          // skip the entry unless attached or attaching
                LOGDEBUG("[%s] volumes [%d] = '%s'\n", instance->instanceId, v, volume->stateName);

                ebs_volume_data *vol_data = NULL;
                char *libvirt_xml = NULL;
                char serial[128];
                char bus[16];
                set_serial_and_bus(volume->volumeId, volume->devName, serial, sizeof(serial), bus, sizeof(bus));

                if ((ret = connect_ebs(volume->devName, serial, bus, nc, instance->instanceId, volume->volumeId, volume->attachmentToken, &libvirt_xml, &vol_data)) != EUCA_OK) {
                    goto unroll;
                }
                // update the volume struct with connection string obtained from SC
                euca_strncpy(volume->connectionString, vol_data->connect_string, sizeof(volume->connectionString));
                // save volume info into vol-XXX-libvirt.xml for future detach
                if (create_vol_xml(instance->instanceId, volume->volumeId, libvirt_xml, &libvirt_xml_modified) != EUCA_OK) {
                    goto unroll;
                }

                continue;
unroll:
                ret = EUCA_ERROR;

                // @TODO: unroll all previous ones
                //  for (int uv = v - 1; uv >= 0; uv--) {
                //    disconnect_ebs(nc, instance->instanceId, volume->volumeId, )
                //  }

                goto failed_dest;
            }

                        // build any secondary network interface xml files
            for (int w=0; w < EUCA_MAX_NICS; w++) {
                if (strlen(instance->secNetCfgs[w].interfaceId) == 0)
                    continue;
                gen_libvirt_nic_xml(instance->instancePath, instance->secNetCfgs[w].interfaceId);
            }

            sem_p(inst_sem);
            instance->migration_state = MIGRATION_READY;
            instance->migrationTime = 0; //Reset the timer, to ensure monitoring thread handles this properly. This is required when setting BOOTING state
            instance->bootTime = time(NULL);    // otherwise nc_state.booting_cleanup_threshold will kick in
            change_state(instance, BOOTING);    // not STAGING, since in that mode we don't poll hypervisor for info
            LOGINFO("[%s] migration destination ready %s > %s\n", instance->instanceId, instance->migration_src, instance->migration_dst);
            save_instance_struct(instance);
            copy_instances();
            sem_v(inst_sem);
            continue;

failed_dest:
            sem_p(inst_sem);
            // Just making sure...
            if (instance != NULL) {
                LOGERROR("[%s] setting instance to Teardown(cleaning) after destination failure to prepare for migration\n", instance->instanceId);
                // Set state to Teardown(cleaning) so source won't wait until timeout to roll back.
                instance->migration_state = MIGRATION_CLEANING;
                instance->terminationTime = time(NULL);
                change_state(instance, TEARDOWN);
                save_instance_struct(instance);
                add_instance(&global_instances, instance);  // OK if this fails--that should mean it's already been added.
                copy_instances();
            }
            // If no remaining incoming or pending migrations, deauthorize all clients.
            // TO-DO: Consolidate with similar sequence in handlers.c into a utility function?
            if (!incoming_migrations_in_progress) {
                int incoming_migrations_pending = 0;
                LOGINFO("[%s] no remaining active incoming migrations -- checking to see if there are any pending migrations\n", instance->instanceId);
                bunchOfInstances *head = NULL;
                for (head = global_instances; head; head = head->next) {
                    if ((head->instance->migration_state == MIGRATION_PREPARING) || (head->instance->migration_state == MIGRATION_READY)) {
                        LOGINFO("[%s] is pending migration, state='%s', deferring deauthorization of migration keys\n", head->instance->instanceId,
                                migration_state_names[head->instance->migration_state]);
                        incoming_migrations_pending++;
                    }
                }
                // TO-DO: Add belt and suspenders?
                if (!incoming_migrations_pending) {
                    LOGINFO("[%s] no remaining incoming or pending migrations -- deauthorizing all migration client keys\n", instance->instanceId);
                    authorize_migration_keys("-D -r", NULL, NULL, NULL, FALSE);
                }
            }
            sem_v(inst_sem);
            // Set to generic EUCA_ERROR unless already set to a more-specific error.
            if (ret == EUCA_OK) {
                ret = EUCA_ERROR;
            }
        } else {
            LOGERROR("unexpected migration request (node %s is neither source nor destination)\n", pMeta->nodeName);
            ret = EUCA_ERROR;
        }
    }
    return ret;
}
Esempio n. 2
0
//!
//! Handles the instance migration request.
//!
//! @param[in]  nc a pointer to the node controller (NC) state
//! @param[in]  pMeta a pointer to the node controller (NC) metadata structure
//! @param[in]  instances metadata for the instance to migrate to destination
//! @param[in]  instancesLen number of instances in the instance list
//! @param[in]  action IP of the destination Node Controller
//! @param[in]  credentials credentials that enable the migration
//!
//! @return EUCA_OK on success or EUCA_*ERROR on failure
//!
//! @pre
//!
//! @post
static int doMigrateInstances(struct nc_state_t *nc, ncMetadata * pMeta, ncInstance ** instances, int instancesLen, char *action, char *credentials)
{
    int ret = EUCA_OK;
    int credentials_prepared = 0;

    if (instancesLen <= 0) {
        LOGERROR("called with invalid instancesLen (%d)\n", instancesLen);
        pMeta->replyString = strdup("internal error (invalid instancesLen)");
        return (EUCA_INVALID_ERROR);
    }

    LOGDEBUG("verifying %d instance[s] for migration...\n", instancesLen);
    for (int inst_idx = 0; inst_idx < instancesLen; inst_idx++) {
        LOGDEBUG("verifying instance # %d...\n", inst_idx);
        if (instances[inst_idx]) {
            ncInstance *instance_idx = instances[inst_idx];
            LOGDEBUG("[%s] proposed migration action '%s' (%s > %s) [creds=%s]\n", SP(instance_idx->instanceId), SP(action), SP(instance_idx->migration_src),
                     SP(instance_idx->migration_dst), (instance_idx->migration_credentials == NULL) ? "UNSET" : "present");
        } else {
            pMeta->replyString = strdup("internal error (instance count mismatch)");
            LOGERROR("Mismatch between migration instance count (%d) and length of instance list\n", instancesLen);
            return (EUCA_ERROR);
        }
    }

    // TO-DO: Optimize the location of this loop, placing it inside various conditionals below?
    for (int inst_idx = 0; inst_idx < instancesLen; inst_idx++) {
        ncInstance *instance_req = instances[inst_idx];
        char *sourceNodeName = instance_req->migration_src;
        char *destNodeName = instance_req->migration_dst;

        LOGDEBUG("[%s] processing instance # %d (%s > %s)\n", instance_req->instanceId, inst_idx, instance_req->migration_src, instance_req->migration_dst);

        // this is a call to the source of migration
        if (!strcmp(pMeta->nodeName, sourceNodeName)) {

            // locate the instance structure
            ncInstance *instance;
            sem_p(inst_sem);
            {
                instance = find_instance(&global_instances, instance_req->instanceId);
            }
            sem_v(inst_sem);
            if (instance == NULL) {
                LOGERROR("[%s] cannot find instance\n", instance_req->instanceId);
                pMeta->replyString = strdup("failed to locate instance to migrate");
                return (EUCA_NOT_FOUND_ERROR);
            }

            if (strcmp(action, "prepare") == 0) {
                sem_p(inst_sem);
                instance->migration_state = MIGRATION_PREPARING;
                euca_strncpy(instance->migration_src, sourceNodeName, HOSTNAME_SIZE);
                euca_strncpy(instance->migration_dst, destNodeName, HOSTNAME_SIZE);
                euca_strncpy(instance->migration_credentials, credentials, CREDENTIAL_SIZE);
                instance->migrationTime = time(NULL);
                save_instance_struct(instance);
                copy_instances();
                sem_v(inst_sem);

                // Establish migration-credential keys if this is the first instance preparation for this host.
                LOGINFO("[%s] migration source preparing %s > %s [creds=%s]\n", SP(instance->instanceId), SP(instance->migration_src), SP(instance->migration_dst),
                        (instance->migration_credentials == NULL) ? "UNSET" : "present");
                if (!credentials_prepared) {
                    if (generate_migration_keys(sourceNodeName, credentials, TRUE, instance) != EUCA_OK) {
                        pMeta->replyString = strdup("internal error (migration credentials generation failed)");
                        return (EUCA_SYSTEM_ERROR);
                    } else {
                        credentials_prepared++;
                    }
                }
                sem_p(inst_sem);
                instance->migration_state = MIGRATION_READY;
                save_instance_struct(instance);
                copy_instances();
                sem_v(inst_sem);

            } else if (strcmp(action, "commit") == 0) {

                sem_p(inst_sem);
                if (instance->migration_state == MIGRATION_IN_PROGRESS) {
                    LOGWARN("[%s] duplicate request to migration source to initiate %s > %s (already migrating)\n", instance->instanceId,
                            instance->migration_src, instance->migration_dst);
                    sem_v(inst_sem);
                    return (EUCA_DUPLICATE_ERROR);
                } else if (instance->migration_state != MIGRATION_READY) {
                    LOGERROR("[%s] request to commit migration %s > %s when source migration_state='%s' (not 'ready')\n", instance->instanceId,
                             SP(sourceNodeName), SP(destNodeName), migration_state_names[instance->migration_state]);
                    sem_v(inst_sem);
                    return (EUCA_UNSUPPORTED_ERROR);
                }
                instance->migration_state = MIGRATION_IN_PROGRESS;
                outgoing_migrations_in_progress++;
                LOGINFO("[%s] migration source initiating %s > %s [creds=%s] (1 of %d active outgoing migrations)\n", instance->instanceId, instance->migration_src,
                        instance->migration_dst, (instance->migration_credentials == NULL) ? "UNSET" : "present", outgoing_migrations_in_progress);
                save_instance_struct(instance);
                copy_instances();
                sem_v(inst_sem);

                // since migration may take a while, we do them in a thread
                pthread_t tcb = { 0 };
                if (pthread_create(&tcb, NULL, migrating_thread, (void *)instance)) {
                    LOGERROR("[%s] failed to spawn a migration thread\n", instance->instanceId);
                    return (EUCA_THREAD_ERROR);
                }
                set_corrid_pthread( get_corrid()!=NULL ? get_corrid()->correlation_id : NULL , tcb); 
                if (pthread_detach(tcb)) {
                    LOGERROR("[%s] failed to detach the migration thread\n", instance->instanceId);
                    return (EUCA_THREAD_ERROR);
                }
            } else if (strcmp(action, "rollback") == 0) {
                if ((instance->migration_state == MIGRATION_READY) || (instance->migration_state == MIGRATION_PREPARING)) {
                    LOGINFO("[%s] rolling back migration (%s > %s) on source\n", instance->instanceId, instance->migration_src, instance->migration_dst);
                    sem_p(inst_sem);
                    migration_rollback(instance);
                    sem_v(inst_sem);
                } else {
                    LOGINFO("[%s] ignoring request to roll back migration on source with instance in state %s(%s) -- duplicate rollback request?\n", instance->instanceId,
                            instance->stateName, migration_state_names[instance->migration_state]);
                }
            } else {
                LOGERROR("[%s] action '%s' is not valid\n", instance->instanceId, action);
                return (EUCA_INVALID_ERROR);
            }

        } else if (!strcmp(pMeta->nodeName, destNodeName)) {    // this is a migrate request to destination

            if (!strcmp(action, "commit")) {
                LOGERROR("[%s] action '%s' for migration (%s > %s) is not valid on destination node\n", instance_req->instanceId, action, SP(sourceNodeName), SP(destNodeName));
                return (EUCA_UNSUPPORTED_ERROR);
            } else if (!strcmp(action, "rollback")) {
                LOGINFO("[%s] rolling back migration (%s > %s) on destination\n", instance_req->instanceId, SP(sourceNodeName), SP(destNodeName));
                sem_p(inst_sem);
                {
                    ncInstance *instance = find_instance(&global_instances, instance_req->instanceId);
                    if (instance != NULL) {
                        LOGDEBUG("[%s] marked for cleanup\n", instance->instanceId);
                        change_state(instance, SHUTOFF);
                        instance->migration_state = MIGRATION_CLEANING;
                        save_instance_struct(instance);
                    }
                }
                sem_v(inst_sem);
                return EUCA_OK;
            } else if (strcmp(action, "prepare") != 0) {
                LOGERROR("[%s] action '%s' is not valid or not implemented\n", instance_req->instanceId, action);
                return (EUCA_INVALID_ERROR);
            }
            // Everything from here on is specific to "prepare" on a destination.

            // allocate a new instance struct
            ncInstance *instance = clone_instance(instance_req);
            if (instance == NULL) {
                LOGERROR("[%s] could not allocate instance struct\n", instance_req->instanceId);
                goto failed_dest;
            }

            sem_p(inst_sem);
            instance->migration_state = MIGRATION_PREPARING;
            euca_strncpy(instance->migration_src, sourceNodeName, HOSTNAME_SIZE);
            euca_strncpy(instance->migration_dst, destNodeName, HOSTNAME_SIZE);
            euca_strncpy(instance->migration_credentials, credentials, CREDENTIAL_SIZE);
            save_instance_struct(instance);
            sem_v(inst_sem);

            // Establish migration-credential keys.
            LOGINFO("[%s] migration destination preparing %s > %s [creds=%s]\n", instance->instanceId, SP(instance->migration_src), SP(instance->migration_dst),
                    (instance->migration_credentials == NULL) ? "UNSET" : "present");
            // First, call config-file modification script to authorize source node.
            LOGDEBUG("[%s] authorizing migration source node %s\n", instance->instanceId, instance->migration_src);
            if (authorize_migration_keys("-a", instance->migration_src, instance->migration_credentials, instance, TRUE) != EUCA_OK) {
                goto failed_dest;
            }
            // Then, generate keys and restart libvirtd.
            if (generate_migration_keys(instance->migration_dst, instance->migration_credentials, TRUE, instance) != EUCA_OK) {
                goto failed_dest;
            }
            int error;

            if (vbr_parse(&(instance->params), pMeta) != EUCA_OK) {
                goto failed_dest;
            }
            // set up networking
            char *brname = NULL;
            if ((error = vnetStartNetwork(nc->vnetconfig, instance->ncnet.vlan, NULL, NULL, NULL, &brname)) != EUCA_OK) {
                LOGERROR("[%s] start network failed for instance, terminating it\n", instance->instanceId);
                EUCA_FREE(brname);
                goto failed_dest;
            }
            euca_strncpy(instance->params.guestNicDeviceName, brname, sizeof(instance->params.guestNicDeviceName));
            EUCA_FREE(brname);

            // TODO: move stuff in startup_thread() into a function?

            set_instance_params(instance);

            if ((error = create_instance_backing(instance, TRUE))   // create files that back the disks
                || (error = gen_instance_xml(instance)) // create euca-specific instance XML file
                || (error = gen_libvirt_instance_xml(instance))) {  // transform euca-specific XML into libvirt XML
                LOGERROR("[%s] failed to prepare images for migrating instance (error=%d)\n", instance->instanceId, error);
                goto failed_dest;
            }
            // attach any volumes
            for (int v = 0; v < EUCA_MAX_VOLUMES; v++) {
                ncVolume *volume = &instance->volumes[v];
                if (strcmp(volume->stateName, VOL_STATE_ATTACHED) && strcmp(volume->stateName, VOL_STATE_ATTACHING))
                    continue;          // skip the entry unless attached or attaching
                LOGDEBUG("[%s] volumes [%d] = '%s'\n", instance->instanceId, v, volume->stateName);

                // TODO: factor what the following out of here and doAttachVolume() in handlers_default.c

                int have_remote_device = 0;
                char *xml = NULL;
                char *remoteDevStr = NULL;
                char scUrl[512];
                char localDevReal[32], localDevTag[256], remoteDevReal[132];
                char *tagBuf = localDevTag;
                ebs_volume_data *vol_data = NULL;
                ret = convert_dev_names(volume->localDev, localDevReal, tagBuf);
                if (ret)
                    goto unroll;

                //Do the ebs connect.
                LOGTRACE("[%s][%s] Connecting EBS volume to local host\n", instance->instanceId, volume->volumeId);
                get_service_url("storage", nc, scUrl);

                if (strlen(scUrl) == 0) {
                    LOGERROR("[%s][%s] Failed to lookup enabled Storage Controller. Cannot attach volume %s\n", instance->instanceId, volume->volumeId, scUrl);
                    have_remote_device = 0;
                    goto unroll;
                } else {
                    LOGTRACE("[%s][%s] Using SC URL: %s\n", instance->instanceId, volume->volumeId, scUrl);
                }

                //Do the ebs connect.
                LOGTRACE("[%s][%s] Connecting EBS volume to local host\n", instance->instanceId, volume->volumeId);
                int rc = connect_ebs_volume(scUrl, volume->attachmentToken, nc->config_use_ws_sec, nc->config_sc_policy_file, nc->ip, nc->iqn, &remoteDevStr, &vol_data);
                if (rc) {
                    LOGERROR("Error connecting ebs volume %s\n", volume->attachmentToken);
                    have_remote_device = 0;
                    ret = EUCA_ERROR;
                    goto unroll;
                }
                // update the volume struct with connection string obtained from SC
                euca_strncpy(volume->connectionString, vol_data->connect_string, sizeof(volume->connectionString));

                if (!remoteDevStr || !strstr(remoteDevStr, "/dev")) {
                    LOGERROR("[%s][%s] failed to connect to iscsi target\n", instance->instanceId, volume->volumeId);
                    remoteDevReal[0] = '\0';
                } else {
                    LOGDEBUG("[%s][%s] attached iSCSI target of host device '%s'\n", instance->instanceId, volume->volumeId, remoteDevStr);
                    snprintf(remoteDevReal, sizeof(remoteDevReal), "%s", remoteDevStr);
                    have_remote_device = 1;
                }
                EUCA_FREE(remoteDevStr);

                // something went wrong above, abort
                if (!have_remote_device) {
                    goto unroll;
                }
                // make sure there is a block device
                if (check_block(remoteDevReal)) {
                    LOGERROR("[%s][%s] cannot verify that host device '%s' is available for hypervisor attach\n", instance->instanceId, volume->volumeId, remoteDevReal);
                    goto unroll;
                }
                // generate XML for libvirt attachment request
                if (gen_volume_xml(volume->volumeId, instance, localDevReal, remoteDevReal) // creates vol-XXX.xml
                    || gen_libvirt_volume_xml(volume->volumeId, instance)) {    // creates vol-XXX-libvirt.xml via XSLT transform
                    LOGERROR("[%s][%s] could not produce attach device xml\n", instance->instanceId, volume->volumeId);
                    goto unroll;
                }
                // invoke hooks
                char path[EUCA_MAX_PATH];
                char lpath[EUCA_MAX_PATH];
                snprintf(path, sizeof(path), EUCALYPTUS_VOLUME_XML_PATH_FORMAT, instance->instancePath, volume->volumeId);  // vol-XXX.xml
                snprintf(lpath, sizeof(lpath), EUCALYPTUS_VOLUME_LIBVIRT_XML_PATH_FORMAT, instance->instancePath, volume->volumeId);    // vol-XXX-libvirt.xml
                if (call_hooks(NC_EVENT_PRE_ATTACH, lpath)) {
                    LOGERROR("[%s][%s] cancelled volume attachment via hooks\n", instance->instanceId, volume->volumeId);
                    goto unroll;
                }
                // read in libvirt XML, which may have been modified by the hook above
                if ((xml = file2str(lpath)) == NULL) {
                    LOGERROR("[%s][%s] failed to read volume XML from %s\n", instance->instanceId, volume->volumeId, lpath);
                    goto unroll;
                }

                continue;
unroll:
                ret = EUCA_ERROR;

                // TODO: unroll all volume attachments

                goto failed_dest;
            }

            sem_p(inst_sem);
            instance->migration_state = MIGRATION_READY;
            instance->bootTime = time(NULL);    // otherwise nc_state.booting_cleanup_threshold will kick in
            change_state(instance, BOOTING);    // not STAGING, since in that mode we don't poll hypervisor for info
            LOGINFO("[%s] migration destination ready %s > %s\n", instance->instanceId, instance->migration_src, instance->migration_dst);
            save_instance_struct(instance);

            error = add_instance(&global_instances, instance);
            copy_instances();
            sem_v(inst_sem);
            if (error) {
                if (error == EUCA_DUPLICATE_ERROR) {
                    LOGINFO("[%s] instance struct already exists (from previous migration?), deleting and re-adding...\n", instance->instanceId);
                    error = remove_instance(&global_instances, instance);
                    if (error) {
                        LOGERROR("[%s] could not replace (remove) instance struct, failing...\n", instance->instanceId);
                        goto failed_dest;
                    }
                    error = add_instance(&global_instances, instance);
                    if (error) {
                        LOGERROR("[%s] could not replace (add) instance struct, failing...\n", instance->instanceId);
                        goto failed_dest;
                    }
                } else {
                    LOGERROR("[%s] could not add instance struct, failing...\n", instance->instanceId);
                    goto failed_dest;
                }
            }

            continue;

failed_dest:
            sem_p(inst_sem);
            // Just making sure...
            if (instance != NULL) {
                LOGERROR("[%s] setting instance to Teardown(cleaning) after destination failure to prepare for migration\n", instance->instanceId);
                // Set state to Teardown(cleaning) so source won't wait until timeout to roll back.
                instance->migration_state = MIGRATION_CLEANING;
                instance->terminationTime = time(NULL);
                change_state(instance, TEARDOWN);
                save_instance_struct(instance);
                add_instance(&global_instances, instance);  // OK if this fails--that should mean it's already been added.
                copy_instances();
            }
            // If no remaining incoming or pending migrations, deauthorize all clients.
            // TO-DO: Consolidate with similar sequence in handlers.c into a utility function?
            if (!incoming_migrations_in_progress) {
                int incoming_migrations_pending = 0;
                LOGINFO("[%s] no remaining active incoming migrations -- checking to see if there are any pending migrations\n", instance->instanceId);
                bunchOfInstances *head = NULL;
                for (head = global_instances; head; head = head->next) {
                    if ((head->instance->migration_state == MIGRATION_PREPARING) || (head->instance->migration_state == MIGRATION_READY)) {
                        LOGINFO("[%s] is pending migration, state='%s', deferring deauthorization of migration keys\n", head->instance->instanceId,
                                migration_state_names[head->instance->migration_state]);
                        incoming_migrations_pending++;
                    }
                }
                // TO-DO: Add belt and suspenders?
                if (!incoming_migrations_pending) {
                    LOGINFO("[%s] no remaining incoming or pending migrations -- deauthorizing all migration client keys\n", instance->instanceId);
                    authorize_migration_keys("-D -r", NULL, NULL, NULL, FALSE);
                }
            }
            sem_v(inst_sem);
            // Set to generic EUCA_ERROR unless already set to a more-specific error.
            if (ret == EUCA_OK) {
                ret = EUCA_ERROR;
            }
        } else {
            LOGERROR("unexpected migration request (node %s is neither source nor destination)\n", pMeta->nodeName);
            ret = EUCA_ERROR;
        }
    }
    return ret;
}