//============================================================================= // METHOD: SPELLvariableMonitor::registerVariable() //============================================================================= bool SPELLvariableMonitor::registerVariable( SPELLvarInfo& var ) { SPELLsafePythonOperations ops("SPELLvariableMonitor::registerVariable()"); if ((m_frame->f_globals)&&(var.isGlobal)) { var.varValue = PYSSTR( PyDict_GetItemString( m_frame->f_globals, var.varName.c_str() )); if (var.varValue == "") var.varValue = EMPTY_STRING; var.varType = PYSSTR( PyObject_Type( PyDict_GetItemString( m_frame->f_globals, var.varName.c_str() )) ); var.isGlobal = true; var.isRegistered = true; m_variables.insert( std::make_pair( var.varName, var ) ); DEBUG("[VM] Registered global variable: " + var.varName + ", current value: " + var.varValue ); return true; } else if ((m_frame->f_locals)&&(!var.isGlobal)) { // We need to retrieve the function arguments and other locals, which are only stored in // fast locals by default PyFrame_FastToLocals(m_frame); var.varValue = PYSSTR( PyDict_GetItemString( m_frame->f_locals, var.varName.c_str() )); if (var.varValue == "") var.varValue = EMPTY_STRING; var.varType = PYSSTR( PyObject_Type( PyDict_GetItemString( m_frame->f_locals, var.varName.c_str() )) ); var.isGlobal = true; var.isRegistered = true; m_variables.insert( std::make_pair( var.varName, var ) ); DEBUG("[VM] Registered local variable: " + var.varName + ", current value: " + var.varValue ); return true; } var.varValue = "???"; var.varType = "???"; var.isRegistered = false; DEBUG("[VM] No such variable: " + var.varName); return false; }
//============================================================================= // METHOD: SPELLvariableMonitor::retrieveNames() //============================================================================= std::vector<std::string> SPELLvariableMonitor::retrieveNames() { std::vector<std::string> varNames; PyObject* varList = m_frame->f_code->co_names; DEBUG("[VM] CO_NAMES : " + PYREPR(varList)); unsigned int numVars = PyTuple_Size(varList); /* * co_varnames contains the names of the local variables * (starting with the argument names) */ varList = m_frame->f_code->co_varnames; DEBUG("[VM] CO_VARNAMES: " + PYREPR(varList)); numVars = PyTuple_Size(varList); for( unsigned int index = 0; index<numVars; index++) { PyObject* varName = PyTuple_GetItem( varList, index ); varNames.push_back( PYSSTR( varName ) ); } varList = m_frame->f_code->co_freevars; DEBUG("[VM] CO_FREEVARS : " + PYREPR(varList)); varList = m_frame->f_code->co_cellvars; DEBUG("[VM] CO_CELLVARS : " + PYREPR(varList)); return varNames; }
//============================================================================= // METHOD: SPELLvariableMonitor::analyze() //============================================================================= void SPELLvariableMonitor::analyze() { std::vector<SPELLvarInfo> changed; bool copyFast = true; VarMap::iterator it; for( it = m_variables.begin(); it != m_variables.end(); it++ ) { std::string varName = it->first; std::string currentValue = ""; std::string lastValue = ""; if ( (it->second.isGlobal) && (m_frame->f_globals) ) { PyObject* pyCurrentValue = PyDict_GetItemString( m_frame->f_globals, varName.c_str() ); currentValue = PYSSTR(pyCurrentValue); lastValue = it->second.varValue; } else if ( (!it->second.isGlobal) && (m_frame->f_locals) ) { if (copyFast) { copyFast = false; // We need to retrieve the function arguments and other locals, which are only stored in // fast locals by default PyFrame_FastToLocals(m_frame); } PyObject* pyCurrentValue = PyDict_GetItemString( m_frame->f_locals, varName.c_str() ); currentValue = PYSSTR(pyCurrentValue); lastValue = it->second.varValue; } if (lastValue != currentValue) { if (currentValue == "") currentValue == EMPTY_STRING; DEBUG("[VM] Variable change: " + varName + ", current value: " + currentValue + ", previous: " + it->second.varValue ); it->second.varValue = currentValue; changed.push_back(it->second); } } if (changed.size()>0) { m_listener->variableChanged( changed ); } }
//============================================================================= // METHOD: SPELLvariableMonitor::retrieveLocalVariables() //============================================================================= void SPELLvariableMonitor::retrieveLocalVariables(std::vector<SPELLvarInfo>& vars) { DEBUG("[VM] Retrieve Locals"); /* * Bottom stack frame is discarded, * as globals and locals are the same dictionary */ if (m_frame->f_back == NULL) return; /* * Get the names defined in the current code, including arguments */ std::vector<std::string> varNames = retrieveNames(); /* * Iterate over the locals dictionary, retrieving the names contained in * varNames */ PyFrame_FastToLocals(m_frame); PyObject* dict = m_frame->f_locals; DEBUG("[VM] Frame: " + PYCREPR(m_frame)); for( unsigned int index = 0; index< varNames.size(); index++) { std::string varName = varNames[index]; PyObject* pyVarName = SSTRPY(varName); if (PyDict_Contains( dict, pyVarName )) { PyObject* object = PyDict_GetItem( dict, pyVarName ); if (!SPELLpythonHelper::instance().isInstance(object, "Database", "spell.lib.adapter.databases.database")) { if (PyCallable_Check(object)) continue; if (PyClass_Check(object)) continue; if (PyModule_Check(object)) continue; if (PyInstance_Check(object)) continue; } DEBUG("[VM] Processing " + varName); std::string type = PYSSTR( PyObject_Type(object) ); DEBUG("[VM] Type : " + type); std::string value = PYREPR( object ); DEBUG("[VM] Value : " + value); DEBUG("[VM] Global : " + BSTR(false)); DEBUG("[VM] Registered: " + BSTR(isRegistered(varName))); // Mark empty values (empty strings) as "<empty>" if (value == "") value = EMPTY_STRING; vars.push_back( SPELLvarInfo(varName, type, value, false, isRegistered(varName)) ); } } PyFrame_LocalsToFast(m_frame,0); }
//============================================================================= // METHOD: SPELLvariableMonitor::changeVariable() //============================================================================= void SPELLvariableMonitor::changeVariable( SPELLvarInfo& var ) { SPELLsafePythonOperations ops("SPELLvariableMonitor::changeVariable()"); DEBUG("[VM] Request changing variable " + var.varName); // Evaluate the value for the variable PyObject* value = NULL; // If varValue is '<empty>' or empty string, do not try to evaluate it, // directly assign Python empty string if ((var.varValue == EMPTY_STRING)||(var.varValue == "")) { value = STRPY(""); } else { // Build assignment expression. We need to check first, if there // are double quotes, convert them to single quotes SPELLutils::replace(var.varValue, "\"", "'"); DEBUG("[VM] Evaluating value expression: " + var.varValue ); // Check value correctness and evaluate it value = SPELLpythonHelper::instance().eval(var.varValue, false); } if ((m_frame->f_globals)&&(var.isGlobal)) { DEBUG("[VM] Setting " + var.varName + " to " + PYREPR(value) + " in globals"); PyDict_SetItemString( m_frame->f_globals, var.varName.c_str(), value ); } else if ((m_frame->f_locals)&&(!var.isGlobal)) { DEBUG("[VM] Setting " + var.varName + " to " + PYREPR(value) + " in locals"); // Update locals from fast locals first PyFrame_FastToLocals(m_frame); PyDict_SetItemString( m_frame->f_locals, var.varName.c_str(), value ); PyFrame_LocalsToFast(m_frame,0); } var.varValue = PYSSTR(value); if (var.varValue == "") var.varValue = EMPTY_STRING; // Update the variable if it is registered, and notify to listeners VarMap::iterator it = m_variables.find(var.varName); if (it != m_variables.end()) { std::vector<SPELLvarInfo> changed; it->second.varValue = var.varValue; DEBUG("[VM] Variable changed by user: "******", current value: " + var.varValue ); changed.push_back(it->second); m_listener->variableChanged( changed ); } }
//============================================================================= // METHOD: SPELLconfigDict::reset() //============================================================================= void SPELLconfigDict::reset( PyObject* dict ) { m_values.clear(); if (dict) { SPELLsafePythonOperations ops ("SPELLconfigDict::reset()"); SPELLpyHandle keys = PyDict_Keys(dict); unsigned int numKeys = PyList_Size(keys.get()); for( unsigned int idx = 0; idx < numKeys; idx++ ) { PyObject* key = PyList_GetItem( keys.get(), idx ); PyObject* value = PyDict_GetItem( dict, key ); m_values.insert( std::make_pair( PYSSTR(key), SPELLpyValue(value) )); } } }
//============================================================================= // METHOD : SPELLwsWarmStartImpl::restoreState() //============================================================================= PyFrameObject* SPELLwsWarmStartImpl::restoreState() { DEBUG("[WS] Restoring state ========================================"); // We need a separate scope so that we can invoke saveState at the end without deadlock { std::cerr << "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@" << std::endl; std::cerr << "RESTORE STATE START" << std::endl; std::cerr << "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@" << std::endl; // Synchronize so that nothing can be done while saving SPELLmonitor m(m_lock); SPELLsafePythonOperations ops("SPELLwsWarmStartImpl::restoreState()"); // Create a fresh thread state PyThreadState* tstate = PyThreadState_Get(); DEBUG("[WS] Restoring interpreter parameters"); tstate->recursion_depth = m_storage->loadLong(); DEBUG(" - Recursion depth : " + ISTR(tstate->recursion_depth)); tstate->tick_counter = m_storage->loadLong(); DEBUG(" - Tcik counter : " + ISTR(tstate->tick_counter)); tstate->gilstate_counter = m_storage->loadLong(); DEBUG(" - GIL counter : " + ISTR(tstate->gilstate_counter)); int numFrames = m_storage->loadLong(); DEBUG(" - Number of frames: " + ISTR(numFrames)); if (numFrames < 0) { THROW_EXCEPTION("Unable to restore state", "Failed to restore interpreter parameters", SPELL_ERROR_WSTART); } DEBUG("[WS] Restoring frames"); m_frames.clear(); m_recursionDepth = 0; // Restore the frames now. Consider that the top frame is already there // Use the originally created frame to copy globals PyFrameObject* prevFrame = NULL; for( int count = 0; count < numFrames; count++ ) { // In recursion depth zero, use the original frame // to copy the globals from std::string frameId = PYSSTR(m_storage->loadObject()); m_topFrame = new SPELLwsFrame( frameId, m_startup, m_frames.size(), prevFrame ); m_frames.push_back(m_topFrame); // For the head frame, store its address in the interpreter thread state if (m_recursionDepth == 0) { if (tstate->frame) delete tstate->frame; tstate->frame = m_topFrame->getFrameObject(); } m_recursionDepth++; prevFrame = m_topFrame->getFrameObject(); } // Update the recursion depth tstate->recursion_depth = m_recursionDepth; DEBUG("[WS] Checking errors"); SPELLpythonHelper::instance().checkError(); DEBUG("[WS] Switching to save state mode"); // Reset the storage now, to mode write delete m_storage; m_storage = new SPELLwsStorage( m_persistentFile, SPELLwsStorage::MODE_WRITE ); std::cerr << "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@" << std::endl; std::cerr << "RESTORE STATE END" << std::endl; std::cerr << "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@" << std::endl; } // Re-save the current state, so that the new persistent files are ok saveState(); DEBUG("[WS] Checking errors"); SPELLpythonHelper::instance().checkError(); // Reset values m_startup.recoveryFile = ""; m_startup.performRecovery = false; DEBUG("[WS] State recovered ==========================================="); return m_topFrame->getFrameObject(); }
//============================================================================ // METHOD: SPELLpyValue::set //============================================================================ void SPELLpyValue::set( PyObject* pyValue ) { m_type = NONE; m_intValue = 0; m_boolValue = false; m_floatValue = 0.0; m_timeValue.set(0,0); m_stringValue = ""; if (pyValue == NULL) return; DEBUG("[PYVAL] Set value from " + PYREPR(pyValue)); if (pyValue != Py_None) { if (PyBool_Check(pyValue)) { m_type = BOOLEAN; m_boolValue = pyValue == Py_True; } else if (PyLong_Check(pyValue)) { DEBUG("[PYVAL] Long check"); m_type = LONG; m_intValue = PyLong_AsLongLong(pyValue); } else if (PyInt_Check(pyValue)) { DEBUG("[PYVAL] Int check"); m_type = LONG; m_intValue = PyInt_AsLong(pyValue); } else if (PyFloat_Check(pyValue)) { m_type = DOUBLE; m_floatValue = PyFloat_AsDouble(pyValue); } else if (SPELLpythonHelper::instance().isTime(pyValue)) { m_timeValue = SPELLpythonHelper::instance().evalTime(PYSSTR(pyValue)); if (m_timeValue.isDelta()) { m_type = RELTIME; } else { m_type = ABSTIME; } } else if (PyString_Check(pyValue)) { m_type = STRING; m_stringValue = PYSTR(pyValue); } else if (PyList_Check(pyValue)) { m_type = LIST; m_listValue.clear(); unsigned int numItems = PyList_Size(pyValue); for(unsigned int idx = 0; idx < numItems; idx++) { m_listValue.push_back( SPELLpyValue( PyList_GetItem( pyValue, idx) )); } } else if (PyDict_Check(pyValue)) { m_type = DICT; m_dictValue.clear(); PyObject* keys = PyDict_Keys(pyValue); unsigned int numItems = PyList_Size(keys); for(unsigned int idx = 0; idx < numItems; idx++) { PyObject* key = PyList_GetItem(keys,idx); PyObject* value = PyDict_GetItem( pyValue, key ); m_dictValue.insert( std::make_pair( PYSSTR(key), SPELLpyValue(value) ) ); } } else { THROW_EXCEPTION("Cannot create variable value", "Cannot infer type from value (" + PYREPR(pyValue) + ")", SPELL_ERROR_LANGUAGE); } SPELLpythonHelper::instance().checkError(); } }
//============================================================================= // METHOD: SPELLvariableMonitor::retrieveGlobalVariables() //============================================================================= void SPELLvariableMonitor::retrieveGlobalVariables(std::vector<SPELLvarInfo>& vars, std::set<std::string> locals) { DEBUG("[VM] Retrieve Globals"); /* * Once we get the bottom stack frame, we have to iterate over all the keys * in the globals dictionary, and filter them agains the m_initialVariables */ PyObject* dict = m_frame->f_globals; PyObject* itemList = PyDict_Keys(dict); unsigned int numItems = PyList_Size(itemList); for( unsigned int index = 0; index<numItems; index++) { PyObject* key = PyList_GetItem( itemList, index ); std::string varName = PYSSTR(key); // Do the following check just when the considered variables are not internal databases if ( (varName != DatabaseConstants::SCDB) && (varName != DatabaseConstants::GDB) && (varName != DatabaseConstants::PROC) && (varName != DatabaseConstants::ARGS) && (varName != DatabaseConstants::IVARS)) { /* If they key is contained in the initial variables set, then ignore it */ if (m_initialVariables.find(varName) != m_initialVariables.end()) { continue; } } /* If a variable with the same name has been retrieved in the local scope * then it must be ignored */ if (locals.find(varName) != locals.end()) { continue; } // Ignore internal flags if (varName == "__USERLIB__") continue; PyObject* object = PyDict_GetItem( dict, key ); if (!SPELLpythonHelper::instance().isInstance(object, "Database", "spell.lib.adapter.databases.database")) { if (PyCallable_Check(object)) continue; if (PyClass_Check(object)) continue; if (PyModule_Check(object)) continue; if (PyInstance_Check(object)) continue; } DEBUG("[VM] Processing " + varName); std::string type = PYSSTR( PyObject_Type(object) ); std::string value = PYREPR( object ); DEBUG("[VM] Type : " + type); DEBUG("[VM] Value : " + value); DEBUG("[VM] Global : " + BSTR(true)); DEBUG("[VM] Registered: " + BSTR(isRegistered(PYSSTR(key)))); // Mark empty values (empty strings) as "<empty>" if (value == "") value = EMPTY_STRING; vars.push_back( SPELLvarInfo(varName, type, value, true, isRegistered(PYSSTR(key))) ); } }
//============================================================================= // METHOD : SPELLwsDictDataHandler::write() //============================================================================= void SPELLwsDictDataHandler::write() { if (getObject() == NULL) { getStorage()->storeLong( -1 ); return; } assert( PyDict_Check(getObject())); SPELLpyHandle keys = PyDict_Keys( getObject() ); unsigned int numItems = PyList_Size( keys.get() ); DEBUG("[DDH] Storing dictionary items (total " + ISTR(numItems) + ") address " + PSTR(getObject())); PyObject* key = NULL; PyObject* item = NULL; long toStore = 0; // Calculate the number of items to store // Store each list item DEBUG("[DDH] Keys of the dictionary:"); for( unsigned int index = 0; index < numItems; index++) { key = PyList_GetItem( keys.get(), index ); item = PyDict_GetItem( getObject(), key ); if (SPELLwsWarmStartImpl::shouldFilter(key,item)) { continue; } else { DEBUG(" - " + PYSSTR(key)); } toStore++; } DEBUG("[DDH] Will store " + ISTR(toStore) + " keys"); // Store the number of items getStorage()->storeLong( (long) toStore ); DEBUG("[DDH] Start checking items"); if (toStore>0) { // Store each list item for( unsigned int index = 0; index < numItems; index++) { key = PyList_GetItem( keys.get(), index ); item = PyDict_GetItem( getObject(), key ); DEBUG("[DDH] Checking item " + PYCREPR(key)); // Do not consider filtered values if (SPELLwsWarmStartImpl::shouldFilter(key,item)) continue; SPELLpythonHelper::instance().checkError(); DEBUG(" [DDH] Key index " + ISTR(index)); DEBUG(" [DDH] Key to use" + PYREPR(key)); DEBUG(" [DDH] Item type:" + PYREPR(PyObject_Type(item))); // Handler for the key SPELLwsObjectDataHandler keyHandler(key); keyHandler.setStorage(getStorage()); DEBUG(" [DDH] Creating handler"); // Create a handler for the item SPELLwsDataHandler* handler = SPELLwsDataHandlerFactory::createDataHandler(item); handler->setStorage(getStorage()); // Store the key DEBUG(" [DDH] Storing key: " + PYREPR(key)); keyHandler.write(); // Store the item data code in order to recognise it later DEBUG(" [DDH] Storing data code: " + SPELLwsData::codeStr(handler->getCode())); handler->storeDataCode(); // IMPORTANT in the case of lists and dictionaries, we want to be able to continue // the storage even if there is a problem in the handler processing at this point. // If that is the case, a fake empty object will be replaced by the object being // processed by the handler, and the dumping of this collection will continue. try { if (handler->getCode() == SPELLwsData::DATA_CUSTOM_TYPE ) { std::string msg = "WARNING! warm start not supported for custom Python types (" + PYREPR(key) + "=" + PYREPR(item) + ")"; LOG_WARN(msg); SPELLexecutor::instance().getCIF().warning(msg); storeFakeObject( SPELLwsData::DATA_NONE ); } else { // Store the value DEBUG(" [DDH] Storing value: " + PYREPR(item)); handler->write(); DEBUG(" [DDH] Storing value done"); } } catch(SPELLcoreException& ex) { std::string msg = "WARNING! WS storage of element " + ISTR(index) + " failed: " + ex.what(); LOG_WARN(msg); SPELLexecutor::instance().getCIF().warning(msg); storeFakeObject( handler->getCode() ); } delete handler; } } DEBUG("[DDH] Storing dictionary done"); }