//=============================================================================
// CONSTRUCTOR: SPELLwsFrame::SPELLwsFrame
//=============================================================================
SPELLwsFrame::SPELLwsFrame( const SPELLwsStartupInfo& info, unsigned int depth, PyFrameObject* frame )
{
// If mode is recover and depth > 0, 'frame' contains the f_back frame.
// If depth is zero and mode is recover, we just need to fix the dynamics.
// If mode is other, 'frame' contains the frame associated to this structure.
if (info.performRecovery)
{
DEBUG("[FRM] Recovering " + ISTR(depth) + " level frame");
m_static = new SPELLwsStaticData(info,depth,frame);
// Recover this frame
m_frame = m_static->restore();
// In recovery mode, the given frame is the previous one in the stack
m_frame->f_back = frame;
// Use the recovered frame to restore dynamic data
m_dynamic = new SPELLwsDynamicData(info,depth,m_frame);
m_dynamic->restore();
DEBUG("[FRM] Recovering " + ISTR(depth) + " level frame done: " + PYCREPR(m_frame));
}
else
{
m_frame = frame;
m_static = new SPELLwsStaticData(info,depth,m_frame);
m_dynamic = new SPELLwsDynamicData(info,depth,m_frame);
}
DEBUG("[FRM] Created manager for frame " + PYCREPR(m_frame));
m_lastInstruction = m_frame->f_lasti;
m_lastLine = m_frame->f_lineno;
}
//=============================================================================
// METHOD : SPELLwsFrame::fixState()
//=============================================================================
void SPELLwsFrame::fixState( PyThreadState* newState, bool isHead )
{
DEBUG("[FRM] Fix state on frame " + PYCREPR(m_frame) + ", head=" + BSTR(isHead));
// This is required due to how the Python evaluation loop works. The
// instruction interesting for us is the one after the function call, if
// the frame is no the head of the tree.
if (isHead)
{
DEBUG("[FRM] Set instruction as head");
m_lastInstruction--;
}
else
{
DEBUG("[FRM] Set instruction as intermediate");
DEBUG("[FRM] Original instruction was " + ISTR(m_lastInstruction));
DEBUG("[FRM] Last line was " + ISTR(m_lastLine));
std::string filename = PYSTR(m_frame->f_code->co_filename);
std::string codename = PYSTR(m_frame->f_code->co_name);
std::string code_id = filename + "-" + codename;
int nextLine = SPELLexecutor::instance().getFrame().getModel(code_id).lineAfter(m_lastLine);
DEBUG("[FRM] Next line is " + ISTR(nextLine));
int nextInstruction = SPELLexecutor::instance().getFrame().getModel(code_id).offset(nextLine);
m_lastInstruction = nextInstruction -1; // Will position it in the lastLine but POP_TOP instr.
DEBUG("[FRM] Set instruction to: " + ISTR(m_lastInstruction));
}
DEBUG("[FRM] Applying: INS(" + ISTR(m_lastInstruction) + "), LIN(" + ISTR(m_lastLine) + ") on frame " + PYCREPR(m_frame));
// Reset the frame values
m_frame->f_lasti = m_lastInstruction;
m_frame->f_lineno = m_lastLine;
m_frame->f_tstate = newState;
m_frame->f_stacktop = m_frame->f_valuestack;
// Recover the dynamic data and update the frame
DEBUG("[FRM] Recovering dynamic data");
m_dynamic.recover();
// Connect the head with the thread state (the head is the frame going to
// be executed after recovery)
if (isHead)
{
newState->frame = m_frame;
}
//DumpFrameInfo( m_frame );
DEBUG("[FRM] State fixed on frame " + PYCREPR(m_frame));
}
//============================================================================
// FUNCTION: SPELLutils::dumpFrameInfo()
//============================================================================
void SPELLutils::dumpFrameInfo( const std::string& id, PyFrameObject* frame, int iStateId, int TStateId, int FrameId )
{
LOG_INFO("Dump information for frame " + ISTR(iStateId) + "." + ISTR(TStateId) + "." + ISTR(FrameId));
std::string dataDir = getSPELL_DATA() + PATH_SEPARATOR + "Runtime" + PATH_SEPARATOR;
std::string filename = dataDir + id + "_frame_state_" + ISTR(iStateId) + "." + ISTR(TStateId) + "." + ISTR(FrameId) + ".dump";
std::ofstream dumpFile;
dumpFile.open( filename.c_str() , std::ios::out );
dumpFile << "FRAME STATE " << TStateId << " DATA" << std::endl;
dumpFile << "--------------------------------------" << std::endl;
dumpFile << "Address : " << PSTR(frame) << std::endl;
dumpFile << "Next address : " << PSTR(frame->f_back) << std::endl;
dumpFile << "Thread state address : " << PSTR(frame->f_tstate) << std::endl;
dumpFile << "Last instruction : " << frame->f_lasti << std::endl;
dumpFile << "Last line : " << frame->f_lineno << std::endl;
dumpFile << "Try blocks count : " << frame->f_iblock << std::endl;
dumpFile << "Try blocks : " << PSTR(frame->f_blockstack) << std::endl;
dumpFile << "Value stack : " << PSTR(frame->f_valuestack) << std::endl;
dumpFile << "Stack top : " << PSTR(frame->f_stacktop) << std::endl;
dumpFile << "Stack count : " << (frame->f_stacktop - frame->f_valuestack) << std::endl;
dumpFile << "Fast locals : " << (frame->f_code->co_nlocals-1) << std::endl;
dumpFile << "Exception type : " << PYREPR(frame->f_exc_type) << std::endl;
dumpFile << "Exception value : " << PYREPR(frame->f_exc_value) << std::endl;
dumpFile << "Exception traceback : " << PYREPR(frame->f_exc_traceback) << std::endl;
dumpFile << "Trace function : " << PYREPR(frame->f_trace) << std::endl;
dumpFile << "Builtins : " << PYSIZE(frame->f_builtins) << std::endl;
dumpFile << "Globals : " << PYSIZE(frame->f_globals) << std::endl;
dumpFile << "Locals : " << PYSIZE(frame->f_locals) << std::endl;
dumpFile << "Code : " << PYCREPR(frame->f_code) << std::endl;
// Close the frame state dump, no more to add
dumpFile.flush();
dumpFile.close();
}
//=============================================================================
// CONSTRUCTOR: SPELLwsFrame::SPELLwsFrame
//=============================================================================
SPELLwsFrame::SPELLwsFrame( const std::string& persisFile, unsigned int depth, PyFrameObject* frame, const SPELLwsWorkingMode& mode )
: m_frame(frame),
m_static(persisFile,depth,frame,mode),
m_dynamic(persisFile,depth,frame,mode)
{
DEBUG("[FRM] Created manager for frame " + PYCREPR(m_frame));
m_lastInstruction = frame->f_lasti;
m_lastLine = frame->f_lineno;
}
//=============================================================================
// METHOD : SPELLwsWarmStartImpl::notifyLine()
//=============================================================================
void SPELLwsWarmStartImpl::notifyLine()
{
DEBUG("[WS] Notify line, top frame " + PYCREPR(m_topFrame->getFrameObject()));
// Notify the top frame to keep updated the recovery information
m_topFrame->eventLine();
// Perform state save if working mode is ON_LINE
if (getWorkingMode()==MODE_ON_LINE)
{
saveState();
}
}
//=============================================================================
// METHOD : SPELLwsWarmStartImpl::addFrame()
//=============================================================================
void SPELLwsWarmStartImpl::addFrame( const std::string& id, PyFrameObject* frame )
{
DEBUG("[WS] Adding new frame");
// Dont actually add it if it is the head (this happens after fixing the state)
if ( m_topFrame == NULL || frame != m_topFrame->getFrameObject() )
{
DEBUG("[WS] Adding frame manager for " + PYCREPR(frame));
m_topFrame = new SPELLwsFrame( id, m_startup, m_frames.size(), frame );
m_frames.push_back(m_topFrame);
m_recursionDepth++;
}
}
//=============================================================================
// METHOD: SPELLvariableMonitor::retrieveLocalVariables()
//=============================================================================
void SPELLvariableMonitor::retrieveLocalVariables(std::vector<SPELLvarInfo>& vars)
{
DEBUG("[VM] Retrieve Locals");
/*
* Bottom stack frame is discarded,
* as globals and locals are the same dictionary
*/
if (m_frame->f_back == NULL) return;
/*
* Get the names defined in the current code, including arguments
*/
std::vector<std::string> varNames = retrieveNames();
/*
* Iterate over the locals dictionary, retrieving the names contained in
* varNames
*/
PyFrame_FastToLocals(m_frame);
PyObject* dict = m_frame->f_locals;
DEBUG("[VM] Frame: " + PYCREPR(m_frame));
for( unsigned int index = 0; index< varNames.size(); index++)
{
std::string varName = varNames[index];
PyObject* pyVarName = SSTRPY(varName);
if (PyDict_Contains( dict, pyVarName ))
{
PyObject* object = PyDict_GetItem( dict, pyVarName );
if (!SPELLpythonHelper::instance().isInstance(object, "Database", "spell.lib.adapter.databases.database"))
{
if (PyCallable_Check(object)) continue;
if (PyClass_Check(object)) continue;
if (PyModule_Check(object)) continue;
if (PyInstance_Check(object)) continue;
}
DEBUG("[VM] Processing " + varName);
std::string type = PYSSTR( PyObject_Type(object) );
DEBUG("[VM] Type : " + type);
std::string value = PYREPR( object );
DEBUG("[VM] Value : " + value);
DEBUG("[VM] Global : " + BSTR(false));
DEBUG("[VM] Registered: " + BSTR(isRegistered(varName)));
// Mark empty values (empty strings) as "<empty>"
if (value == "") value = EMPTY_STRING;
vars.push_back( SPELLvarInfo(varName, type, value, false, isRegistered(varName)) );
}
}
PyFrame_LocalsToFast(m_frame,0);
}
//=============================================================================
// METHOD : SPELLwsFrame::eventLine()
//=============================================================================
void SPELLwsFrame::eventLine()
{
// On a line event we need to keep the latest instruction and line number used
// so that we can reapply them after a recovery.
m_lastInstruction = m_frame->f_lasti;
m_lastLine = m_frame->f_lineno;
DEBUG("[FRM] Frame " + PYCREPR(m_frame) + ": INS(" + ISTR(m_lastInstruction) + "), LIN(" + ISTR(m_lastLine) + ")");
// Update the tracked dynamic data of the frame
m_dynamic->update();
DEBUG("[FRM] Update on line event finished");
}
/** For debugging purposes only */
void DumpFrameInfo( PyFrameObject* frame )
{
std::cerr << "=============================================" << std::endl;
std::cerr << "Last instruction " << frame->f_lasti << std::endl;
std::cerr << "Last line " << frame->f_lineno << std::endl;
std::cerr << "Previous frame " << PYCREPR(frame->f_back) << std::endl;
std::cerr << "Thread state " << PSTR(frame->f_tstate) << std::endl;
std::cerr << "Try blocks count " << frame->f_iblock << std::endl;
std::cerr << "Try blocks " << PSTR(frame->f_blockstack) << std::endl;
std::cerr << "Value stack " << PSTR(frame->f_valuestack) << std::endl;
std::cerr << "Stack top " << PSTR(frame->f_stacktop) << std::endl;
std::cerr << "Stack count " << (frame->f_stacktop - frame->f_valuestack) << std::endl;
std::cerr << "Fast locals " << (frame->f_code->co_nlocals-1) << std::endl;
std::cerr << "=============================================" << std::endl;
}
//=============================================================================
// METHOD : SPELLwsStorage::loadObject
//=============================================================================
PyObject* SPELLwsStorage::loadObject()
{
if (!m_file.is_open()) return NULL;
if (m_mode == MODE_WRITE)
{
THROW_EXCEPTION("Unable to load object", "Initialized in write mode", SPELL_ERROR_WSTART);
}
// FORMAT IN FILE:
// COUNT \1 PTYPE \1 MARSHAL LENGTH
// MARSHAL
std::string line = "";
// Get first the line with the info
while(!m_file.eof() && (line == ""))
{
std::getline(m_file,line);
DEBUG("Obtained line [" + line + "]");
}
DEBUG("Load object from line [" + line + "]");
std::vector<std::string> elements = SPELLutils::tokenize(line,"\1");
PyObject* obj = NULL;
std::string ptype = elements[1];
int marshal_len = STRI(elements[2]);
DEBUG("Decoding object of type " + elements[1] + ", marshal length " + elements[2]);
// Get the line with the marshal
char buffer[4512];
m_file.read(buffer,marshal_len);
obj = (PyObject*) PyMarshal_ReadObjectFromString( buffer, marshal_len );
DEBUG("Decoded: " + PYCREPR(obj));
// Check that the unmarshal was ok
SPELLpythonHelper::instance().checkError();
m_opCounter++;
m_trace << "[" << m_opCounter << "] LOAD (" << m_filename << ") " << PYREPR(obj) << " [ Type=" << PYREPR(PyObject_Type(obj)) << ", Marshal length=" + ISTR(marshal_len) + " ]" << std::endl;
std::flush(m_trace);
if (obj != NULL) Py_INCREF(obj);
return obj;
}
//=============================================================================
// METHOD : SPELLwsWarmStartImpl::fixState()
//=============================================================================
PyFrameObject* SPELLwsWarmStartImpl::fixState()
{
DEBUG("[WS] Fixing state ==============================================");
// Synchronize so that nothing can be done while saving
SPELLmonitor m(m_lock);
// Get the head interpreter state
PyInterpreterState* istate = PyInterpreterState_Head();
// Get the current thread state
PyThreadState* oldState = PyThreadState_GET();
DEBUG("[WS] Old state: " + PSTR(oldState));
DEBUG("[WS] Interpreter head: " + PSTR(istate->tstate_head));
DEBUG("[WS] Interpreter next: " + PSTR(istate->next));
DEBUG("[WS] State recursion depth: " + ISTR(oldState->recursion_depth));
DEBUG("[WS] State next: " + PSTR(oldState->next));
// Create a fresh thread state
PyThreadState* newState = PyThreadState_New(istate);
istate->tstate_head = newState;
newState->recursion_depth = oldState->recursion_depth;
newState->tracing = oldState->tracing;
newState->use_tracing = oldState->use_tracing;
newState->tick_counter = oldState->tick_counter;
newState->gilstate_counter = oldState->gilstate_counter;
newState->dict = PyDict_Copy(oldState->dict);
FrameList::iterator it;
unsigned int frameCount = m_frames.size();
DEBUG("[WS] Total frames to fix " + ISTR(frameCount));
m_topFrame = NULL;
for( unsigned int index = 0; index < frameCount; index++)
{
bool isHead = (index == (frameCount-1));
DEBUG("[WS] Fix state on frame index " + ISTR(index) + " frame=" + PYCREPR(m_frames[index]));
m_topFrame = m_frames[index];
m_topFrame->fixState(newState, isHead);
}
DEBUG("[WS] State fixed ===============================================");
PyErr_Clear();
return m_topFrame->getFrameObject();
}
//=============================================================================
// STATIC : SPELLwsDataHandlerFactory::createDataHandler()
//=============================================================================
SPELLwsDataHandler* SPELLwsDataHandlerFactory::createDataHandler( PyObject* object )
{
assert(object != NULL);
SPELLwsDataHandler* handler = NULL;
DEBUG("[DHF] Creating handler for object of type " + PYREPR( PyObject_Type(object) ));
if (PyDict_Check(object))
{
DEBUG("[DHF] Object is a dictionary");
handler = new SPELLwsDictDataHandler(object);
}
else if (PyList_Check(object))
{
DEBUG("[DHF] Object is a list");
handler = new SPELLwsListDataHandler(object);
}
else if ( Py_None == object )
{
DEBUG("[DHF] Object is None");
handler = new SPELLwsNoneDataHandler();
}
else if (SPELLpythonHelper::instance().isDatabase(object))
{
DEBUG("[DHF] Object is database");
handler = new SPELLwsDbDataHandler(object);
}
else if (SPELLpythonHelper::instance().isTime(object))
{
DEBUG("[DHF] Object is TIME");
handler = new SPELLwsTimeDataHandler(object);
}
else if (PyClass_Check(object))
{
DEBUG("[DHF] Object is a class: " + PYCREPR(object));
handler = new SPELLwsClassDataHandler(object);
}
else if (PyInstance_Check(object))
{
DEBUG("[DHF] Object is a instance: " + PYCREPR(object));
handler = new SPELLwsInstanceDataHandler(object);
}
else if (SPELLpythonHelper::instance().isSubclassInstance(object, "TmItemClass", "spell.lib.adapter.tm_item"))
{
DEBUG("[DHF] Object is a TM item: " + PYCREPR(object));
handler = new SPELLwsTmItemDataHandler(object);
}
// Look discussion at mail.python.org/pipermail/python-dev/2004-July/046074.html
else if ((object->ob_type->tp_flags & Py_TPFLAGS_HEAPTYPE)>0)
{
DEBUG("[DHF] Object is a custom type: " + PYCREPR(object));
handler = new SPELLwsCustomTypeDataHandler(object);
}
else
{
DEBUG("[DHF] Default to object handler: " + PYCREPR(object));
handler = new SPELLwsObjectDataHandler(object);
}
return handler;
}
//=============================================================================
// DESTRUCTOR: SPELLwsFrame::~SPELLwsFrame
//=============================================================================
SPELLwsFrame::~SPELLwsFrame()
{
DEBUG("[FRM] Destroyed manager for frame " + PYCREPR(m_frame));
delete m_dynamic;
delete m_static;
}
//=============================================================================
// METHOD : SPELLwsWarmStartImpl::notifyCall()
//=============================================================================
void SPELLwsWarmStartImpl::notifyCall( const std::string& id, PyFrameObject* newFrame )
{
DEBUG("[WS] Notify call on " + PYCREPR(newFrame) + ", recursion depth " + ISTR(m_recursionDepth) +", id=" + id);
// Add the frame to the list of frames
addFrame( id, newFrame );
}
//=============================================================================
// METHOD : SPELLwsDictDataHandler::write()
//=============================================================================
void SPELLwsDictDataHandler::write()
{
if (getObject() == NULL)
{
getStorage()->storeLong( -1 );
return;
}
assert( PyDict_Check(getObject()));
SPELLpyHandle keys = PyDict_Keys( getObject() );
unsigned int numItems = PyList_Size( keys.get() );
DEBUG("[DDH] Storing dictionary items (total " + ISTR(numItems) + ") address " + PSTR(getObject()));
PyObject* key = NULL;
PyObject* item = NULL;
long toStore = 0;
// Calculate the number of items to store
// Store each list item
DEBUG("[DDH] Keys of the dictionary:");
for( unsigned int index = 0; index < numItems; index++)
{
key = PyList_GetItem( keys.get(), index );
item = PyDict_GetItem( getObject(), key );
if (SPELLwsWarmStartImpl::shouldFilter(key,item))
{
continue;
}
else
{
DEBUG(" - " + PYSSTR(key));
}
toStore++;
}
DEBUG("[DDH] Will store " + ISTR(toStore) + " keys");
// Store the number of items
getStorage()->storeLong( (long) toStore );
DEBUG("[DDH] Start checking items");
if (toStore>0)
{
// Store each list item
for( unsigned int index = 0; index < numItems; index++)
{
key = PyList_GetItem( keys.get(), index );
item = PyDict_GetItem( getObject(), key );
DEBUG("[DDH] Checking item " + PYCREPR(key));
// Do not consider filtered values
if (SPELLwsWarmStartImpl::shouldFilter(key,item)) continue;
SPELLpythonHelper::instance().checkError();
DEBUG(" [DDH] Key index " + ISTR(index));
DEBUG(" [DDH] Key to use" + PYREPR(key));
DEBUG(" [DDH] Item type:" + PYREPR(PyObject_Type(item)));
// Handler for the key
SPELLwsObjectDataHandler keyHandler(key);
keyHandler.setStorage(getStorage());
DEBUG(" [DDH] Creating handler");
// Create a handler for the item
SPELLwsDataHandler* handler = SPELLwsDataHandlerFactory::createDataHandler(item);
handler->setStorage(getStorage());
// Store the key
DEBUG(" [DDH] Storing key: " + PYREPR(key));
keyHandler.write();
// Store the item data code in order to recognise it later
DEBUG(" [DDH] Storing data code: " + SPELLwsData::codeStr(handler->getCode()));
handler->storeDataCode();
// IMPORTANT in the case of lists and dictionaries, we want to be able to continue
// the storage even if there is a problem in the handler processing at this point.
// If that is the case, a fake empty object will be replaced by the object being
// processed by the handler, and the dumping of this collection will continue.
try
{
if (handler->getCode() == SPELLwsData::DATA_CUSTOM_TYPE )
{
std::string msg = "WARNING! warm start not supported for custom Python types (" + PYREPR(key) + "=" + PYREPR(item) + ")";
LOG_WARN(msg);
SPELLexecutor::instance().getCIF().warning(msg);
storeFakeObject( SPELLwsData::DATA_NONE );
}
else
{
// Store the value
DEBUG(" [DDH] Storing value: " + PYREPR(item));
handler->write();
DEBUG(" [DDH] Storing value done");
}
}
catch(SPELLcoreException& ex)
{
std::string msg = "WARNING! WS storage of element " + ISTR(index) + " failed: " + ex.what();
LOG_WARN(msg);
SPELLexecutor::instance().getCIF().warning(msg);
storeFakeObject( handler->getCode() );
}
delete handler;
}
}
DEBUG("[DDH] Storing dictionary done");
}
//=============================================================================
// DESTRUCTOR: SPELLwsFrame::~SPELLwsFrame
//=============================================================================
SPELLwsFrame::~SPELLwsFrame()
{
DEBUG("[FRM] Destroyed manager for frame " + PYCREPR(m_frame));
// IMPORTANT if this frame manager is destroyed we do not need the persistent data
// anymore, destroy the files
}
