/* Given the contents of a .py[co] file in a buffer, unmarshal the data
and return the code object. Return None if it the magic word doesn't
match (we do this instead of raising an exception as we fall back
to .py if available and we don't want to mask other errors).
Returns a new reference. */
static PyObject *
unmarshal_code(char *pathname, PyObject *data)
{
PyObject *code;
char *buf = PyBytes_AsString(data);
Py_ssize_t size = PyBytes_Size(data);
if (size <= 9) {
PyErr_SetString(PyExc_TypeError,
"bad pyc data");
return NULL;
}
if (get_long((unsigned char *)buf) != PyImport_GetMagicNumber()) {
if (Py_VerboseFlag)
PySys_WriteStderr("# %s has bad magic\n",
pathname);
Py_INCREF(Py_None);
return NULL;
}
code = PyMarshal_ReadObjectFromString(buf + 8, size - 8);
if (code == NULL)
return NULL;
if (!PyCode_Check(code)) {
Py_DECREF(code);
PyErr_Format(PyExc_TypeError,
"compiled module %.200s is not a code object",
pathname);
return NULL;
}
return code;
}
static PyObject *
run_pyc_file(FILE *fp, char *filename, PyObject *globals, PyObject *locals)
{
PyCodeObject *co;
PyObject *v;
long magic;
long PyImport_GetMagicNumber(void);
magic = PyMarshal_ReadLongFromFile(fp);
if (magic != PyImport_GetMagicNumber()) {
PyErr_SetString(PyExc_RuntimeError,
"Bad magic number in .pyc file");
return NULL;
}
(void) PyMarshal_ReadLongFromFile(fp);
v = PyMarshal_ReadObjectFromFile(fp);
fclose(fp);
if (v == NULL || !PyCode_Check(v)) {
Py_XDECREF(v);
PyErr_SetString(PyExc_RuntimeError,
"Bad code object in .pyc file");
return NULL;
}
co = (PyCodeObject *)v;
v = PyEval_EvalCode(co, globals, locals);
Py_DECREF(co);
return v;
}
/**
* This routine checks that the named module's Python Object data is valid,
* at least as far as the file header is concerned.
*
* This method will not set a Python error if the object is invalid.
*
* @param name The name of the python module to check
*
* @param pyc_data A BinaryPtr holding the data from the Python Object
*
* @param mtime Optional mtime to test against
*
* @return Whether or not the file is considered valid and up-to-date
*/
bool PyResMgrImportLoader::check_compiled_module( const std::string& name,
BinaryPtr pyc_data )
{
// Check it's not too short
if ( pyc_data->len() < 8 )
{
return false;
}
// Check we have PYC magic
// PyImport_GetMagicNumber() returns a long, but the on-disk
// format is 4 bytes. If we don't do this unsigned, there's
// a sign-extension risk. So we just truncate to 32-bits instead.
// The current value as of Python 2.5.2 has the high-bit unset,
// and that should never change.
// If you decide to customise the PYC storage format, make sure
// you change the magic number
// XXX: On-disk format is little-endian, we're not checking that here
if ( reinterpret_cast< const uint32* >(pyc_data->data())[ 0 ]
!= static_cast< const uint32 >(PyImport_GetMagicNumber()) )
{
return false;
}
return true;
}
/* Given the contents of a .pyc file in a buffer, unmarshal the data
and return the code object. Return None if it the magic word doesn't
match (we do this instead of raising an exception as we fall back
to .py if available and we don't want to mask other errors).
Returns a new reference. */
static PyObject *
unmarshal_code(PyObject *pathname, PyObject *data, time_t mtime)
{
PyObject *code;
unsigned char *buf = (unsigned char *)PyBytes_AsString(data);
Py_ssize_t size = PyBytes_Size(data);
if (size < 16) {
PyErr_SetString(ZipImportError,
"bad pyc data");
return NULL;
}
if (get_uint32(buf) != (unsigned int)PyImport_GetMagicNumber()) {
if (Py_VerboseFlag) {
PySys_FormatStderr("# %R has bad magic\n",
pathname);
}
Py_RETURN_NONE; /* signal caller to try alternative */
}
uint32_t flags = get_uint32(buf + 4);
if (flags != 0) {
// Hash-based pyc. We currently refuse to handle checked hash-based
// pycs. We could validate hash-based pycs against the source, but it
// seems likely that most people putting hash-based pycs in a zipfile
// will use unchecked ones.
if (strcmp(_Py_CheckHashBasedPycsMode, "never") &&
(flags != 0x1 || !strcmp(_Py_CheckHashBasedPycsMode, "always")))
Py_RETURN_NONE;
} else if ((mtime != 0 && !eq_mtime(get_uint32(buf + 8), mtime))) {
if (Py_VerboseFlag) {
PySys_FormatStderr("# %R has bad mtime\n",
pathname);
}
Py_RETURN_NONE; /* signal caller to try alternative */
}
/* XXX the pyc's size field is ignored; timestamp collisions are probably
unimportant with zip files. */
code = PyMarshal_ReadObjectFromString((char *)buf + 16, size - 16);
if (code == NULL) {
return NULL;
}
if (!PyCode_Check(code)) {
Py_DECREF(code);
PyErr_Format(PyExc_TypeError,
"compiled module %R is not a code object",
pathname);
return NULL;
}
return code;
}
/* Given the contents of a .py[co] file in a buffer, unmarshal the data
and return the code object. Return None if it the magic word doesn't
match (we do this instead of raising an exception as we fall back
to .py if available and we don't want to mask other errors).
Returns a new reference. */
static PyObject *
unmarshal_code(PyObject *pathname, PyObject *data, time_t mtime)
{
PyObject *code;
unsigned char *buf = (unsigned char *)PyBytes_AsString(data);
Py_ssize_t size = PyBytes_Size(data);
if (size < 12) {
PyErr_SetString(ZipImportError,
"bad pyc data");
return NULL;
}
if (get_uint32(buf) != (unsigned int)PyImport_GetMagicNumber()) {
if (Py_VerboseFlag) {
PySys_FormatStderr("# %R has bad magic\n",
pathname);
}
Py_INCREF(Py_None);
return Py_None; /* signal caller to try alternative */
}
if (mtime != 0 && !eq_mtime(get_uint32(buf + 4), mtime)) {
if (Py_VerboseFlag) {
PySys_FormatStderr("# %R has bad mtime\n",
pathname);
}
Py_INCREF(Py_None);
return Py_None; /* signal caller to try alternative */
}
/* XXX the pyc's size field is ignored; timestamp collisions are probably
unimportant with zip files. */
code = PyMarshal_ReadObjectFromString((char *)buf + 12, size - 12);
if (code == NULL) {
return NULL;
}
if (!PyCode_Check(code)) {
Py_DECREF(code);
PyErr_Format(PyExc_TypeError,
"compiled module %R is not a code object",
pathname);
return NULL;
}
return code;
}
/* Given the contents of a .py[co] file in a buffer, unmarshal the data
and return the code object. Return None if it the magic word doesn't
match (we do this instead of raising an exception as we fall back
to .py if available and we don't want to mask other errors).
Returns a new reference. */
static PyObject *
unmarshal_code(const char *pathname, PyObject *data, time_t mtime)
{
PyObject *code;
unsigned char *buf = (unsigned char *)PyString_AsString(data);
Py_ssize_t size = PyString_Size(data);
if (size < 8) {
PyErr_SetString(ZipImportError,
"bad pyc data");
return NULL;
}
if (get_uint32(buf) != (unsigned int)PyImport_GetMagicNumber()) {
if (Py_VerboseFlag) {
PySys_WriteStderr("# %s has bad magic\n",
pathname);
}
Py_INCREF(Py_None);
return Py_None; /* signal caller to try alternative */
}
if (mtime != 0 && !eq_mtime(get_uint32(buf + 4), mtime)) {
if (Py_VerboseFlag) {
PySys_WriteStderr("# %s has bad mtime\n",
pathname);
}
Py_INCREF(Py_None);
return Py_None; /* signal caller to try alternative */
}
code = PyMarshal_ReadObjectFromString((char *)buf + 8, size - 8);
if (code == NULL) {
return NULL;
}
if (!PyCode_Check(code)) {
Py_DECREF(code);
PyErr_Format(PyExc_TypeError,
"compiled module %.200s is not a code object",
pathname);
return NULL;
}
return code;
}
static PyObject *
run_pyc_file(FILE *fp, const char *filename, PyObject *globals,
PyObject *locals, PyCompilerFlags *flags)
{
PyCodeObject *co;
PyObject *v;
long magic;
long PyImport_GetMagicNumber(void);
magic = PyMarshal_ReadLongFromFile(fp);
if (magic != PyImport_GetMagicNumber()) {
if (!PyErr_Occurred())
PyErr_SetString(PyExc_RuntimeError,
"Bad magic number in .pyc file");
goto error;
}
/* Skip the rest of the header. */
(void) PyMarshal_ReadLongFromFile(fp);
(void) PyMarshal_ReadLongFromFile(fp);
(void) PyMarshal_ReadLongFromFile(fp);
if (PyErr_Occurred()) {
goto error;
}
v = PyMarshal_ReadLastObjectFromFile(fp);
if (v == NULL || !PyCode_Check(v)) {
Py_XDECREF(v);
PyErr_SetString(PyExc_RuntimeError,
"Bad code object in .pyc file");
goto error;
}
fclose(fp);
co = (PyCodeObject *)v;
v = run_eval_code_obj(co, globals, locals);
if (v && flags)
flags->cf_flags |= (co->co_flags & PyCF_MASK);
Py_DECREF(co);
return v;
error:
fclose(fp);
return NULL;
}
static int
maybe_pyc_file(FILE *fp, const char* filename, const char* ext, int closeit)
{
if (strcmp(ext, ".pyc") == 0)
return 1;
/* Only look into the file if we are allowed to close it, since
it then should also be seekable. */
if (closeit) {
/* Read only two bytes of the magic. If the file was opened in
text mode, the bytes 3 and 4 of the magic (\r\n) might not
be read as they are on disk. */
unsigned int halfmagic = PyImport_GetMagicNumber() & 0xFFFF;
unsigned char buf[2];
/* Mess: In case of -x, the stream is NOT at its start now,
and ungetc() was used to push back the first newline,
which makes the current stream position formally undefined,
and a x-platform nightmare.
Unfortunately, we have no direct way to know whether -x
was specified. So we use a terrible hack: if the current
stream position is not 0, we assume -x was specified, and
give up. Bug 132850 on SourceForge spells out the
hopelessness of trying anything else (fseek and ftell
don't work predictably x-platform for text-mode files).
*/
int ispyc = 0;
if (ftell(fp) == 0) {
if (fread(buf, 1, 2, fp) == 2 &&
((unsigned int)buf[1]<<8 | buf[0]) == halfmagic)
ispyc = 1;
rewind(fp);
}
return ispyc;
}
return 0;
}
/*
* This routine needs to emulate load_source_module in import.c in
* python, but with DataSectionPtr using FILE*
* However, we can't get an mTime (there's no interface for it in DataSection)
* Need to resolve this before we can commit
*/
PyObject* PyResMgrImportLoader::load_source_module( const std::string& name, BinaryPtr py_data, DataSectionPtr pDirectory )
{
std::string compiledExtension = "pyc";
if ( Py_OptimizeFlag )
compiledExtension = "pyo";
std::string moduleName = name;
std::string::size_type dotPos = name.rfind( "." );
if ( dotPos != std::string::npos )
{
moduleName = name.substr( dotPos + 1 );
}
// Find source (.py) and object (.pyc/.pyo) files to
// process for this source module or package.
std::string modulePath;
bool isPackage = false;
std::string objectModuleName;
if ( modules_[ name ].first == PKG_DIRECTORY )
{
isPackage = true;
modulePath = path_ + "/" + moduleName + "/__init__.py";
objectModuleName = "__init__." + compiledExtension;
} else {
modulePath = path_ + "/" + moduleName + ".py";
objectModuleName = moduleName + "." + compiledExtension;
}
// Remove this from the cache. We have it now, and will feed it to Python.
// This ensures that a reload() call will work correctly.
modules_.erase( name );
// Fetch mtime of py file
time_t pyModTime = static_cast< time_t >( -1 );
IFileSystem::FileInfo fInfo;
IFileSystem::FileType fType =
BWResource::instance().fileSystem()->getFileType( modulePath, &fInfo );
if ( fType != IFileSystem::FT_NOT_FOUND )
{
pyModTime = fInfo.modified;
}
// If possible, palm this off to load_compiled_module
DataSectionPtr pycSection = pDirectory->openSection( objectModuleName );
if ( pycSection && check_compiled_module( name, pycSection->asBinary() ))
{
if ( check_compiled_module_mtime( name, pycSection->asBinary(), pyModTime ) )
{
// We know the module was valid and up-to-date, so trust the loader
// to either load it or fail noisily
return load_compiled_module( name, pycSection->asBinary(), true );
}
}
if ( pycSection )
{
// Get rid of our reference to the old compiled python file.
// TODO: Purge this section from the DataSection cache, we're about
// to replace it on disk
pycSection = NULL;
}
// We got here, the object file for this source either doesn't exist, isn't
// valid, or isn't as recent as the source.
// Emulate parse_source_module
// The code string needs to have (\n) as a line seperator, and needs an EOF
// (-1) or null termination, and has to end in a newline.
// Also, need to ensure there's no embedded nulls.
// So have to make a copy of the string.
// We shouldn't ever do this in release anyway.
std::string codeString( py_data->cdata(), py_data->len() );
if ( codeString.find( '\0' ) != std::string::npos )
{
PyErr_Format( PyExc_ImportError,
"%s contains an embedded null character",
modulePath.c_str()
);
return NULL;
}
std::string::size_type winNLpos;
// Convert any Windows newlines into UNIX newlines
if ( ( winNLpos = codeString.find( "\r\n", 0 ) ) != std::string::npos )
{
do
{
codeString.replace( winNLpos, 2, "\n" );
winNLpos = codeString.find( "\r\n", winNLpos );
} while ( winNLpos != std::string::npos );
}
// Ensure we're newline-terminated
codeString.append( "\n" );
PyObject* codeObject = Py_CompileString( codeString.c_str(),
const_cast< char* >( modulePath.c_str() ),
Py_file_input );
if ( codeObject == NULL )
// Compiler didn't like it. Propagate the error up
return NULL;
// OK, we have a module, now we just execute it into the correct space.
// Always call it a .py, even though we've created a .pyc
PyObject* module = PyImport_ExecCodeModuleEx(
const_cast< char* >( name.c_str() ), codeObject,
const_cast< char* >( modulePath.c_str() )
);
if ( module == NULL )
{
Py_DECREF( codeObject );
return NULL;
}
// It executed OK, so write out an object file for later use, if possible
// Emulates write_compiled_module( co, cpathname, mtime )
do {
PyObject* codeString = PyMarshal_WriteObjectToString( codeObject, Py_MARSHAL_VERSION );
// XXX: Maybe we should care if _this_ fails, or at least report it?
if ( codeString == NULL || !PyString_Check( codeString ) )
{
PyErr_Clear();
break;
}
char* dataBlock = new char[ PyString_Size( codeString ) + 8 ];
// XXX: On-disk format is little-endian, we're not checking that here
reinterpret_cast< uint32* >(dataBlock)[ 0 ] = PyImport_GetMagicNumber();
reinterpret_cast< int32* >(dataBlock)[ 1 ] = static_cast< int32 >( pyModTime );
memcpy( dataBlock + 8, PyString_AsString( codeString ), PyString_Size( codeString ) );
// The following is a little nasty, we end up copying the data a couple of times
// Wrap dataBlock in a BinaryBlock (which takes a copy of it)
BinaryPtr pycData = new BinaryBlock( dataBlock,
PyString_Size( codeString ) + 8,
"PyResMgrImportLoader::load_source_module" );
delete[] dataBlock;
if ( !pycData )
break;
// Save out our new pyc file
pycSection = pDirectory->openSection( objectModuleName, true, BinSection::creator() );
if ( !pycSection )
break;
pycSection->setBinary( pycData );
pycSection->save();
} while( false );
Py_DECREF( codeObject );
PyObject *moduleDict = PyModule_GetDict( module );
if ( moduleDict != NULL )
{
int err = PyDict_SetItemString( moduleDict, "__loader__", this );
if ( err != 0 )
{
Py_DECREF( module );
return NULL;
}
}
// TRACE_MSG( "PyResMgrImportLoader(%s)::load_source_module: loaded %s\n", path_.c_str(),
// name.c_str() );
return module;
}
