int _write_value_to_handle(const void *value, void *handle)
{
PyObject *py_value = (PyObject *)value,
*py_marshalled = NULL;
char *marshalled;
Py_ssize_t length;
int success = 0;
#ifdef Py_MARSHAL_VERSION
if(!(py_marshalled =
PyMarshal_WriteObjectToString(py_value, Py_MARSHAL_VERSION)))
goto _write_value_to_handle_cleanup;
#else
if(!(py_marshalled = PyMarshal_WriteObjectToString(py_value)))
goto _write_value_to_handle_cleanup;
#endif
if(PyString_AsStringAndSize(py_marshalled, &marshalled, &length) == -1)
goto _write_value_to_handle_cleanup;
if(!_write_to_handle(&length, sizeof(length), handle))
goto _write_value_to_handle_cleanup;
if (length != (int)length)
goto _write_value_to_handle_cleanup;
if(!_write_to_handle(marshalled, (int)length, handle))
goto _write_value_to_handle_cleanup;
success = 1;
_write_value_to_handle_cleanup:
if(py_marshalled) {
Py_DECREF(py_marshalled);
}
return success;
}
static int _write_value_to_handle(const void *value, void *handle)
{
PyObject *py_value = (PyObject *)value,
*py_marshalled = NULL,
*bytes = NULL;
char *marshalled;
Py_ssize_t length;
int success = 0;
#ifdef Py_MARSHAL_VERSION
if(!(py_marshalled =
PyMarshal_WriteObjectToString(py_value, Py_MARSHAL_VERSION)))
goto _write_value_to_handle_cleanup;
#else
if(!(py_marshalled = PyMarshal_WriteObjectToString(py_value)))
goto _write_value_to_handle_cleanup;
#endif
#ifdef IS_PY3K
if (!PyBytes_Check(py_marshalled)) {
PyErr_SetString(PyExc_TypeError, "marshalled data expected to be bytes");
goto _write_value_to_handle_cleanup;
}
if(PyBytes_AsStringAndSize(py_marshalled, &marshalled, &length) == -1)
#else
if(PyString_AsStringAndSize(py_marshalled, &marshalled, &length) == -1)
#endif
goto _write_value_to_handle_cleanup;
if(!_write_to_handle(&length, sizeof(length), handle))
goto _write_value_to_handle_cleanup;
if (length != (int)length)
goto _write_value_to_handle_cleanup;
if(!_write_to_handle(marshalled, (int)length, handle))
goto _write_value_to_handle_cleanup;
success = 1;
_write_value_to_handle_cleanup:
Py_XDECREF(py_marshalled);
Py_XDECREF(bytes);
return success;
}
int
csl_compile(char *str, char *name, char **codeptr, size_t *sizeptr)
{
PyObject *pCode, *pStr;
node *n;
size_t size;
// compile into a code object
n = PyParser_SimpleParseString(str, Py_file_input);
if (!n) {
log_exception();
return CSL_BADCODE;
}
pCode = (PyObject *) PyNode_Compile(n, name);
PyNode_Free(n);
if (!pCode) {
log_exception();
return CSL_BADCODE;
}
// serialize code object
#if PY_MINOR_VERSION == 3
pStr = PyMarshal_WriteObjectToString(pCode);
#else
pStr = PyMarshal_WriteObjectToString(pCode, 0);
#endif
Py_DECREF(pCode);
if (!pStr) {
return CSL_NOMEM;
}
size = PyString_Size(pStr);
*codeptr = malloc(size);
if (!*codeptr) {
Py_DECREF(pStr);
return CSL_NOMEM;
}
memcpy(*codeptr, PyString_AsString(pStr), size);
*sizeptr = size;
Py_DECREF(pStr);
return 0;
}
/* uwsgi MARSHAL|33 */
int uwsgi_request_marshal(struct uwsgi_server *uwsgi, struct wsgi_request *wsgi_req) {
PyObject *func_result;
PyObject *umm = PyDict_GetItemString(uwsgi->embedded_dict,
"message_manager_marshal");
if (umm) {
PyObject *ummo = PyMarshal_ReadObjectFromString(wsgi_req->buffer,
wsgi_req->uh.pktsize);
if (ummo) {
if (!PyTuple_SetItem(uwsgi->embedded_args, 0, ummo)) {
if (!PyTuple_SetItem(uwsgi->embedded_args, 1, PyInt_FromLong(wsgi_req->uh.modifier2))) {
func_result = PyEval_CallObject(umm, uwsgi->embedded_args);
if (PyErr_Occurred()) {
PyErr_Print();
}
if (func_result) {
PyObject *marshalled = PyMarshal_WriteObjectToString(func_result, 1);
if (!marshalled) {
PyErr_Print();
}
else {
if (PyString_Size(marshalled) <= 0xFFFF) {
wsgi_req->uh.pktsize = (uint16_t)
PyString_Size(marshalled);
if (write(wsgi_req->poll.fd, wsgi_req, 4) == 4) {
if (write(wsgi_req->poll.fd, PyString_AsString(marshalled), wsgi_req->uh.pktsize) != wsgi_req->uh.pktsize) {
uwsgi_error("write()");
}
}
else {
uwsgi_error("write()");
}
}
else {
fprintf(stderr, "marshalled object is too big. skip\n");
}
Py_DECREF(marshalled);
}
Py_DECREF(func_result);
}
}
}
//Py_DECREF(ummo);
}
}
PyErr_Clear();
return 0;
}
//=============================================================================
// METHOD : SPELLwsStorage::storeObject
//=============================================================================
void SPELLwsStorage::storeObject( PyObject* object )
{
if (!m_file.is_open()) return;
if (object == NULL)
{
THROW_EXCEPTION("Unable to store object", "Null reference given", SPELL_ERROR_WSTART);
}
if (m_mode == MODE_READ)
{
THROW_EXCEPTION("Unable to store object", "Initialized in read mode", SPELL_ERROR_WSTART);
}
std::string ptype = PYREPR(PyObject_Type(object));
DEBUG("Encoding object " + PYREPR(object) + " of type " + ptype);
// Marshal the object to the persistent storage file
PyObject* marshal = PyMarshal_WriteObjectToString( object, LATEST_MARSHAL_VERSION );
// Ensure there is no internal Python error
SPELLpythonHelper::instance().checkError();
int marshal_len = PyObject_Length(marshal);
DEBUG("Marshal length: " + ISTR(marshal_len));
char* encoded = PyString_AsString(marshal);
// FORMAT IN FILE:
// COUNT \1 PTYPE \1 MARSHAL LENGTH
// MARSHAL
// If the marshal was OK, continue and store in file: OP,TYPE,MARSHALLED
m_opCounter++;
m_file << m_opCounter << "\1" << ptype << "\1" << marshal_len << std::endl;
m_file.write(encoded,marshal_len); m_file << std::endl;
std::flush(m_file);
m_trace << "[" << m_opCounter << "] STORE (" << m_filename << ") " + PYREPR(object) << " [ Type=" + ptype << ", Marhsal length=" + ISTR(marshal_len) + " ]" << std::endl;
std::flush(m_trace);
}
int
main(int argc, char *argv[])
{
char *inpath, *outpath;
FILE *infile, *outfile = NULL;
struct stat st;
size_t text_size, data_size, n;
char *text;
unsigned char *data;
PyObject *code, *marshalled;
if (argc != 3) {
fprintf(stderr, "need to specify input and output paths\n");
return 2;
}
inpath = argv[1];
outpath = argv[2];
infile = fopen(inpath, "rb");
if (infile == NULL) {
fprintf(stderr, "cannot open '%s' for reading\n", inpath);
return 1;
}
if (fstat(fileno(infile), &st)) {
fclose(infile);
fprintf(stderr, "cannot fstat '%s'\n", inpath);
return 1;
}
text_size = st.st_size;
text = (char *) malloc(text_size + 1);
if (text == NULL) {
fclose(infile);
fprintf(stderr, "could not allocate %ld bytes\n", (long) text_size);
return 1;
}
n = fread(text, 1, text_size, infile);
fclose(infile);
infile = NULL;
if (n < text_size) {
fprintf(stderr, "read too short: got %ld instead of %ld bytes\n",
(long) n, (long) text_size);
return 1;
}
text[text_size] = '\0';
Py_NoUserSiteDirectory++;
Py_NoSiteFlag++;
Py_IgnoreEnvironmentFlag++;
Py_SetProgramName(L"./_freeze_importlib");
/* Don't install importlib, since it could execute outdated bytecode. */
_Py_InitializeEx_Private(1, 0);
code = Py_CompileStringExFlags(text, "<frozen importlib._bootstrap>",
Py_file_input, NULL, 0);
if (code == NULL)
goto error;
marshalled = PyMarshal_WriteObjectToString(code, Py_MARSHAL_VERSION);
Py_DECREF(code);
if (marshalled == NULL)
goto error;
assert(PyBytes_CheckExact(marshalled));
data = (unsigned char *) PyBytes_AS_STRING(marshalled);
data_size = PyBytes_GET_SIZE(marshalled);
outfile = fopen(outpath, "wb");
if (outfile == NULL) {
fprintf(stderr, "cannot open '%s' for writing\n", outpath);
return 1;
}
fprintf(outfile, "%s\n", header);
fprintf(outfile, "unsigned char _Py_M__importlib[] = {\n");
for (n = 0; n < data_size; n += 16) {
size_t i, end = Py_MIN(n + 16, data_size);
fprintf(outfile, " ");
for (i = n; i < end; i++) {
fprintf(outfile, "%d,", (unsigned int) data[i]);
}
fprintf(outfile, "\n");
}
fprintf(outfile, "};\n");
Py_DECREF(marshalled);
Py_Finalize();
if (infile)
fclose(infile);
if (outfile) {
if (ferror(outfile)) {
fprintf(stderr, "error when writing to '%s'\n", outpath);
fclose(outfile);
return 1;
}
fclose(outfile);
}
return 0;
error:
PyErr_Print();
Py_Finalize();
if (infile)
fclose(infile);
if (outfile)
fclose(outfile);
return 1;
}
/*
* This routine needs to emulate load_source_module in import.c in
* python, but with DataSectionPtr using FILE*
* However, we can't get an mTime (there's no interface for it in DataSection)
* Need to resolve this before we can commit
*/
PyObject* PyResMgrImportLoader::load_source_module( const std::string& name, BinaryPtr py_data, DataSectionPtr pDirectory )
{
std::string compiledExtension = "pyc";
if ( Py_OptimizeFlag )
compiledExtension = "pyo";
std::string moduleName = name;
std::string::size_type dotPos = name.rfind( "." );
if ( dotPos != std::string::npos )
{
moduleName = name.substr( dotPos + 1 );
}
// Find source (.py) and object (.pyc/.pyo) files to
// process for this source module or package.
std::string modulePath;
bool isPackage = false;
std::string objectModuleName;
if ( modules_[ name ].first == PKG_DIRECTORY )
{
isPackage = true;
modulePath = path_ + "/" + moduleName + "/__init__.py";
objectModuleName = "__init__." + compiledExtension;
} else {
modulePath = path_ + "/" + moduleName + ".py";
objectModuleName = moduleName + "." + compiledExtension;
}
// Remove this from the cache. We have it now, and will feed it to Python.
// This ensures that a reload() call will work correctly.
modules_.erase( name );
// Fetch mtime of py file
time_t pyModTime = static_cast< time_t >( -1 );
IFileSystem::FileInfo fInfo;
IFileSystem::FileType fType =
BWResource::instance().fileSystem()->getFileType( modulePath, &fInfo );
if ( fType != IFileSystem::FT_NOT_FOUND )
{
pyModTime = fInfo.modified;
}
// If possible, palm this off to load_compiled_module
DataSectionPtr pycSection = pDirectory->openSection( objectModuleName );
if ( pycSection && check_compiled_module( name, pycSection->asBinary() ))
{
if ( check_compiled_module_mtime( name, pycSection->asBinary(), pyModTime ) )
{
// We know the module was valid and up-to-date, so trust the loader
// to either load it or fail noisily
return load_compiled_module( name, pycSection->asBinary(), true );
}
}
if ( pycSection )
{
// Get rid of our reference to the old compiled python file.
// TODO: Purge this section from the DataSection cache, we're about
// to replace it on disk
pycSection = NULL;
}
// We got here, the object file for this source either doesn't exist, isn't
// valid, or isn't as recent as the source.
// Emulate parse_source_module
// The code string needs to have (\n) as a line seperator, and needs an EOF
// (-1) or null termination, and has to end in a newline.
// Also, need to ensure there's no embedded nulls.
// So have to make a copy of the string.
// We shouldn't ever do this in release anyway.
std::string codeString( py_data->cdata(), py_data->len() );
if ( codeString.find( '\0' ) != std::string::npos )
{
PyErr_Format( PyExc_ImportError,
"%s contains an embedded null character",
modulePath.c_str()
);
return NULL;
}
std::string::size_type winNLpos;
// Convert any Windows newlines into UNIX newlines
if ( ( winNLpos = codeString.find( "\r\n", 0 ) ) != std::string::npos )
{
do
{
codeString.replace( winNLpos, 2, "\n" );
winNLpos = codeString.find( "\r\n", winNLpos );
} while ( winNLpos != std::string::npos );
}
// Ensure we're newline-terminated
codeString.append( "\n" );
PyObject* codeObject = Py_CompileString( codeString.c_str(),
const_cast< char* >( modulePath.c_str() ),
Py_file_input );
if ( codeObject == NULL )
// Compiler didn't like it. Propagate the error up
return NULL;
// OK, we have a module, now we just execute it into the correct space.
// Always call it a .py, even though we've created a .pyc
PyObject* module = PyImport_ExecCodeModuleEx(
const_cast< char* >( name.c_str() ), codeObject,
const_cast< char* >( modulePath.c_str() )
);
if ( module == NULL )
{
Py_DECREF( codeObject );
return NULL;
}
// It executed OK, so write out an object file for later use, if possible
// Emulates write_compiled_module( co, cpathname, mtime )
do {
PyObject* codeString = PyMarshal_WriteObjectToString( codeObject, Py_MARSHAL_VERSION );
// XXX: Maybe we should care if _this_ fails, or at least report it?
if ( codeString == NULL || !PyString_Check( codeString ) )
{
PyErr_Clear();
break;
}
char* dataBlock = new char[ PyString_Size( codeString ) + 8 ];
// XXX: On-disk format is little-endian, we're not checking that here
reinterpret_cast< uint32* >(dataBlock)[ 0 ] = PyImport_GetMagicNumber();
reinterpret_cast< int32* >(dataBlock)[ 1 ] = static_cast< int32 >( pyModTime );
memcpy( dataBlock + 8, PyString_AsString( codeString ), PyString_Size( codeString ) );
// The following is a little nasty, we end up copying the data a couple of times
// Wrap dataBlock in a BinaryBlock (which takes a copy of it)
BinaryPtr pycData = new BinaryBlock( dataBlock,
PyString_Size( codeString ) + 8,
"PyResMgrImportLoader::load_source_module" );
delete[] dataBlock;
if ( !pycData )
break;
// Save out our new pyc file
pycSection = pDirectory->openSection( objectModuleName, true, BinSection::creator() );
if ( !pycSection )
break;
pycSection->setBinary( pycData );
pycSection->save();
} while( false );
Py_DECREF( codeObject );
PyObject *moduleDict = PyModule_GetDict( module );
if ( moduleDict != NULL )
{
int err = PyDict_SetItemString( moduleDict, "__loader__", this );
if ( err != 0 )
{
Py_DECREF( module );
return NULL;
}
}
// TRACE_MSG( "PyResMgrImportLoader(%s)::load_source_module: loaded %s\n", path_.c_str(),
// name.c_str() );
return module;
}
