int _write_value_to_handle(const void *value, void *handle) { PyObject *py_value = (PyObject *)value, *py_marshalled = NULL; char *marshalled; Py_ssize_t length; int success = 0; #ifdef Py_MARSHAL_VERSION if(!(py_marshalled = PyMarshal_WriteObjectToString(py_value, Py_MARSHAL_VERSION))) goto _write_value_to_handle_cleanup; #else if(!(py_marshalled = PyMarshal_WriteObjectToString(py_value))) goto _write_value_to_handle_cleanup; #endif if(PyString_AsStringAndSize(py_marshalled, &marshalled, &length) == -1) goto _write_value_to_handle_cleanup; if(!_write_to_handle(&length, sizeof(length), handle)) goto _write_value_to_handle_cleanup; if (length != (int)length) goto _write_value_to_handle_cleanup; if(!_write_to_handle(marshalled, (int)length, handle)) goto _write_value_to_handle_cleanup; success = 1; _write_value_to_handle_cleanup: if(py_marshalled) { Py_DECREF(py_marshalled); } return success; }
static int _write_value_to_handle(const void *value, void *handle) { PyObject *py_value = (PyObject *)value, *py_marshalled = NULL, *bytes = NULL; char *marshalled; Py_ssize_t length; int success = 0; #ifdef Py_MARSHAL_VERSION if(!(py_marshalled = PyMarshal_WriteObjectToString(py_value, Py_MARSHAL_VERSION))) goto _write_value_to_handle_cleanup; #else if(!(py_marshalled = PyMarshal_WriteObjectToString(py_value))) goto _write_value_to_handle_cleanup; #endif #ifdef IS_PY3K if (!PyBytes_Check(py_marshalled)) { PyErr_SetString(PyExc_TypeError, "marshalled data expected to be bytes"); goto _write_value_to_handle_cleanup; } if(PyBytes_AsStringAndSize(py_marshalled, &marshalled, &length) == -1) #else if(PyString_AsStringAndSize(py_marshalled, &marshalled, &length) == -1) #endif goto _write_value_to_handle_cleanup; if(!_write_to_handle(&length, sizeof(length), handle)) goto _write_value_to_handle_cleanup; if (length != (int)length) goto _write_value_to_handle_cleanup; if(!_write_to_handle(marshalled, (int)length, handle)) goto _write_value_to_handle_cleanup; success = 1; _write_value_to_handle_cleanup: Py_XDECREF(py_marshalled); Py_XDECREF(bytes); return success; }
int csl_compile(char *str, char *name, char **codeptr, size_t *sizeptr) { PyObject *pCode, *pStr; node *n; size_t size; // compile into a code object n = PyParser_SimpleParseString(str, Py_file_input); if (!n) { log_exception(); return CSL_BADCODE; } pCode = (PyObject *) PyNode_Compile(n, name); PyNode_Free(n); if (!pCode) { log_exception(); return CSL_BADCODE; } // serialize code object #if PY_MINOR_VERSION == 3 pStr = PyMarshal_WriteObjectToString(pCode); #else pStr = PyMarshal_WriteObjectToString(pCode, 0); #endif Py_DECREF(pCode); if (!pStr) { return CSL_NOMEM; } size = PyString_Size(pStr); *codeptr = malloc(size); if (!*codeptr) { Py_DECREF(pStr); return CSL_NOMEM; } memcpy(*codeptr, PyString_AsString(pStr), size); *sizeptr = size; Py_DECREF(pStr); return 0; }
/* uwsgi MARSHAL|33 */ int uwsgi_request_marshal(struct uwsgi_server *uwsgi, struct wsgi_request *wsgi_req) { PyObject *func_result; PyObject *umm = PyDict_GetItemString(uwsgi->embedded_dict, "message_manager_marshal"); if (umm) { PyObject *ummo = PyMarshal_ReadObjectFromString(wsgi_req->buffer, wsgi_req->uh.pktsize); if (ummo) { if (!PyTuple_SetItem(uwsgi->embedded_args, 0, ummo)) { if (!PyTuple_SetItem(uwsgi->embedded_args, 1, PyInt_FromLong(wsgi_req->uh.modifier2))) { func_result = PyEval_CallObject(umm, uwsgi->embedded_args); if (PyErr_Occurred()) { PyErr_Print(); } if (func_result) { PyObject *marshalled = PyMarshal_WriteObjectToString(func_result, 1); if (!marshalled) { PyErr_Print(); } else { if (PyString_Size(marshalled) <= 0xFFFF) { wsgi_req->uh.pktsize = (uint16_t) PyString_Size(marshalled); if (write(wsgi_req->poll.fd, wsgi_req, 4) == 4) { if (write(wsgi_req->poll.fd, PyString_AsString(marshalled), wsgi_req->uh.pktsize) != wsgi_req->uh.pktsize) { uwsgi_error("write()"); } } else { uwsgi_error("write()"); } } else { fprintf(stderr, "marshalled object is too big. skip\n"); } Py_DECREF(marshalled); } Py_DECREF(func_result); } } } //Py_DECREF(ummo); } } PyErr_Clear(); return 0; }
//============================================================================= // METHOD : SPELLwsStorage::storeObject //============================================================================= void SPELLwsStorage::storeObject( PyObject* object ) { if (!m_file.is_open()) return; if (object == NULL) { THROW_EXCEPTION("Unable to store object", "Null reference given", SPELL_ERROR_WSTART); } if (m_mode == MODE_READ) { THROW_EXCEPTION("Unable to store object", "Initialized in read mode", SPELL_ERROR_WSTART); } std::string ptype = PYREPR(PyObject_Type(object)); DEBUG("Encoding object " + PYREPR(object) + " of type " + ptype); // Marshal the object to the persistent storage file PyObject* marshal = PyMarshal_WriteObjectToString( object, LATEST_MARSHAL_VERSION ); // Ensure there is no internal Python error SPELLpythonHelper::instance().checkError(); int marshal_len = PyObject_Length(marshal); DEBUG("Marshal length: " + ISTR(marshal_len)); char* encoded = PyString_AsString(marshal); // FORMAT IN FILE: // COUNT \1 PTYPE \1 MARSHAL LENGTH // MARSHAL // If the marshal was OK, continue and store in file: OP,TYPE,MARSHALLED m_opCounter++; m_file << m_opCounter << "\1" << ptype << "\1" << marshal_len << std::endl; m_file.write(encoded,marshal_len); m_file << std::endl; std::flush(m_file); m_trace << "[" << m_opCounter << "] STORE (" << m_filename << ") " + PYREPR(object) << " [ Type=" + ptype << ", Marhsal length=" + ISTR(marshal_len) + " ]" << std::endl; std::flush(m_trace); }
int main(int argc, char *argv[]) { char *inpath, *outpath; FILE *infile, *outfile = NULL; struct stat st; size_t text_size, data_size, n; char *text; unsigned char *data; PyObject *code, *marshalled; if (argc != 3) { fprintf(stderr, "need to specify input and output paths\n"); return 2; } inpath = argv[1]; outpath = argv[2]; infile = fopen(inpath, "rb"); if (infile == NULL) { fprintf(stderr, "cannot open '%s' for reading\n", inpath); return 1; } if (fstat(fileno(infile), &st)) { fclose(infile); fprintf(stderr, "cannot fstat '%s'\n", inpath); return 1; } text_size = st.st_size; text = (char *) malloc(text_size + 1); if (text == NULL) { fclose(infile); fprintf(stderr, "could not allocate %ld bytes\n", (long) text_size); return 1; } n = fread(text, 1, text_size, infile); fclose(infile); infile = NULL; if (n < text_size) { fprintf(stderr, "read too short: got %ld instead of %ld bytes\n", (long) n, (long) text_size); return 1; } text[text_size] = '\0'; Py_NoUserSiteDirectory++; Py_NoSiteFlag++; Py_IgnoreEnvironmentFlag++; Py_SetProgramName(L"./_freeze_importlib"); /* Don't install importlib, since it could execute outdated bytecode. */ _Py_InitializeEx_Private(1, 0); code = Py_CompileStringExFlags(text, "<frozen importlib._bootstrap>", Py_file_input, NULL, 0); if (code == NULL) goto error; marshalled = PyMarshal_WriteObjectToString(code, Py_MARSHAL_VERSION); Py_DECREF(code); if (marshalled == NULL) goto error; assert(PyBytes_CheckExact(marshalled)); data = (unsigned char *) PyBytes_AS_STRING(marshalled); data_size = PyBytes_GET_SIZE(marshalled); outfile = fopen(outpath, "wb"); if (outfile == NULL) { fprintf(stderr, "cannot open '%s' for writing\n", outpath); return 1; } fprintf(outfile, "%s\n", header); fprintf(outfile, "unsigned char _Py_M__importlib[] = {\n"); for (n = 0; n < data_size; n += 16) { size_t i, end = Py_MIN(n + 16, data_size); fprintf(outfile, " "); for (i = n; i < end; i++) { fprintf(outfile, "%d,", (unsigned int) data[i]); } fprintf(outfile, "\n"); } fprintf(outfile, "};\n"); Py_DECREF(marshalled); Py_Finalize(); if (infile) fclose(infile); if (outfile) { if (ferror(outfile)) { fprintf(stderr, "error when writing to '%s'\n", outpath); fclose(outfile); return 1; } fclose(outfile); } return 0; error: PyErr_Print(); Py_Finalize(); if (infile) fclose(infile); if (outfile) fclose(outfile); return 1; }
/* * This routine needs to emulate load_source_module in import.c in * python, but with DataSectionPtr using FILE* * However, we can't get an mTime (there's no interface for it in DataSection) * Need to resolve this before we can commit */ PyObject* PyResMgrImportLoader::load_source_module( const std::string& name, BinaryPtr py_data, DataSectionPtr pDirectory ) { std::string compiledExtension = "pyc"; if ( Py_OptimizeFlag ) compiledExtension = "pyo"; std::string moduleName = name; std::string::size_type dotPos = name.rfind( "." ); if ( dotPos != std::string::npos ) { moduleName = name.substr( dotPos + 1 ); } // Find source (.py) and object (.pyc/.pyo) files to // process for this source module or package. std::string modulePath; bool isPackage = false; std::string objectModuleName; if ( modules_[ name ].first == PKG_DIRECTORY ) { isPackage = true; modulePath = path_ + "/" + moduleName + "/__init__.py"; objectModuleName = "__init__." + compiledExtension; } else { modulePath = path_ + "/" + moduleName + ".py"; objectModuleName = moduleName + "." + compiledExtension; } // Remove this from the cache. We have it now, and will feed it to Python. // This ensures that a reload() call will work correctly. modules_.erase( name ); // Fetch mtime of py file time_t pyModTime = static_cast< time_t >( -1 ); IFileSystem::FileInfo fInfo; IFileSystem::FileType fType = BWResource::instance().fileSystem()->getFileType( modulePath, &fInfo ); if ( fType != IFileSystem::FT_NOT_FOUND ) { pyModTime = fInfo.modified; } // If possible, palm this off to load_compiled_module DataSectionPtr pycSection = pDirectory->openSection( objectModuleName ); if ( pycSection && check_compiled_module( name, pycSection->asBinary() )) { if ( check_compiled_module_mtime( name, pycSection->asBinary(), pyModTime ) ) { // We know the module was valid and up-to-date, so trust the loader // to either load it or fail noisily return load_compiled_module( name, pycSection->asBinary(), true ); } } if ( pycSection ) { // Get rid of our reference to the old compiled python file. // TODO: Purge this section from the DataSection cache, we're about // to replace it on disk pycSection = NULL; } // We got here, the object file for this source either doesn't exist, isn't // valid, or isn't as recent as the source. // Emulate parse_source_module // The code string needs to have (\n) as a line seperator, and needs an EOF // (-1) or null termination, and has to end in a newline. // Also, need to ensure there's no embedded nulls. // So have to make a copy of the string. // We shouldn't ever do this in release anyway. std::string codeString( py_data->cdata(), py_data->len() ); if ( codeString.find( '\0' ) != std::string::npos ) { PyErr_Format( PyExc_ImportError, "%s contains an embedded null character", modulePath.c_str() ); return NULL; } std::string::size_type winNLpos; // Convert any Windows newlines into UNIX newlines if ( ( winNLpos = codeString.find( "\r\n", 0 ) ) != std::string::npos ) { do { codeString.replace( winNLpos, 2, "\n" ); winNLpos = codeString.find( "\r\n", winNLpos ); } while ( winNLpos != std::string::npos ); } // Ensure we're newline-terminated codeString.append( "\n" ); PyObject* codeObject = Py_CompileString( codeString.c_str(), const_cast< char* >( modulePath.c_str() ), Py_file_input ); if ( codeObject == NULL ) // Compiler didn't like it. Propagate the error up return NULL; // OK, we have a module, now we just execute it into the correct space. // Always call it a .py, even though we've created a .pyc PyObject* module = PyImport_ExecCodeModuleEx( const_cast< char* >( name.c_str() ), codeObject, const_cast< char* >( modulePath.c_str() ) ); if ( module == NULL ) { Py_DECREF( codeObject ); return NULL; } // It executed OK, so write out an object file for later use, if possible // Emulates write_compiled_module( co, cpathname, mtime ) do { PyObject* codeString = PyMarshal_WriteObjectToString( codeObject, Py_MARSHAL_VERSION ); // XXX: Maybe we should care if _this_ fails, or at least report it? if ( codeString == NULL || !PyString_Check( codeString ) ) { PyErr_Clear(); break; } char* dataBlock = new char[ PyString_Size( codeString ) + 8 ]; // XXX: On-disk format is little-endian, we're not checking that here reinterpret_cast< uint32* >(dataBlock)[ 0 ] = PyImport_GetMagicNumber(); reinterpret_cast< int32* >(dataBlock)[ 1 ] = static_cast< int32 >( pyModTime ); memcpy( dataBlock + 8, PyString_AsString( codeString ), PyString_Size( codeString ) ); // The following is a little nasty, we end up copying the data a couple of times // Wrap dataBlock in a BinaryBlock (which takes a copy of it) BinaryPtr pycData = new BinaryBlock( dataBlock, PyString_Size( codeString ) + 8, "PyResMgrImportLoader::load_source_module" ); delete[] dataBlock; if ( !pycData ) break; // Save out our new pyc file pycSection = pDirectory->openSection( objectModuleName, true, BinSection::creator() ); if ( !pycSection ) break; pycSection->setBinary( pycData ); pycSection->save(); } while( false ); Py_DECREF( codeObject ); PyObject *moduleDict = PyModule_GetDict( module ); if ( moduleDict != NULL ) { int err = PyDict_SetItemString( moduleDict, "__loader__", this ); if ( err != 0 ) { Py_DECREF( module ); return NULL; } } // TRACE_MSG( "PyResMgrImportLoader(%s)::load_source_module: loaded %s\n", path_.c_str(), // name.c_str() ); return module; }