/* * This function returns a NEW reference, i.e. caller must decref it in the end. */ PyObject* JySync_Init_PyInstance_From_JyInstance(jobject src, PyTypeObject* nonNativeSubtype) { env(NULL); (*env)->CallStaticVoidMethod(env, JyNIClass, JyNI_suspendPyInstanceFinalizer, src); PyObject* er = PyInstance_NewRaw( JyNI_PyObject_FromJythonPyObject((*env)->GetObjectField(env, src, pyInstance_instclassField)), JyNI_PyObject_FromJythonPyObject((*env)->GetObjectField(env, src, pyInstance___dict__Field))); return er; }
static PyObject * marshal_Load_internal(PyObject *py_stream, PyObject *py_callback, int skipcrc) { // Return value: New Reference char *stream; Py_ssize_t size; char *s; char *end; int type = -1; // current object type int shared = -1; // indicates whether current object is shared int i; char *error = "NO ERROR SPECIFIED"; char errortext[256]; Py_ssize_t length = 0; // generic length value. int shared_mapsize; int shared_count; // shared object index counter int *shared_map; // points to shared object mapping at end of stream PyObject **shared_obj = NULL; // holds the shared objects PyObject *obj = NULL; // currently decoded object PyObject *result = NULL; // final result int ct_ix = 0; struct Container ct_stack[MAX_DEPTH]; struct Container *container = &ct_stack[0]; if(PyString_AsStringAndSize(py_stream, &stream, &size) == -1) return NULL; s = stream; container->obj = NULL; container->type = 0; container->free = -1; container->index = 0; if(size < 6 || *s++ != PROTOCOL_ID) { int offset = 0; result = unpickle(py_stream, &offset); if(!result) goto cleanup; return result; } // how many shared objects in this stream? shared_mapsize = *(int32_t *)s; s += 4; // Security Check: assert there is enough data for that many items. if((5 + shared_mapsize*4) > size) { PyErr_Format(UnmarshalError, "Not enough room in stream for map. Wanted %d, but have only %d bytes remaining...", (shared_mapsize*4), ((int)size-5)); goto cleanup; } // ok, we got the map data right here... shared_map = (int32_t *)&stream[size - shared_mapsize * 4]; // Security Check #2: assert all map entries are between 1 and shared_mapsize for(i=0; i<shared_mapsize; i++) { if( (shared_map[i] > shared_mapsize) || (shared_map[i] < 1) ) { PyErr_SetString(UnmarshalError, "Bogus map data in marshal stream"); goto cleanup; } } // the start of which is incidentally also the end of the object data. end = (char *)shared_map; // create object table shared_obj = PyMem_MALLOC(shared_mapsize * sizeof(PyObject *)); if(!shared_obj) goto cleanup; // zero out object table for(shared_count = 0; shared_count < shared_mapsize; shared_count++) shared_obj[shared_count] = NULL; shared_count = 0; // start decoding. while(s < end) { // This outer loop is responsible for reading and decoding the next // object from the stream. The object is then handed to the inner loop, // which adds it to the current container, or returns it to the caller. // get type of next object to decode and check shared flag type = *s++; shared = type & SHARED_FLAG; type &= ~SHARED_FLAG; // if token uses a normal length value, read it now. if(needlength[type]) { READ_LENGTH; } else length = 0; #if MARSHAL_DEBUG // if(shared) { char text[220]; DEBUG_INDENT; sprintf(text, "pos:%4d type:%s(0x%02x) shared:%d len:%4d map:[", s-stream, tokenname[type], type, shared?1:0, length); printf(text); for(i=0; i<shared_mapsize; i++) printf("%d(%d),", shared_obj[i], shared_obj[i] ? ((PyObject *)(shared_obj[i]))->ob_refcnt : 0); printf("]\r\n"); } #endif // MARSHAL_DEBUG switch(type) { // // break statement: // attempts to add the newly decoded object (in the obj variable) to // the currently building container object. // // continue statement: // indicates the decoded object or type marker was handled/consumed // by the case and should _not_ be added to the currently building // container object or used in any other way; immediately decode a // new object // //--------------------------------------------------------------------- // SCALAR TYPES //--------------------------------------------------------------------- case TYPE_INT8: CHECK_SIZE(1); obj = PyInt_FromLong(*(int8_t *)s); s++; break; case TYPE_INT16: CHECK_SIZE(2); obj = PyInt_FromLong(*(int16_t *)s); s += 2; break; case TYPE_INT32: CHECK_SIZE(4); obj = PyInt_FromLong(*(int32_t *)s); s += 4; break; case TYPE_INT64: CHECK_SIZE(8); obj = PyLong_FromLongLong(*(int64_t *)s); s += 8; break; case TYPE_LONG: CHECK_SIZE(length); if(!length) obj = PyLong_FromLong(0); else { obj = _PyLong_FromByteArray((unsigned char *)s, length, 1, 1); Py_INCREF(obj); } CHECK_SHARED(obj); s += length; break; case TYPE_FLOAT: CHECK_SIZE(8); obj = PyFloat_FromDouble(*(double *)s); s += 8; break; case TYPE_CHECKSUM: CHECK_SIZE(4); if(!skipcrc && (*(uint32_t *)s != (uint32_t)adler32(1, s, (unsigned long)(end-s)))) { error = "checksum error"; goto fail; } s += 4; // because this type does not yield an object, go grab another // object right away! continue; //--------------------------------------------------------------------- // STRING TYPES //--------------------------------------------------------------------- case TYPE_STRINGR: if (length < 1 || length >= PyList_GET_SIZE(string_table)) { if(PyList_GET_SIZE(string_table)) PyErr_Format(UnmarshalError, "Invalid string table index %d", (int)length); else PyErr_SetString(PyExc_RuntimeError, "_stringtable not initialized"); goto cleanup; } obj = PyList_GET_ITEM(string_table, length); Py_INCREF(obj); break; case TYPE_STRING: // appears to be deprecated since machoVersion 213 CHECK_SIZE(1); length = *(unsigned char *)s++; CHECK_SIZE(length); obj = PyString_FromStringAndSize(s, length); s += length; break; case TYPE_STRING1: CHECK_SIZE(1); obj = PyString_FromStringAndSize(s, 1); s++; break; case TYPE_STREAM: // fallthrough, can be treated as string. case TYPE_STRINGL: // fallthrough, deprecated since machoVersion 213 case TYPE_BUFFER: // Type identifier re-used by CCP. treat as string. CHECK_SIZE(length); obj = PyString_FromStringAndSize(s, length); s += length; CHECK_SHARED(obj); break; case TYPE_UNICODE1: CHECK_SIZE(2); #ifdef Py_UNICODE_WIDE obj = _PyUnicodeUCS4_FromUCS2((void *)s, 1); #else obj = PyUnicode_FromWideChar((wchar_t *)s, 1); #endif s += 2; break; case TYPE_UNICODE: CHECK_SIZE(length*2); #ifdef Py_UNICODE_WIDE obj = _PyUnicodeUCS4_FromUCS2((void *)s, (int)length); #else obj = PyUnicode_FromWideChar((wchar_t *)s, length); #endif s += length*2; break; case TYPE_UTF8: CHECK_SIZE(length); obj = PyUnicode_DecodeUTF8(s, length, NULL); s += length; break; //--------------------------------------------------------------------- // SEQUENCE/MAPPING TYPES //--------------------------------------------------------------------- case TYPE_TUPLE1: NEW_SEQUENCE(TYPE_TUPLE, 1); continue; case TYPE_TUPLE2: NEW_SEQUENCE(TYPE_TUPLE, 2); continue; case TYPE_TUPLE: NEW_SEQUENCE(TYPE_TUPLE, (int)length); continue; case TYPE_LIST0: obj = PyList_New(0); CHECK_SHARED(obj); break; case TYPE_LIST1: NEW_SEQUENCE(TYPE_LIST, 1); continue; case TYPE_LIST: NEW_SEQUENCE(TYPE_LIST, (int)length); continue; case TYPE_DICT: if(length) { CHECK_SIZE(length*2); PUSH_CONTAINER(TYPE_DICT, (int)length*2); container->obj = PyDict_New(); container->obj2 = NULL; container->index = 0; CHECK_SHARED(container->obj); continue; } else { obj = PyDict_New(); CHECK_SHARED(obj); break; } //--------------------------------------------------------------------- // OBJECT TYPES //--------------------------------------------------------------------- case TYPE_REF: // length value is index in sharedobj array! if((length < 1 || length > shared_mapsize)) { error = "Shared reference index out of range"; goto fail; } if(!(obj = shared_obj[length-1])) { error = "Shared reference points to invalid object"; goto fail; } Py_INCREF(obj); //printf("Getting object %d from %d (refs:%d)\r\n", (int)obj, length-1, obj->ob_refcnt); break; case TYPE_GLOBAL: { PyObject *name; CHECK_SIZE(length); name = PyString_FromStringAndSize(s, length); if(!name) goto cleanup; s += length; if(!(obj = find_global(name))) { // exception should be set by find_global goto cleanup; } Py_DECREF(name); CHECK_SHARED(obj); break; } case TYPE_DBROW: case TYPE_INSTANCE: case TYPE_NEWOBJ: case TYPE_REDUCE: PUSH_CONTAINER(type, -1); container->obj = NULL; RESERVE_SLOT(container->index); continue; case TYPE_MARK: // this is a marker, not a real object. list/dict iterators check // for this type, but it can't be instantiated. break; default: if((obj = constants[type])) { Py_INCREF(obj); } else { error = "Unsupported type"; goto fail; } } // object decoding and construction done! if(!obj && type != TYPE_MARK) { // if obj is somehow NULL, whatever caused it is expected to have // set an exception at this point. goto cleanup; } #if MARSHAL_DEBUG /* if(obj && obj->ob_refcnt < 0) { char b[200]; sprintf(b, "type: %d, refcount: %d", type, obj->ob_refcnt); DEBUG(b); } */ if(obj) { DEBUG_INDENT; printf("`-- "); PyObject_Print(obj, stdout, 0); printf("\r\n"); fflush(stdout); } else { DEBUG_INDENT; printf("*** MARK\r\n"); } #endif // MARSHAL_DEBUG while(1) { // This inner loop does one of two things: // // - return the finished object to the caller if we're at the root // container. // // - add the object to the current container in a container- // specific manner. note that ownership of the reference is to be // given to the container object. #if MARSHAL_DEBUG { //char text[220]; DEBUG_INDENT; printf("container ix:%d (%08lx) type:%s[0x%02x] free:%d index:%d\r\n", ct_ix, container->obj, tokenname[container->type], container->type, container->free, container->index); } #endif // MARSHAL_DEBUG /* if(!container->obj) { error = "Root container popped off stack"; goto fail; } */ switch(container->type) { case TYPE_TUPLE: // tuples steal references. PyTuple_SET_ITEM(container->obj, container->index++, obj); break; case TYPE_LIST: // lists steal references. PyList_SET_ITEM(container->obj, container->index++, obj); break; case TYPE_DBROW: if(container->obj) { // we have an initialized DBRow. current object is a // non-scalar object for the row. append it. if(!dbrow_append_internal((PyDBRowObject *)container->obj, obj)) { // append call will have set an exception here. goto cleanup; } } else { // we now have a DBRowDescriptor, and the header data // should follow. Pull it and create the DBRow. READ_LENGTH; CHECK_SIZE(length); container->obj = PyDBRow_New((PyDBRowDescriptorObject *)obj, s, (int)length); container->free = 1+((PyDBRowDescriptorObject *)obj)->rd_num_objects; if(!container->obj) goto cleanup; Py_DECREF(obj); s += length; // add as shared object, if neccessary... UPDATE_SLOT(container->index, container->obj); } break; case TYPE_INSTANCE: { PyObject *cls; if(container->free == -1) { // create class instance if(!(cls = find_global(obj))) goto cleanup; container->obj = PyInstance_NewRaw(cls, 0); Py_DECREF(cls); if(!container->obj) goto cleanup; UPDATE_SLOT(container->index, container->obj); Py_DECREF(obj); break; } if(container->free == -2) { container->free = 1; // set state. if(!set_state(container->obj, obj)) goto cleanup; Py_DECREF(obj); break; } error = "invalid container state"; goto fail; } case TYPE_NEWOBJ: { PyObject *cls, *args, *__new__, *state; // instantiate the object... if(!(args = PyTuple_GetItem(obj, 0))) goto cleanup; if(!(cls = PyTuple_GetItem(args, 0))) goto cleanup; __new__ = PyObject_GetAttr(cls, py__new__); if(!__new__) goto cleanup; container->obj = PyObject_CallObject(__new__, args); Py_DECREF(__new__); if(!container->obj) goto cleanup; // add as shared object, if neccessary... UPDATE_SLOT(container->index, container->obj); // is there state data? if(PyTuple_GET_SIZE(obj) > 1) { state = PyTuple_GET_ITEM(obj, 1); if(!set_state(container->obj, state)) goto cleanup; } Py_DECREF(obj); // switch to list iterator LIST_ITERATOR; break; } case TYPE_REDUCE: { PyObject *callable, *args, *state; if(!(args = PyTuple_GetItem(obj, 1))) goto cleanup; if(!(callable = PyTuple_GET_ITEM(obj, 0))) goto cleanup; if(!(container->obj = PyObject_CallObject(callable, args))) goto cleanup; UPDATE_SLOT(container->index, container->obj); if(PyTuple_GET_SIZE(obj) > 2) { state = PyTuple_GET_ITEM(obj, 2); if(!set_state(container->obj, state)) goto cleanup; } Py_DECREF(obj); // switch to list iterator LIST_ITERATOR; break; } case TYPE_LIST_ITERATOR: if(type == TYPE_MARK) { // clear mark so nested iterator containers do not get terminated prematurely. type = -1; // decref the append method Py_XDECREF(container->obj2); container->obj2 = NULL; container->type = TYPE_DICT_ITERATOR; break; } if(!container->obj2) { // grab the append method from the container and keep // it around for speed. if(!(container->obj2 = PyObject_GetAttr(container->obj, pyappend))) goto cleanup; } if(!PyObject_CallFunctionObjArgs(container->obj2, obj, NULL)) goto cleanup; #if MARSHAL_DEBUG DEBUG_INDENT; printf("Appended %08lx to %08lx\r\n", obj, container->obj); #endif // MARSHAL_DEBUG Py_DECREF(obj); break; case TYPE_DICT_ITERATOR: if(type == TYPE_MARK) { // clear mark so nested iterator containers do not get terminated prematurely. type = -1; // we're done with dict iter. container is finished. container->free = 1; break; } POPULATE_DICT(container->obj2, obj); break; case TYPE_DICT: POPULATE_DICT(obj, container->obj2); break; case 0: // we're at the root. return the object to caller. result = obj; // avoid decreffing this object. obj = NULL; goto cleanup; } container->free--; if(container->free) // still room in this container. // break out of container handling to get next object for it. break; // this container is done, it is the next object to put into the // container under it! obj = container->obj; // switch context to said older container POP_CONTAINER; } // we've processed the object. clear it for next one. obj = NULL; } // if we get here, we're out of data, but it's a "clean" eof; we ran out // of data while expecting a new object... error = "Not enough objects in stream"; fail: PyErr_Format(UnmarshalError, "%s - type:0x%02x ctype:0x%02x len:%d share:%d pos:%d size:%d", error, type, container->type, (int)length, shared, (int)(s-stream), (int)(size)); cleanup: // on any error the current object we were working on will be unassociated // with anything, decref it. if decode was succesful or an object failed to // be created, it will be NULL anyway. Py_XDECREF(obj); // same story for containers... while(container->type) { Py_XDECREF(container->obj); // possibly unassociated object for dict entry? if(container->type == TYPE_DICT || container->type == TYPE_DICT_ITERATOR) { Py_XDECREF(container->obj2); } POP_CONTAINER; } if(shared_obj) { /* shared object list held a safety ref to all objects, decref em */ int i; for(i=0; i<shared_mapsize; i++) Py_XDECREF(shared_obj[i]); /* and free the list */ PyMem_FREE(shared_obj); } return result; }