static bool decode_struct(DecodeBuffer* input, PyObject* output, PyObject* spec_seq) { int spec_seq_len = PyTuple_Size(spec_seq); if (spec_seq_len == -1) { return false; } while (true) { TType type; int16_t tag; PyObject* item_spec; PyObject* fieldval = NULL; StructItemSpec parsedspec; type = readByte(input); if (type == -1) { return false; } if (type == T_STOP) { break; } tag = readI16(input); if (INT_CONV_ERROR_OCCURRED(tag)) { return false; } if (tag >= 0 && tag < spec_seq_len) { item_spec = PyTuple_GET_ITEM(spec_seq, tag); } else { item_spec = Py_None; } if (item_spec == Py_None) { if (!skip(input, type)) { return false; } else { continue; } } if (!parse_struct_item_spec(&parsedspec, item_spec)) { return false; } if (parsedspec.type != type) { PyErr_SetString(PyExc_TypeError, "struct field had wrong type while reading"); return false; } fieldval = decode_val(input, parsedspec.type, parsedspec.typeargs); if (fieldval == NULL) { return false; } if (PyObject_SetAttr(output, parsedspec.attrname, fieldval) == -1) { Py_DECREF(fieldval); return false; } Py_DECREF(fieldval); } return true; }
static long as_long_or(PyObject* value, long default_value) { long v = PyInt_AsLong(value); if (INT_CONV_ERROR_OCCURRED(v)) { PyErr_Clear(); return default_value; } return v; }
static inline long as_long_then_delete(PyObject* value, long default_value) { ScopedPyObject scope(value); long v = PyInt_AsLong(value); if (INT_CONV_ERROR_OCCURRED(v)) { PyErr_Clear(); return default_value; } return v; }
static inline bool check_ssize_t_32(Py_ssize_t len) { // error from getting the int if (INT_CONV_ERROR_OCCURRED(len)) { return false; } if (!CHECK_RANGE(len, 0, INT32_MAX)) { PyErr_SetString(PyExc_OverflowError, "size out of range: exceeded INT32_MAX"); return false; } return true; }
static inline bool check_list_length(Py_ssize_t len) { // error from getting the int if (INT_CONV_ERROR_OCCURRED(len)) { return false; } if (!CHECK_RANGE(len, 0, MAX_LIST_SIZE)) { PyErr_SetString(PyExc_OverflowError, "list size out of the sanity limit (10000 items max)"); return false; } return true; }
static bool checkTypeByte(DecodeBuffer* input, TType expected) { TType got = readByte(input); if (INT_CONV_ERROR_OCCURRED(got)) { return false; } if (expected != got) { PyErr_SetString(PyExc_TypeError, "got wrong ttype while reading field"); return false; } return true; }
bool parse_map_args(MapTypeArgs* dest, PyObject* typeargs) { if (PyTuple_Size(typeargs) != 5) { PyErr_SetString(PyExc_TypeError, "expecting 5 arguments for typeargs to map"); return false; } dest->ktag = static_cast<TType>(PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0))); if (INT_CONV_ERROR_OCCURRED(dest->ktag)) { return false; } dest->vtag = static_cast<TType>(PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 2))); if (INT_CONV_ERROR_OCCURRED(dest->vtag)) { return false; } dest->ktypeargs = PyTuple_GET_ITEM(typeargs, 1); dest->vtypeargs = PyTuple_GET_ITEM(typeargs, 3); dest->immutable = Py_True == PyTuple_GET_ITEM(typeargs, 4); return true; }
bool parse_struct_item_spec(StructItemSpec* dest, PyObject* spec_tuple) { // i'd like to use ParseArgs here, but it seems to be a bottleneck. if (PyTuple_Size(spec_tuple) != 5) { PyErr_Format(PyExc_TypeError, "expecting 5 arguments for spec tuple but got %d", static_cast<int>(PyTuple_Size(spec_tuple))); return false; } dest->tag = static_cast<TType>(PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 0))); if (INT_CONV_ERROR_OCCURRED(dest->tag)) { return false; } dest->type = static_cast<TType>(PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 1))); if (INT_CONV_ERROR_OCCURRED(dest->type)) { return false; } dest->attrname = PyTuple_GET_ITEM(spec_tuple, 2); dest->typeargs = PyTuple_GET_ITEM(spec_tuple, 3); dest->defval = PyTuple_GET_ITEM(spec_tuple, 4); return true; }
static inline bool parse_pyint(PyObject* o, int32_t* ret, int32_t min, int32_t max) { long val = PyInt_AsLong(o); if (INT_CONV_ERROR_OCCURRED(val)) { return false; } if (!CHECK_RANGE(val, min, max)) { PyErr_SetString(PyExc_OverflowError, "int out of range"); return false; } *ret = (int32_t) val; return true; }
static bool parse_set_list_args(SetListTypeArgs* dest, PyObject* typeargs) { if (PyTuple_Size(typeargs) != 2) { PyErr_SetString(PyExc_TypeError, "expecting tuple of size 2 for list/set type args"); return false; } dest->element_type = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0)); if (INT_CONV_ERROR_OCCURRED(dest->element_type)) { return false; } dest->typeargs = PyTuple_GET_ITEM(typeargs, 1); return true; }
// Returns a new reference. static PyObject* decode_val(DecodeBuffer* input, TType type, PyObject* typeargs, long string_limit, long container_limit) { switch (type) { case T_BOOL: { int8_t v = readByte(input); if (INT_CONV_ERROR_OCCURRED(v)) { return NULL; } switch (v) { case 0: Py_RETURN_FALSE; case 1: Py_RETURN_TRUE; // Don't laugh. This is a potentially serious issue. default: PyErr_SetString(PyExc_TypeError, "boolean out of range"); return NULL; } break; } case T_I08: { int8_t v = readByte(input); if (INT_CONV_ERROR_OCCURRED(v)) { return NULL; } return PyInt_FromLong(v); } case T_I16: { int16_t v = readI16(input); if (INT_CONV_ERROR_OCCURRED(v)) { return NULL; } return PyInt_FromLong(v); } case T_I32: { int32_t v = readI32(input); if (INT_CONV_ERROR_OCCURRED(v)) { return NULL; } return PyInt_FromLong(v); } case T_I64: { int64_t v = readI64(input); if (INT_CONV_ERROR_OCCURRED(v)) { return NULL; } // TODO(dreiss): Find out if we can take this fastpath always when // sizeof(long) == sizeof(long long). if (CHECK_RANGE(v, LONG_MIN, LONG_MAX)) { return PyInt_FromLong((long) v); } return PyLong_FromLongLong(v); } case T_DOUBLE: { double v = readDouble(input); if (v == -1.0 && PyErr_Occurred()) { return false; } return PyFloat_FromDouble(v); } case T_STRING: { Py_ssize_t len = readI32(input); char* buf; if (!readBytes(input, &buf, len)) { return NULL; } if (!check_length_limit(len, string_limit)) { return NULL; } if (is_utf8(typeargs)) return PyUnicode_DecodeUTF8(buf, len, 0); else return PyString_FromStringAndSize(buf, len); } case T_LIST: case T_SET: { SetListTypeArgs parsedargs; int32_t len; PyObject* ret = NULL; int i; bool use_tuple = false; if (!parse_set_list_args(&parsedargs, typeargs)) { return NULL; } if (!checkTypeByte(input, parsedargs.element_type)) { return NULL; } len = readI32(input); if (!check_length_limit(len, container_limit)) { return NULL; } use_tuple = type == T_LIST && parsedargs.immutable; ret = use_tuple ? PyTuple_New(len) : PyList_New(len); if (!ret) { return NULL; } for (i = 0; i < len; i++) { PyObject* item = decode_val(input, parsedargs.element_type, parsedargs.typeargs, string_limit, container_limit); if (!item) { Py_DECREF(ret); return NULL; } if (use_tuple) { PyTuple_SET_ITEM(ret, i, item); } else { PyList_SET_ITEM(ret, i, item); } } // TODO(dreiss): Consider biting the bullet and making two separate cases // for list and set, avoiding this post facto conversion. if (type == T_SET) { PyObject* setret; setret = parsedargs.immutable ? PyFrozenSet_New(ret) : PySet_New(ret); Py_DECREF(ret); return setret; } return ret; } case T_MAP: { int32_t len; int i; MapTypeArgs parsedargs; PyObject* ret = NULL; if (!parse_map_args(&parsedargs, typeargs)) { return NULL; } if (!checkTypeByte(input, parsedargs.ktag)) { return NULL; } if (!checkTypeByte(input, parsedargs.vtag)) { return NULL; } len = readI32(input); if (!check_length_limit(len, container_limit)) { return NULL; } ret = PyDict_New(); if (!ret) { goto error; } for (i = 0; i < len; i++) { PyObject* k = NULL; PyObject* v = NULL; k = decode_val(input, parsedargs.ktag, parsedargs.ktypeargs, string_limit, container_limit); if (k == NULL) { goto loop_error; } v = decode_val(input, parsedargs.vtag, parsedargs.vtypeargs, string_limit, container_limit); if (v == NULL) { goto loop_error; } if (PyDict_SetItem(ret, k, v) == -1) { goto loop_error; } Py_DECREF(k); Py_DECREF(v); continue; // Yuck! Destructors, anyone? loop_error: Py_XDECREF(k); Py_XDECREF(v); goto error; } if (parsedargs.immutable) { PyObject* thrift = PyImport_ImportModule("thrift.Thrift"); PyObject* cls = NULL; PyObject* arg = NULL; if (!thrift) { goto error; } cls = PyObject_GetAttrString(thrift, "TFrozenDict"); if (!cls) { goto error; } arg = PyTuple_New(1); PyTuple_SET_ITEM(arg, 0, ret); return PyObject_CallObject(cls, arg); } return ret; error: Py_XDECREF(ret); return NULL; } case T_STRUCT: { StructTypeArgs parsedargs; if (!parse_struct_args(&parsedargs, typeargs)) { return NULL; } return decode_struct(input, Py_None, parsedargs.klass, parsedargs.spec, string_limit, container_limit); } case T_STOP: case T_VOID: case T_UTF16: case T_UTF8: case T_U64: default: PyErr_SetString(PyExc_TypeError, "Unexpected TType"); return NULL; } }
static PyObject* decode_struct(DecodeBuffer* input, PyObject* output, PyObject* klass, PyObject* spec_seq, long string_limit, long container_limit) { int spec_seq_len = PyTuple_Size(spec_seq); bool immutable = output == Py_None; PyObject* kwargs = NULL; if (spec_seq_len == -1) { return NULL; } if (immutable) { kwargs = PyDict_New(); if (!kwargs) { PyErr_SetString(PyExc_TypeError, "failed to prepare kwargument storage"); return NULL; } } while (true) { TType type; int16_t tag; PyObject* item_spec; PyObject* fieldval = NULL; StructItemSpec parsedspec; type = readByte(input); if (type == -1) { goto error; } if (type == T_STOP) { break; } tag = readI16(input); if (INT_CONV_ERROR_OCCURRED(tag)) { goto error; } if (tag >= 0 && tag < spec_seq_len) { item_spec = PyTuple_GET_ITEM(spec_seq, tag); } else { item_spec = Py_None; } if (item_spec == Py_None) { if (!skip(input, type)) { goto error; } else { continue; } } if (!parse_struct_item_spec(&parsedspec, item_spec)) { goto error; } if (parsedspec.type != type) { if (!skip(input, type)) { PyErr_Format(PyExc_TypeError, "struct field had wrong type: expected %d but got %d", parsedspec.type, type); goto error; } else { continue; } } fieldval = decode_val(input, parsedspec.type, parsedspec.typeargs, string_limit, container_limit); if (fieldval == NULL) { goto error; } if ((immutable && PyDict_SetItem(kwargs, parsedspec.attrname, fieldval) == -1) || (!immutable && PyObject_SetAttr(output, parsedspec.attrname, fieldval) == -1)) { Py_DECREF(fieldval); goto error; } Py_DECREF(fieldval); } if (immutable) { PyObject* args = PyTuple_New(0); PyObject* ret = NULL; if (!args) { PyErr_SetString(PyExc_TypeError, "failed to prepare argument storage"); goto error; } ret = PyObject_Call(klass, args, kwargs); Py_DECREF(kwargs); Py_DECREF(args); return ret; } Py_INCREF(output); return output; error: Py_XDECREF(kwargs); return NULL; }
static bool output_val(PyObject* output, PyObject* value, TType type, PyObject* typeargs) { /* * Refcounting Strategy: * * We assume that elements of the thrift_spec tuple are not going to be * mutated, so we don't ref count those at all. Other than that, we try to * keep a reference to all the user-created objects while we work with them. * output_val assumes that a reference is already held. The *caller* is * responsible for handling references */ switch (type) { case T_BOOL: { int v = PyObject_IsTrue(value); if (v == -1) { return false; } writeByte(output, (int8_t) v); break; } case T_I08: { int32_t val; if (!parse_pyint(value, &val, INT8_MIN, INT8_MAX)) { return false; } writeByte(output, (int8_t) val); break; } case T_I16: { int32_t val; if (!parse_pyint(value, &val, INT16_MIN, INT16_MAX)) { return false; } writeI16(output, (int16_t) val); break; } case T_I32: { int32_t val; if (!parse_pyint(value, &val, INT32_MIN, INT32_MAX)) { return false; } writeI32(output, val); break; } case T_I64: { int64_t nval = PyLong_AsLongLong(value); if (INT_CONV_ERROR_OCCURRED(nval)) { return false; } if (!CHECK_RANGE(nval, INT64_MIN, INT64_MAX)) { PyErr_SetString(PyExc_OverflowError, "int out of range"); return false; } writeI64(output, nval); break; } case T_DOUBLE: { double nval = PyFloat_AsDouble(value); if (nval == -1.0 && PyErr_Occurred()) { return false; } writeDouble(output, nval); break; } case T_STRING: { Py_ssize_t len = 0; if (is_utf8(typeargs) && PyUnicode_Check(value)) value = PyUnicode_AsUTF8String(value); len = PyString_Size(value); if (!check_ssize_t_32(len)) { return false; } writeI32(output, (int32_t) len); PycStringIO->cwrite(output, PyString_AsString(value), (int32_t) len); break; } case T_LIST: case T_SET: { Py_ssize_t len; SetListTypeArgs parsedargs; PyObject *item; PyObject *iterator; if (!parse_set_list_args(&parsedargs, typeargs)) { return false; } len = PyObject_Length(value); if (!check_ssize_t_32(len)) { return false; } writeByte(output, parsedargs.element_type); writeI32(output, (int32_t) len); iterator = PyObject_GetIter(value); if (iterator == NULL) { return false; } while ((item = PyIter_Next(iterator))) { if (!output_val(output, item, parsedargs.element_type, parsedargs.typeargs)) { Py_DECREF(item); Py_DECREF(iterator); return false; } Py_DECREF(item); } Py_DECREF(iterator); if (PyErr_Occurred()) { return false; } break; } case T_MAP: { PyObject *k, *v; Py_ssize_t pos = 0; Py_ssize_t len; MapTypeArgs parsedargs; len = PyDict_Size(value); if (!check_ssize_t_32(len)) { return false; } if (!parse_map_args(&parsedargs, typeargs)) { return false; } writeByte(output, parsedargs.ktag); writeByte(output, parsedargs.vtag); writeI32(output, len); // TODO(bmaurer): should support any mapping, not just dicts while (PyDict_Next(value, &pos, &k, &v)) { // TODO(dreiss): Think hard about whether these INCREFs actually // turn any unsafe scenarios into safe scenarios. Py_INCREF(k); Py_INCREF(v); if (!output_val(output, k, parsedargs.ktag, parsedargs.ktypeargs) || !output_val(output, v, parsedargs.vtag, parsedargs.vtypeargs)) { Py_DECREF(k); Py_DECREF(v); return false; } Py_DECREF(k); Py_DECREF(v); } break; } // TODO(dreiss): Consider breaking this out as a function // the way we did for decode_struct. case T_STRUCT: { StructTypeArgs parsedargs; Py_ssize_t nspec; Py_ssize_t i; if (!parse_struct_args(&parsedargs, typeargs)) { return false; } nspec = PyTuple_Size(parsedargs.spec); if (nspec == -1) { return false; } for (i = 0; i < nspec; i++) { StructItemSpec parsedspec; PyObject* spec_tuple; PyObject* instval = NULL; spec_tuple = PyTuple_GET_ITEM(parsedargs.spec, i); if (spec_tuple == Py_None) { continue; } if (!parse_struct_item_spec (&parsedspec, spec_tuple)) { return false; } instval = PyObject_GetAttr(value, parsedspec.attrname); if (!instval) { return false; } if (instval == Py_None) { Py_DECREF(instval); continue; } writeByte(output, (int8_t) parsedspec.type); writeI16(output, parsedspec.tag); if (!output_val(output, instval, parsedspec.type, parsedspec.typeargs)) { Py_DECREF(instval); return false; } Py_DECREF(instval); } writeByte(output, (int8_t)T_STOP); break; } case T_STOP: case T_VOID: case T_UTF16: case T_UTF8: case T_U64: default: PyErr_SetString(PyExc_TypeError, "Unexpected TType"); return false; } return true; }
// Returns a new reference. static PyObject* decode_val(DecodeBuffer* input, TType type, PyObject* typeargs) { switch (type) { case T_BOOL: { int8_t v = readByte(input); if (INT_CONV_ERROR_OCCURRED(v)) { return NULL; } switch (v) { case 0: Py_RETURN_FALSE; case 1: Py_RETURN_TRUE; // Don't laugh. This is a potentially serious issue. default: PyErr_SetString(PyExc_TypeError, "boolean out of range"); return NULL; } break; } case T_I08: { int8_t v = readByte(input); if (INT_CONV_ERROR_OCCURRED(v)) { return NULL; } return PyInt_FromLong(v); } case T_I16: { int16_t v = readI16(input); if (INT_CONV_ERROR_OCCURRED(v)) { return NULL; } return PyInt_FromLong(v); } case T_I32: { int32_t v = readI32(input); if (INT_CONV_ERROR_OCCURRED(v)) { return NULL; } return PyInt_FromLong(v); } case T_I64: { int64_t v = readI64(input); if (INT_CONV_ERROR_OCCURRED(v)) { return NULL; } // TODO(dreiss): Find out if we can take this fastpath always when // sizeof(long) == sizeof(long long). if (CHECK_RANGE(v, LONG_MIN, LONG_MAX)) { return PyInt_FromLong((long) v); } return PyLong_FromLongLong(v); } case T_DOUBLE: { double v = readDouble(input); if (v == -1.0 && PyErr_Occurred()) { return false; } return PyFloat_FromDouble(v); } case T_STRING: { Py_ssize_t len = readI32(input); char* buf; if (!readBytes(input, &buf, len)) { return NULL; } return PyString_FromStringAndSize(buf, len); } case T_LIST: case T_SET: { SetListTypeArgs parsedargs; int32_t len; PyObject* ret = NULL; int i; if (!parse_set_list_args(&parsedargs, typeargs)) { return NULL; } if (!checkTypeByte(input, parsedargs.element_type)) { return NULL; } len = readI32(input); if (!check_ssize_t_32(len)) { return NULL; } ret = PyList_New(len); if (!ret) { return NULL; } for (i = 0; i < len; i++) { PyObject* item = decode_val(input, parsedargs.element_type, parsedargs.typeargs); if (!item) { Py_DECREF(ret); return NULL; } PyList_SET_ITEM(ret, i, item); } // TODO(dreiss): Consider biting the bullet and making two separate cases // for list and set, avoiding this post facto conversion. if (type == T_SET) { PyObject* setret; #if (PY_VERSION_HEX < 0x02050000) // hack needed for older versions setret = PyObject_CallFunctionObjArgs((PyObject*)&PySet_Type, ret, NULL); #else // official version setret = PySet_New(ret); #endif Py_DECREF(ret); return setret; } return ret; } case T_MAP: { int32_t len; int i; MapTypeArgs parsedargs; PyObject* ret = NULL; if (!parse_map_args(&parsedargs, typeargs)) { return NULL; } if (!checkTypeByte(input, parsedargs.ktag)) { return NULL; } if (!checkTypeByte(input, parsedargs.vtag)) { return NULL; } len = readI32(input); if (!check_ssize_t_32(len)) { return false; } ret = PyDict_New(); if (!ret) { goto error; } for (i = 0; i < len; i++) { PyObject* k = NULL; PyObject* v = NULL; k = decode_val(input, parsedargs.ktag, parsedargs.ktypeargs); if (k == NULL) { goto loop_error; } v = decode_val(input, parsedargs.vtag, parsedargs.vtypeargs); if (v == NULL) { goto loop_error; } if (PyDict_SetItem(ret, k, v) == -1) { goto loop_error; } Py_DECREF(k); Py_DECREF(v); continue; // Yuck! Destructors, anyone? loop_error: Py_XDECREF(k); Py_XDECREF(v); goto error; } return ret; error: Py_XDECREF(ret); return NULL; } case T_STRUCT: { StructTypeArgs parsedargs; if (!parse_struct_args(&parsedargs, typeargs)) { return NULL; } PyObject* ret = PyObject_CallObject(parsedargs.klass, NULL); if (!ret) { return NULL; } if (!decode_struct(input, ret, parsedargs.spec)) { Py_DECREF(ret); return NULL; } return ret; } case T_STOP: case T_VOID: case T_UTF16: case T_UTF8: case T_U64: default: PyErr_SetString(PyExc_TypeError, "Unexpected TType"); return NULL; } }