static void parse_error(PyObject *exception, const Py_UNICODE *buffer, const ptrdiff_t buffer_length, const Py_UNICODE *pos, const char *msg)
{
PyObject *buffer_str;
PyObject *pos_str;
buffer_str = PyUnicode_Encode(buffer, buffer_length, NULL, NULL);
pos_str = PyUnicode_Encode(pos, 1, NULL, NULL);
if(buffer_str && pos_str)
PyErr_Format(exception, "parse error in '%s' near '%s' at position %td: %s", PyString_AS_STRING(buffer_str), PyString_AS_STRING(pos_str), pos - buffer + 1, msg);
else
PyErr_Format(exception, "parse error (details not available): %s", msg);
Py_XDECREF(buffer_str);
Py_XDECREF(pos_str);
}
static int
tns_render_unicode(const tns_ops *ops, void *val, tns_outbuf *outbuf)
{
PyObject *bytes;
char* encoding = ((tns_ops_with_encoding*)ops)->encoding;
if(PyUnicode_Check(val)) {
bytes = PyUnicode_Encode(PyUnicode_AS_UNICODE(val),
PyUnicode_GET_SIZE(val),
encoding, NULL);
if(bytes == NULL) {
return -1;
}
if(tns_render_string(ops, bytes, outbuf) == -1) {
return -1;
}
Py_DECREF(bytes);
return 0;
}
if(PyString_Check(val)) {
return tns_render_string(ops, val, outbuf);
}
return -1;
}
static PyObject*
th_brk_(PyObject *self, PyObject *args)
{
PyObject *result = NULL;
Py_UNICODE *s1;
int s1_len;
if (!PyArg_ParseTuple(args, "u#", &s1, &s1_len)) {
PyErr_SetString(PyExc_TypeError, "parameter must be unicode");
return NULL;
}
if(s1_len == 0) {
PyErr_SetString(PyExc_ValueError, "parameter must not be empty string");
return NULL;
}
PyObject *txt_cp874 = PyUnicode_Encode(s1, s1_len, "CP874", NULL);
if(txt_cp874 == NULL) {
return NULL;
}
Py_ssize_t len = PyString_Size(txt_cp874);
char *c_txt_cp874 = PyString_AsString(txt_cp874);
int *pos = (int *)malloc(sizeof(int) * (s1_len + 1));
int n = th_brk((unsigned char *)c_txt_cp874, pos, len);
int i, s = 0;
char *buffer;
result = PyList_New(0);
for(i = 0; i < n; i++) {
PyObject *tok;
PyObject *tok_cp874 = PySequence_GetSlice(txt_cp874, s, pos[i]);
Py_ssize_t tok_len;
PyString_AsStringAndSize(tok_cp874, &buffer, &tok_len);
tok = PyUnicode_Decode(buffer, tok_len, "CP874", NULL);
s = pos[i];
PyList_Append(result, tok);
Py_XDECREF(tok_cp874);
Py_XDECREF(tok);
}
if(s < len) {
PyObject *tok_cp874 = PySequence_GetSlice(txt_cp874, s, len);
Py_ssize_t tok_len;
PyObject *tok;
PyString_AsStringAndSize(tok_cp874, &buffer, &tok_len);
tok = PyUnicode_Decode(buffer, tok_len, "CP874", NULL);
PyList_Append(result, tok);
Py_XDECREF(tok_cp874);
Py_XDECREF(tok);
}
Py_XDECREF(txt_cp874);
free(pos);
return result;
}
char *
PyUnicode_AsPgString(PyObject *p_unicode)
{
Py_ssize_t unicode_size = PyUnicode_GET_SIZE(p_unicode);
char *message = NULL;
PyObject *pTempStr = PyUnicode_Encode(PyUnicode_AsUnicode(p_unicode),
unicode_size,
GetDatabaseEncodingName(), NULL);
errorCheck();
message = strdup(PyBytes_AsString(pTempStr));
errorCheck();
Py_DECREF(pTempStr);
return message;
}
char *
PyUnicode_AsPgString(PyObject *p_unicode)
{
Py_ssize_t unicode_size;
char *message = NULL;
PyObject *pTempStr;
if (p_unicode == NULL)
{
elog(ERROR, "Received a null pointer in pyunicode_aspgstring");
}
unicode_size = PyUnicode_GET_SIZE(p_unicode);
pTempStr = PyUnicode_Encode(PyUnicode_AsUnicode(p_unicode),
unicode_size,
getPythonEncodingName(), NULL);
errorCheck();
message = strdup(PyBytes_AsString(pTempStr));
errorCheck();
Py_DECREF(pTempStr);
return message;
}
PyObject *PyUnicode_EncodeCP1250Helper(const Py_UNICODE *data, Py_ssize_t length, const char *errors)
{
return PyUnicode_Encode (data, length, "cp1250", errors);
}
static PyObject *next(PyObject *self)
{
ligolw_Tokenizer *tokenizer = (ligolw_Tokenizer *) self;
PyObject *type;
PyObject *token;
Py_UNICODE *start, *end;
/*
* Identify the start and end of the next token.
*/
do {
type = next_token(tokenizer, &start, &end);
if(!type)
return NULL;
} while(type == Py_None);
/*
* Extract token as desired type.
*/
if(start == NULL) {
/*
* unquoted zero-length string == None
*/
Py_INCREF(Py_None);
token = Py_None;
} else if(type == (PyObject *) &PyFloat_Type) {
char ascii_buffer[end - start + 1];
char *ascii_end;
if(PyUnicode_EncodeDecimal(start, end - start, ascii_buffer, NULL))
return NULL;
token = PyFloat_FromDouble(strtod(ascii_buffer, &ascii_end));
if(ascii_end == ascii_buffer || *ascii_end != 0) {
/*
* strtod() couldn't convert the token, emulate
* float()'s error message
*/
Py_XDECREF(token);
PyErr_Format(PyExc_ValueError, "invalid literal for float(): '%s'", ascii_buffer);
token = NULL;
}
} else if(type == (PyObject *) &PyUnicode_Type) {
token = PyUnicode_FromUnicode(start, end - start);
} else if(type == (PyObject *) &PyString_Type) {
token = PyUnicode_Encode(start, end - start, NULL, NULL);
} else if(type == (PyObject *) &PyInt_Type) {
token = PyInt_FromUnicode(start, end - start, 0);
} else if(type == (PyObject *) &PyLong_Type) {
token = PyLong_FromUnicode(start, end - start, 0);
} else {
token = PyObject_CallFunction(type, "u#", start, end - start);
}
/*
* Done.
*/
return token;
}