int agwrite(Agraph_t * g, FILE * fp)
{
printdict_t *p;
char *t0, *t1;
/* write the graph header */
t0 = (AG_IS_STRICT(g)) ? "strict " : "";
t1 = (AG_IS_DIRECTED(g)) ? "digraph" : "graph";
if (strncmp(g->name, "_anonymous", 10))
fprintf(fp, "%s%s %s {\n", t0, t1, agcanonical(g->name));
else
fprintf(fp, "%s%s {\n", t0, t1);
/* write the top level attribute defs */
write_dict(g->univ->globattr, fp);
write_dict(g->univ->nodeattr, fp);
write_dict(g->univ->edgeattr, fp);
/* write the graph contents */
p = new_printdict_t(g);
write_subg(g, fp, (Agraph_t *) 0, 0, p);
fprintf(fp, "}\n");
free_printdict_t(p);
return ferror(fp);
}
void Tagger::Clear()
{
/* Write c_new arrays */
if (odictfile != NULL)
{
if (Option(verbose))
printf("Writing dictionary (%d entries)\n", dict.size);
write_dict(odictfile, &dict,
Option(training) || Option(reestimate));
fclose(odictfile);
}
if (otranfile != NULL)
{
if (Option(verbose))
printf("Writing transitions\n");
if (Option(training) || Option(reestimate))
write_ascii_trans(otranfile, &c_newtrans);
else
write_ascii_trans(otranfile, &trans);
fclose(otranfile);
}
if (Option(unknown_morph))
{
if (obadwordname[0] != 0)
fclose(features->obadwordfile);
}
free_dict( &dict );
if ( Option(bdbm))
/* Berkeley DB: close it */
dbp->close(dbp,(u_int32_t)0);
}
int agwrite(Agraph_t * g, FILE * fp)
{
printdict_t *p;
if (AG.fwrite == NULL) {
AG.fwrite = fwrite; /* init to system version of fwrite() */
}
if (AG.ferror == NULL) {
#ifdef ferror
#undef ferror
/* if ferror is a macro, then use our wrapper function, but
* undef the macro first so it doesn't subst in "AG.ferror" */
AG.ferror = agferror; /* init to ferror macro wrapper function */
#else
AG.ferror = ferror; /* init to system version of ferror() */
#endif
}
/* write the graph header */
agputs((AG_IS_STRICT(g)) ? "strict " : "", fp);
agputs((AG_IS_DIRECTED(g)) ? "digraph" : "graph", fp);
if (strncmp(g->name, "_anonymous", 10)) {
agputc(' ', fp);
agputs(agcanonical(g->name), fp);
}
agputs(" {\n", fp);
/* write the top level attribute defs */
write_dict(g->univ->globattr, fp);
write_dict(g->univ->nodeattr, fp);
write_dict(g->univ->edgeattr, fp);
/* write the graph contents */
p = new_printdict_t(g);
write_subg(g, fp, (Agraph_t *) 0, 0, p);
agputs("}\n", fp);
free_printdict_t(p);
return AG.ferror(fp);
}
int dict_main(int argc, char *argv[])
{
args_t *args = (args_t*) calloc(1,sizeof(args_t));
args->header = 1;
static const struct option loptions[] =
{
{"help", no_argument, NULL, 'h'},
{"no-header", no_argument, NULL, 'H'},
{"assembly", required_argument, NULL, 'a'},
{"species", required_argument, NULL, 's'},
{"uri", required_argument, NULL, 'u'},
{"output", required_argument, NULL, 'o'},
{NULL, 0, NULL, 0}
};
int c;
while ( (c=getopt_long(argc,argv,"?hHa:s:u:o:",loptions,NULL))>0 )
{
switch (c)
{
case 'a': args->assembly = optarg; break;
case 's': args->species = optarg; break;
case 'u': args->uri = optarg; break;
case 'o': args->output_fname = optarg; break;
case 'H': args->header = 0; break;
case 'h': return dict_usage();
default: return dict_usage();
}
}
char *fname = NULL;
if ( optind>=argc )
{
if ( !isatty(fileno((FILE *)stdin)) ) fname = "-"; // reading from stdin
else return dict_usage();
}
else fname = argv[optind];
write_dict(fname, args);
free(args);
return 0;
}
static PyObject* _cbson_query_message(PyObject* self, PyObject* args) {
/* NOTE just using a random number as the request_id */
struct module_state *state = GETSTATE(self);
int request_id = rand();
unsigned int options;
char* collection_name = NULL;
int collection_name_length;
int begin, cur_size, max_size = 0;
int num_to_skip;
int num_to_return;
PyObject* query;
PyObject* field_selector = Py_None;
unsigned char uuid_subtype = 3;
buffer_t buffer;
int length_location, message_length;
PyObject* result;
if (!PyArg_ParseTuple(args, "Iet#iiO|Ob",
&options,
"utf-8",
&collection_name,
&collection_name_length,
&num_to_skip, &num_to_return,
&query, &field_selector, &uuid_subtype)) {
return NULL;
}
buffer = buffer_new();
if (!buffer) {
PyErr_NoMemory();
PyMem_Free(collection_name);
return NULL;
}
// save space for message length
length_location = buffer_save_space(buffer, 4);
if (length_location == -1) {
PyMem_Free(collection_name);
PyErr_NoMemory();
return NULL;
}
if (!buffer_write_bytes(buffer, (const char*)&request_id, 4) ||
!buffer_write_bytes(buffer, "\x00\x00\x00\x00\xd4\x07\x00\x00", 8) ||
!buffer_write_bytes(buffer, (const char*)&options, 4) ||
!buffer_write_bytes(buffer, collection_name,
collection_name_length + 1) ||
!buffer_write_bytes(buffer, (const char*)&num_to_skip, 4) ||
!buffer_write_bytes(buffer, (const char*)&num_to_return, 4)) {
buffer_free(buffer);
PyMem_Free(collection_name);
return NULL;
}
begin = buffer_get_position(buffer);
if (!write_dict(state->_cbson, buffer, query, 0, uuid_subtype, 1)) {
buffer_free(buffer);
PyMem_Free(collection_name);
return NULL;
}
max_size = buffer_get_position(buffer) - begin;
if (field_selector != Py_None) {
begin = buffer_get_position(buffer);
if (!write_dict(state->_cbson, buffer, field_selector, 0, uuid_subtype, 1)) {
buffer_free(buffer);
PyMem_Free(collection_name);
return NULL;
}
cur_size = buffer_get_position(buffer) - begin;
max_size = (cur_size > max_size) ? cur_size : max_size;
}
PyMem_Free(collection_name);
message_length = buffer_get_position(buffer) - length_location;
memcpy(buffer_get_buffer(buffer) + length_location, &message_length, 4);
/* objectify buffer */
result = Py_BuildValue("i" BYTES_FORMAT_STRING "i", request_id,
buffer_get_buffer(buffer),
buffer_get_position(buffer),
max_size);
buffer_free(buffer);
return result;
}
static PyObject* _cbson_update_message(PyObject* self, PyObject* args) {
/* NOTE just using a random number as the request_id */
struct module_state *state = GETSTATE(self);
int request_id = rand();
char* collection_name = NULL;
int collection_name_length;
int before, cur_size, max_size = 0;
PyObject* doc;
PyObject* spec;
unsigned char multi;
unsigned char upsert;
unsigned char safe;
unsigned char check_keys;
unsigned char uuid_subtype;
PyObject* last_error_args;
int options;
buffer_t buffer;
int length_location, message_length;
PyObject* result;
if (!PyArg_ParseTuple(args, "et#bbOObObb",
"utf-8",
&collection_name,
&collection_name_length,
&upsert, &multi, &spec, &doc, &safe,
&last_error_args, &check_keys, &uuid_subtype)) {
return NULL;
}
options = 0;
if (upsert) {
options += 1;
}
if (multi) {
options += 2;
}
buffer = buffer_new();
if (!buffer) {
PyErr_NoMemory();
PyMem_Free(collection_name);
return NULL;
}
// save space for message length
length_location = buffer_save_space(buffer, 4);
if (length_location == -1) {
PyMem_Free(collection_name);
PyErr_NoMemory();
return NULL;
}
if (!buffer_write_bytes(buffer, (const char*)&request_id, 4) ||
!buffer_write_bytes(buffer,
"\x00\x00\x00\x00"
"\xd1\x07\x00\x00"
"\x00\x00\x00\x00",
12) ||
!buffer_write_bytes(buffer,
collection_name,
collection_name_length + 1) ||
!buffer_write_bytes(buffer, (const char*)&options, 4)) {
buffer_free(buffer);
PyMem_Free(collection_name);
return NULL;
}
before = buffer_get_position(buffer);
if (!write_dict(state->_cbson, buffer, spec, 0, uuid_subtype, 1)) {
buffer_free(buffer);
PyMem_Free(collection_name);
return NULL;
}
max_size = buffer_get_position(buffer) - before;
before = buffer_get_position(buffer);
if (!write_dict(state->_cbson, buffer, doc, check_keys, uuid_subtype, 1)) {
buffer_free(buffer);
PyMem_Free(collection_name);
return NULL;
}
cur_size = buffer_get_position(buffer) - before;
max_size = (cur_size > max_size) ? cur_size : max_size;
message_length = buffer_get_position(buffer) - length_location;
memcpy(buffer_get_buffer(buffer) + length_location, &message_length, 4);
if (safe) {
if (!add_last_error(self, buffer, request_id, collection_name,
collection_name_length, last_error_args)) {
buffer_free(buffer);
PyMem_Free(collection_name);
return NULL;
}
}
PyMem_Free(collection_name);
/* objectify buffer */
result = Py_BuildValue("i" BYTES_FORMAT_STRING "i", request_id,
buffer_get_buffer(buffer),
buffer_get_position(buffer),
max_size);
buffer_free(buffer);
return result;
}
static PyObject* _cbson_insert_message(PyObject* self, PyObject* args) {
/* NOTE just using a random number as the request_id */
struct module_state *state = GETSTATE(self);
int request_id = rand();
char* collection_name = NULL;
int collection_name_length;
PyObject* docs;
PyObject* doc;
PyObject* iterator;
int before, cur_size, max_size = 0;
int options = 0;
unsigned char check_keys;
unsigned char safe;
unsigned char continue_on_error;
unsigned char uuid_subtype;
PyObject* last_error_args;
buffer_t buffer;
int length_location, message_length;
PyObject* result;
if (!PyArg_ParseTuple(args, "et#ObbObb",
"utf-8",
&collection_name,
&collection_name_length,
&docs, &check_keys, &safe,
&last_error_args,
&continue_on_error, &uuid_subtype)) {
return NULL;
}
if (continue_on_error) {
options += 1;
}
buffer = buffer_new();
if (!buffer) {
PyErr_NoMemory();
PyMem_Free(collection_name);
return NULL;
}
// save space for message length
length_location = buffer_save_space(buffer, 4);
if (length_location == -1) {
PyMem_Free(collection_name);
PyErr_NoMemory();
return NULL;
}
if (!buffer_write_bytes(buffer, (const char*)&request_id, 4) ||
!buffer_write_bytes(buffer,
"\x00\x00\x00\x00"
"\xd2\x07\x00\x00",
8) ||
!buffer_write_bytes(buffer, (const char*)&options, 4) ||
!buffer_write_bytes(buffer,
collection_name,
collection_name_length + 1)) {
PyMem_Free(collection_name);
buffer_free(buffer);
return NULL;
}
iterator = PyObject_GetIter(docs);
if (iterator == NULL) {
PyObject* InvalidOperation = _error("InvalidOperation");
if (InvalidOperation) {
PyErr_SetString(InvalidOperation, "input is not iterable");
Py_DECREF(InvalidOperation);
}
buffer_free(buffer);
PyMem_Free(collection_name);
return NULL;
}
while ((doc = PyIter_Next(iterator)) != NULL) {
before = buffer_get_position(buffer);
if (!write_dict(state->_cbson, buffer, doc, check_keys, uuid_subtype, 1)) {
Py_DECREF(doc);
Py_DECREF(iterator);
buffer_free(buffer);
PyMem_Free(collection_name);
return NULL;
}
Py_DECREF(doc);
cur_size = buffer_get_position(buffer) - before;
max_size = (cur_size > max_size) ? cur_size : max_size;
}
Py_DECREF(iterator);
if (PyErr_Occurred()) {
buffer_free(buffer);
PyMem_Free(collection_name);
return NULL;
}
if (!max_size) {
PyObject* InvalidOperation = _error("InvalidOperation");
if (InvalidOperation) {
PyErr_SetString(InvalidOperation, "cannot do an empty bulk insert");
Py_DECREF(InvalidOperation);
}
buffer_free(buffer);
PyMem_Free(collection_name);
return NULL;
}
message_length = buffer_get_position(buffer) - length_location;
memcpy(buffer_get_buffer(buffer) + length_location, &message_length, 4);
if (safe) {
if (!add_last_error(self, buffer, request_id, collection_name,
collection_name_length, last_error_args)) {
buffer_free(buffer);
PyMem_Free(collection_name);
return NULL;
}
}
PyMem_Free(collection_name);
/* objectify buffer */
result = Py_BuildValue("i" BYTES_FORMAT_STRING "i", request_id,
buffer_get_buffer(buffer),
buffer_get_position(buffer),
max_size);
buffer_free(buffer);
return result;
}
void plist_to_bin(plist_t plist, char **plist_bin, uint32_t * length)
{
GPtrArray *objects = NULL;
GHashTable *ref_table = NULL;
struct serialize_s ser_s;
uint8_t offset_size = 0;
uint8_t dict_param_size = 0;
uint64_t num_objects = 0;
uint64_t root_object = 0;
uint64_t offset_table_index = 0;
GByteArray *bplist_buff = NULL;
uint64_t i = 0;
uint8_t *buff = NULL;
uint64_t *offsets = NULL;
uint8_t pad[6] = { 0, 0, 0, 0, 0, 0 };
uint8_t trailer[BPLIST_TRL_SIZE];
//for string
glong len = 0;
int type = 0;
glong items_read = 0;
glong items_written = 0;
GError *error = NULL;
gunichar2 *unicodestr = NULL;
//check for valid input
if (!plist || !plist_bin || *plist_bin || !length)
return;
//list of objects
objects = g_ptr_array_new();
//hashtable to write only once same nodes
ref_table = g_hash_table_new(plist_data_hash, plist_data_compare);
//serialize plist
ser_s.objects = objects;
ser_s.ref_table = ref_table;
serialize_plist(plist, &ser_s);
//now stream to output buffer
offset_size = 0; //unknown yet
dict_param_size = get_needed_bytes(objects->len);
num_objects = objects->len;
root_object = 0; //root is first in list
offset_table_index = 0; //unknown yet
//setup a dynamic bytes array to store bplist in
bplist_buff = g_byte_array_new();
//set magic number and version
g_byte_array_append(bplist_buff, BPLIST_MAGIC, BPLIST_MAGIC_SIZE);
g_byte_array_append(bplist_buff, BPLIST_VERSION, BPLIST_VERSION_SIZE);
//write objects and table
offsets = (uint64_t *) malloc(num_objects * sizeof(uint64_t));
for (i = 0; i < num_objects; i++)
{
plist_data_t data = plist_get_data(g_ptr_array_index(objects, i));
offsets[i] = bplist_buff->len;
switch (data->type)
{
case PLIST_BOOLEAN:
buff = (uint8_t *) malloc(sizeof(uint8_t));
buff[0] = data->boolval ? BPLIST_TRUE : BPLIST_FALSE;
g_byte_array_append(bplist_buff, buff, sizeof(uint8_t));
free(buff);
break;
case PLIST_UINT:
write_int(bplist_buff, data->intval);
break;
case PLIST_REAL:
write_real(bplist_buff, data->realval);
break;
case PLIST_KEY:
case PLIST_STRING:
len = strlen(data->strval);
if ( is_ascii_string(data->strval, len) )
{
write_string(bplist_buff, data->strval);
}
else
{
unicodestr = g_utf8_to_utf16(data->strval, len, &items_read, &items_written, &error);
write_unicode(bplist_buff, unicodestr, items_written);
g_free(unicodestr);
}
break;
case PLIST_DATA:
write_data(bplist_buff, data->buff, data->length);
case PLIST_ARRAY:
write_array(bplist_buff, g_ptr_array_index(objects, i), ref_table, dict_param_size);
break;
case PLIST_DICT:
write_dict(bplist_buff, g_ptr_array_index(objects, i), ref_table, dict_param_size);
break;
case PLIST_DATE:
write_date(bplist_buff, data->timeval.tv_sec + (double) data->timeval.tv_usec / G_USEC_PER_SEC);
break;
default:
break;
}
}
//free intermediate objects
g_hash_table_foreach_remove(ref_table, free_index, NULL);
g_ptr_array_free(objects, TRUE);
g_hash_table_destroy(ref_table);
//write offsets
offset_size = get_needed_bytes(bplist_buff->len);
offset_table_index = bplist_buff->len;
for (i = 0; i < num_objects; i++)
{
uint8_t *offsetbuff = (uint8_t *) malloc(offset_size);
#if G_BYTE_ORDER == G_BIG_ENDIAN
offsets[i] = offsets[i] << ((sizeof(uint64_t) - offset_size) * 8);
#endif
memcpy(offsetbuff, &offsets[i], offset_size);
byte_convert(offsetbuff, offset_size);
g_byte_array_append(bplist_buff, offsetbuff, offset_size);
free(offsetbuff);
}
//experimental pad to reflect apple's files
g_byte_array_append(bplist_buff, pad, 6);
//setup trailer
num_objects = GUINT64_FROM_BE(num_objects);
root_object = GUINT64_FROM_BE(root_object);
offset_table_index = GUINT64_FROM_BE(offset_table_index);
memcpy(trailer + BPLIST_TRL_OFFSIZE_IDX, &offset_size, sizeof(uint8_t));
memcpy(trailer + BPLIST_TRL_PARMSIZE_IDX, &dict_param_size, sizeof(uint8_t));
memcpy(trailer + BPLIST_TRL_NUMOBJ_IDX, &num_objects, sizeof(uint64_t));
memcpy(trailer + BPLIST_TRL_ROOTOBJ_IDX, &root_object, sizeof(uint64_t));
memcpy(trailer + BPLIST_TRL_OFFTAB_IDX, &offset_table_index, sizeof(uint64_t));
g_byte_array_append(bplist_buff, trailer, BPLIST_TRL_SIZE);
//duplicate buffer
*plist_bin = (char *) malloc(bplist_buff->len);
memcpy(*plist_bin, bplist_buff->data, bplist_buff->len);
*length = bplist_buff->len;
g_byte_array_free(bplist_buff, TRUE);
free(offsets);
}
static void
write_core(pic_state *pic, pic_value obj, pic_value port, struct writer_control *p)
{
pic_value labels = p->labels;
int i;
/* shared objects */
if (is_shared_object(pic, obj, p)) {
if (pic_weak_has(pic, labels, obj)) {
pic_fprintf(pic, port, "#%d#", pic_int(pic, pic_weak_ref(pic, labels, obj)));
return;
}
i = p->cnt++;
pic_fprintf(pic, port, "#%d=", i);
pic_weak_set(pic, labels, obj, pic_int_value(pic, i));
}
switch (pic_type(pic, obj)) {
case PIC_TYPE_UNDEF:
pic_fprintf(pic, port, "#undefined");
break;
case PIC_TYPE_NIL:
pic_fprintf(pic, port, "()");
break;
case PIC_TYPE_TRUE:
pic_fprintf(pic, port, "#t");
break;
case PIC_TYPE_FALSE:
pic_fprintf(pic, port, "#f");
break;
case PIC_TYPE_ID:
pic_fprintf(pic, port, "#<identifier %s>", pic_str(pic, pic_id_name(pic, obj)));
break;
case PIC_TYPE_EOF:
pic_fprintf(pic, port, "#.(eof-object)");
break;
case PIC_TYPE_INT:
pic_fprintf(pic, port, "%d", pic_int(pic, obj));
break;
case PIC_TYPE_SYMBOL:
pic_fprintf(pic, port, "%s", pic_sym(pic, obj));
break;
case PIC_TYPE_FLOAT:
write_float(pic, obj, port);
break;
case PIC_TYPE_BLOB:
write_blob(pic, obj, port);
break;
case PIC_TYPE_CHAR:
write_char(pic, obj, port, p);
break;
case PIC_TYPE_STRING:
write_str(pic, obj, port, p);
break;
case PIC_TYPE_PAIR:
write_pair(pic, obj, port, p);
break;
case PIC_TYPE_VECTOR:
write_vec(pic, obj, port, p);
break;
case PIC_TYPE_DICT:
write_dict(pic, obj, port, p);
break;
default:
pic_fprintf(pic, port, "#<%s %p>", pic_typename(pic, pic_type(pic, obj)), pic_obj_ptr(obj));
break;
}
if (p->op == OP_WRITE) {
if (is_shared_object(pic, obj, p)) {
pic_weak_del(pic, labels, obj);
}
}
}
static PyObject*
_cbson_do_batched_write_command(PyObject* self, PyObject* args) {
struct module_state *state = GETSTATE(self);
long max_bson_size;
long max_cmd_size;
long max_write_batch_size;
long idx_offset = 0;
int idx = 0;
int cmd_len_loc;
int lst_len_loc;
int ns_len;
int ordered;
char *ns = NULL;
PyObject* max_bson_size_obj;
PyObject* max_write_batch_size_obj;
PyObject* command;
PyObject* doc;
PyObject* docs;
PyObject* ctx;
PyObject* iterator;
PyObject* result;
PyObject* results;
PyObject* to_publish = NULL;
unsigned char op;
unsigned char check_keys;
codec_options_t options;
unsigned char empty = 1;
unsigned char errors = 0;
buffer_t buffer;
if (!PyArg_ParseTuple(args, "et#bOObO&O", "utf-8",
&ns, &ns_len, &op, &command, &docs, &check_keys,
convert_codec_options, &options,
&ctx)) {
return NULL;
}
max_bson_size_obj = PyObject_GetAttrString(ctx, "max_bson_size");
#if PY_MAJOR_VERSION >= 3
max_bson_size = PyLong_AsLong(max_bson_size_obj);
#else
max_bson_size = PyInt_AsLong(max_bson_size_obj);
#endif
Py_XDECREF(max_bson_size_obj);
if (max_bson_size == -1) {
destroy_codec_options(&options);
PyMem_Free(ns);
return NULL;
}
/*
* Max BSON object size + 16k - 2 bytes for ending NUL bytes
* XXX: This should come from the server - SERVER-10643
*/
max_cmd_size = max_bson_size + 16382;
max_write_batch_size_obj = PyObject_GetAttrString(ctx, "max_write_batch_size");
#if PY_MAJOR_VERSION >= 3
max_write_batch_size = PyLong_AsLong(max_write_batch_size_obj);
#else
max_write_batch_size = PyInt_AsLong(max_write_batch_size_obj);
#endif
Py_XDECREF(max_write_batch_size_obj);
if (max_write_batch_size == -1) {
destroy_codec_options(&options);
PyMem_Free(ns);
return NULL;
}
/* Default to True */
ordered = !((PyDict_GetItemString(command, "ordered")) == Py_False);
if (!(results = PyList_New(0))) {
destroy_codec_options(&options);
PyMem_Free(ns);
return NULL;
}
if (!(to_publish = PyList_New(0))) {
destroy_codec_options(&options);
PyMem_Free(ns);
Py_DECREF(results);
return NULL;
}
if (!(buffer = _command_buffer_new(ns, ns_len))) {
destroy_codec_options(&options);
PyMem_Free(ns);
Py_DECREF(results);
Py_DECREF(to_publish);
return NULL;
}
PyMem_Free(ns);
/* Position of command document length */
cmd_len_loc = buffer_get_position(buffer);
if (!write_dict(state->_cbson, buffer, command, 0,
&options, 0)) {
goto cmdfail;
}
/* Write type byte for array */
*(buffer_get_buffer(buffer) + (buffer_get_position(buffer) - 1)) = 0x4;
switch (op) {
case _INSERT:
{
if (!buffer_write_bytes(buffer, "documents\x00", 10))
goto cmdfail;
break;
}
case _UPDATE:
{
/* MongoDB does key validation for update. */
check_keys = 0;
if (!buffer_write_bytes(buffer, "updates\x00", 8))
goto cmdfail;
break;
}
case _DELETE:
{
/* Never check keys in a delete command. */
check_keys = 0;
if (!buffer_write_bytes(buffer, "deletes\x00", 8))
goto cmdfail;
break;
}
default:
{
PyObject* InvalidOperation = _error("InvalidOperation");
if (InvalidOperation) {
PyErr_SetString(InvalidOperation, "Unknown command");
Py_DECREF(InvalidOperation);
}
goto cmdfail;
}
}
/* Save space for list document */
lst_len_loc = buffer_save_space(buffer, 4);
if (lst_len_loc == -1) {
PyErr_NoMemory();
goto cmdfail;
}
iterator = PyObject_GetIter(docs);
if (iterator == NULL) {
PyObject* InvalidOperation = _error("InvalidOperation");
if (InvalidOperation) {
PyErr_SetString(InvalidOperation, "input is not iterable");
Py_DECREF(InvalidOperation);
}
goto cmdfail;
}
while ((doc = PyIter_Next(iterator)) != NULL) {
int sub_doc_begin = buffer_get_position(buffer);
int cur_doc_begin;
int cur_size;
int enough_data = 0;
int enough_documents = 0;
char key[16];
empty = 0;
INT2STRING(key, idx);
if (!buffer_write_bytes(buffer, "\x03", 1) ||
!buffer_write_bytes(buffer, key, (int)strlen(key) + 1)) {
goto cmditerfail;
}
cur_doc_begin = buffer_get_position(buffer);
if (!write_dict(state->_cbson, buffer, doc,
check_keys, &options, 1)) {
goto cmditerfail;
}
/* We have enough data, maybe send this batch. */
enough_data = (buffer_get_position(buffer) > max_cmd_size);
enough_documents = (idx >= max_write_batch_size);
if (enough_data || enough_documents) {
buffer_t new_buffer;
cur_size = buffer_get_position(buffer) - cur_doc_begin;
/* This single document is too large for the command. */
if (!idx) {
if (op == _INSERT) {
_set_document_too_large(cur_size, max_bson_size);
} else {
PyObject* DocumentTooLarge = _error("DocumentTooLarge");
if (DocumentTooLarge) {
/*
* There's nothing intelligent we can say
* about size for update and remove.
*/
PyErr_SetString(DocumentTooLarge,
"command document too large");
Py_DECREF(DocumentTooLarge);
}
}
goto cmditerfail;
}
if (!(new_buffer = buffer_new())) {
PyErr_NoMemory();
goto cmditerfail;
}
/* New buffer including the current overflow document */
if (!buffer_write_bytes(new_buffer,
(const char*)buffer_get_buffer(buffer), lst_len_loc + 5) ||
!buffer_write_bytes(new_buffer, "0\x00", 2) ||
!buffer_write_bytes(new_buffer,
(const char*)buffer_get_buffer(buffer) + cur_doc_begin, cur_size)) {
buffer_free(new_buffer);
goto cmditerfail;
}
/*
* Roll the existing buffer back to the beginning
* of the last document encoded.
*/
buffer_update_position(buffer, sub_doc_begin);
if (!buffer_write_bytes(buffer, "\x00\x00", 2)) {
buffer_free(new_buffer);
goto cmditerfail;
}
result = _send_write_command(ctx, buffer, lst_len_loc,
cmd_len_loc, &errors, to_publish);
buffer_free(buffer);
buffer = new_buffer;
if (!result)
goto cmditerfail;
#if PY_MAJOR_VERSION >= 3
result = Py_BuildValue("NN",
PyLong_FromLong(idx_offset), result);
#else
result = Py_BuildValue("NN",
PyInt_FromLong(idx_offset), result);
#endif
if (!result)
goto cmditerfail;
if (PyList_Append(results, result) < 0) {
Py_DECREF(result);
goto cmditerfail;
}
Py_DECREF(result);
if (errors && ordered) {
destroy_codec_options(&options);
Py_DECREF(iterator);
buffer_free(buffer);
return results;
}
idx_offset += idx;
idx = 0;
Py_DECREF(to_publish);
if (!(to_publish = PyList_New(0))) {
goto cmditerfail;
}
}
if (PyList_Append(to_publish, doc) < 0) {
goto cmditerfail;
}
Py_CLEAR(doc);
idx += 1;
}
Py_DECREF(iterator);
if (PyErr_Occurred()) {
goto cmdfail;
}
if (empty) {
PyObject* InvalidOperation = _error("InvalidOperation");
if (InvalidOperation) {
PyErr_SetString(InvalidOperation, "cannot do an empty bulk write");
Py_DECREF(InvalidOperation);
}
goto cmdfail;
}
if (!buffer_write_bytes(buffer, "\x00\x00", 2))
goto cmdfail;
result = _send_write_command(ctx, buffer, lst_len_loc,
cmd_len_loc, &errors, to_publish);
if (!result)
goto cmdfail;
#if PY_MAJOR_VERSION >= 3
result = Py_BuildValue("NN", PyLong_FromLong(idx_offset), result);
#else
result = Py_BuildValue("NN", PyInt_FromLong(idx_offset), result);
#endif
if (!result)
goto cmdfail;
buffer_free(buffer);
if (PyList_Append(results, result) < 0) {
Py_DECREF(result);
goto cmdfail;
}
Py_DECREF(result);
Py_DECREF(to_publish);
destroy_codec_options(&options);
return results;
cmditerfail:
Py_XDECREF(doc);
Py_DECREF(iterator);
cmdfail:
destroy_codec_options(&options);
Py_DECREF(results);
Py_XDECREF(to_publish);
buffer_free(buffer);
return NULL;
}
static PyObject* _cbson_do_batched_insert(PyObject* self, PyObject* args) {
struct module_state *state = GETSTATE(self);
/* NOTE just using a random number as the request_id */
int request_id = rand();
int send_safe, flags = 0;
int length_location, message_length;
int collection_name_length;
char* collection_name = NULL;
PyObject* docs;
PyObject* doc;
PyObject* iterator;
PyObject* ctx;
PyObject* last_error_args;
PyObject* result;
PyObject* max_bson_size_obj;
PyObject* max_message_size_obj;
PyObject* to_publish = NULL;
unsigned char check_keys;
unsigned char safe;
unsigned char continue_on_error;
codec_options_t options;
unsigned char empty = 1;
long max_bson_size;
long max_message_size;
buffer_t buffer;
PyObject *exc_type = NULL, *exc_value = NULL, *exc_trace = NULL;
if (!PyArg_ParseTuple(args, "et#ObbObO&O",
"utf-8",
&collection_name,
&collection_name_length,
&docs, &check_keys, &safe,
&last_error_args,
&continue_on_error,
convert_codec_options, &options,
&ctx)) {
return NULL;
}
if (continue_on_error) {
flags += 1;
}
/*
* If we are doing unacknowledged writes *and* continue_on_error
* is True it's pointless (and slower) to send GLE.
*/
send_safe = (safe || !continue_on_error);
max_bson_size_obj = PyObject_GetAttrString(ctx, "max_bson_size");
#if PY_MAJOR_VERSION >= 3
max_bson_size = PyLong_AsLong(max_bson_size_obj);
#else
max_bson_size = PyInt_AsLong(max_bson_size_obj);
#endif
Py_XDECREF(max_bson_size_obj);
if (max_bson_size == -1) {
destroy_codec_options(&options);
PyMem_Free(collection_name);
return NULL;
}
max_message_size_obj = PyObject_GetAttrString(ctx, "max_message_size");
#if PY_MAJOR_VERSION >= 3
max_message_size = PyLong_AsLong(max_message_size_obj);
#else
max_message_size = PyInt_AsLong(max_message_size_obj);
#endif
Py_XDECREF(max_message_size_obj);
if (max_message_size == -1) {
destroy_codec_options(&options);
PyMem_Free(collection_name);
return NULL;
}
buffer = buffer_new();
if (!buffer) {
destroy_codec_options(&options);
PyErr_NoMemory();
PyMem_Free(collection_name);
return NULL;
}
length_location = init_insert_buffer(buffer,
request_id,
flags,
collection_name,
collection_name_length);
if (length_location == -1) {
goto insertfail;
}
if (!(to_publish = PyList_New(0))) {
goto insertfail;
}
iterator = PyObject_GetIter(docs);
if (iterator == NULL) {
PyObject* InvalidOperation = _error("InvalidOperation");
if (InvalidOperation) {
PyErr_SetString(InvalidOperation, "input is not iterable");
Py_DECREF(InvalidOperation);
}
goto insertfail;
}
while ((doc = PyIter_Next(iterator)) != NULL) {
int before = buffer_get_position(buffer);
int cur_size;
if (!write_dict(state->_cbson, buffer, doc, check_keys,
&options, 1)) {
goto iterfail;
}
cur_size = buffer_get_position(buffer) - before;
if (cur_size > max_bson_size) {
/* If we've encoded anything send it before raising. */
if (!empty) {
buffer_update_position(buffer, before);
message_length = buffer_get_position(buffer) - length_location;
memcpy(buffer_get_buffer(buffer) + length_location,
&message_length, 4);
result = _send_insert(self, ctx, last_error_args, buffer,
collection_name, collection_name_length,
request_id, send_safe, &options,
to_publish);
if (!result)
goto iterfail;
Py_DECREF(result);
}
_set_document_too_large(cur_size, max_bson_size);
goto iterfail;
}
empty = 0;
/* We have enough data, send this batch. */
if (buffer_get_position(buffer) > max_message_size) {
int new_request_id = rand();
int message_start;
buffer_t new_buffer = buffer_new();
if (!new_buffer) {
PyErr_NoMemory();
goto iterfail;
}
message_start = init_insert_buffer(new_buffer,
new_request_id,
flags,
collection_name,
collection_name_length);
if (message_start == -1) {
buffer_free(new_buffer);
goto iterfail;
}
/* Copy the overflow encoded document into the new buffer. */
if (!buffer_write_bytes(new_buffer,
(const char*)buffer_get_buffer(buffer) + before, cur_size)) {
buffer_free(new_buffer);
goto iterfail;
}
/* Roll back to the beginning of this document. */
buffer_update_position(buffer, before);
message_length = buffer_get_position(buffer) - length_location;
memcpy(buffer_get_buffer(buffer) + length_location, &message_length, 4);
result = _send_insert(self, ctx, last_error_args, buffer,
collection_name, collection_name_length,
request_id, send_safe, &options, to_publish);
buffer_free(buffer);
buffer = new_buffer;
request_id = new_request_id;
length_location = message_start;
Py_DECREF(to_publish);
if (!(to_publish = PyList_New(0))) {
goto insertfail;
}
if (!result) {
PyObject *etype = NULL, *evalue = NULL, *etrace = NULL;
PyObject* OperationFailure;
PyErr_Fetch(&etype, &evalue, &etrace);
OperationFailure = _error("OperationFailure");
if (OperationFailure) {
if (PyErr_GivenExceptionMatches(etype, OperationFailure)) {
if (!safe || continue_on_error) {
Py_DECREF(OperationFailure);
if (!safe) {
/* We're doing unacknowledged writes and
* continue_on_error is False. Just return. */
Py_DECREF(etype);
Py_XDECREF(evalue);
Py_XDECREF(etrace);
Py_DECREF(to_publish);
Py_DECREF(iterator);
Py_DECREF(doc);
buffer_free(buffer);
PyMem_Free(collection_name);
Py_RETURN_NONE;
}
/* continue_on_error is True, store the error
* details to re-raise after the final batch */
Py_XDECREF(exc_type);
Py_XDECREF(exc_value);
Py_XDECREF(exc_trace);
exc_type = etype;
exc_value = evalue;
exc_trace = etrace;
if (PyList_Append(to_publish, doc) < 0) {
goto iterfail;
}
Py_CLEAR(doc);
continue;
}
}
Py_DECREF(OperationFailure);
}
/* This isn't OperationFailure, we couldn't
* import OperationFailure, or we are doing
* acknowledged writes. Re-raise immediately. */
PyErr_Restore(etype, evalue, etrace);
goto iterfail;
} else {
Py_DECREF(result);
}
}
if (PyList_Append(to_publish, doc) < 0) {
goto iterfail;
}
Py_CLEAR(doc);
}
Py_DECREF(iterator);
if (PyErr_Occurred()) {
goto insertfail;
}
if (empty) {
PyObject* InvalidOperation = _error("InvalidOperation");
if (InvalidOperation) {
PyErr_SetString(InvalidOperation, "cannot do an empty bulk insert");
Py_DECREF(InvalidOperation);
}
goto insertfail;
}
message_length = buffer_get_position(buffer) - length_location;
memcpy(buffer_get_buffer(buffer) + length_location, &message_length, 4);
/* Send the last (or only) batch */
result = _send_insert(self, ctx, last_error_args, buffer,
collection_name, collection_name_length,
request_id, safe, &options, to_publish);
Py_DECREF(to_publish);
PyMem_Free(collection_name);
buffer_free(buffer);
if (!result) {
Py_XDECREF(exc_type);
Py_XDECREF(exc_value);
Py_XDECREF(exc_trace);
return NULL;
} else {
Py_DECREF(result);
}
if (exc_type) {
/* Re-raise any previously stored exception
* due to continue_on_error being True */
PyErr_Restore(exc_type, exc_value, exc_trace);
return NULL;
}
Py_RETURN_NONE;
iterfail:
Py_XDECREF(doc);
Py_DECREF(iterator);
insertfail:
Py_XDECREF(exc_type);
Py_XDECREF(exc_value);
Py_XDECREF(exc_trace);
Py_XDECREF(to_publish);
buffer_free(buffer);
PyMem_Free(collection_name);
return NULL;
}
static PyObject* _cbson_insert_message(PyObject* self, PyObject* args) {
/* Used by the Bulk API to insert into pre-2.6 servers. Collection.insert
* uses _cbson_do_batched_insert. */
struct module_state *state = GETSTATE(self);
/* NOTE just using a random number as the request_id */
int request_id = rand();
char* collection_name = NULL;
int collection_name_length;
PyObject* docs;
PyObject* doc;
PyObject* iterator;
int before, cur_size, max_size = 0;
int flags = 0;
unsigned char check_keys;
unsigned char safe;
unsigned char continue_on_error;
codec_options_t options;
PyObject* last_error_args;
buffer_t buffer;
int length_location, message_length;
PyObject* result;
if (!PyArg_ParseTuple(args, "et#ObbObO&",
"utf-8",
&collection_name,
&collection_name_length,
&docs, &check_keys, &safe,
&last_error_args,
&continue_on_error,
convert_codec_options, &options)) {
return NULL;
}
if (continue_on_error) {
flags += 1;
}
buffer = buffer_new();
if (!buffer) {
PyErr_NoMemory();
destroy_codec_options(&options);
PyMem_Free(collection_name);
return NULL;
}
length_location = init_insert_buffer(buffer,
request_id,
flags,
collection_name,
collection_name_length);
if (length_location == -1) {
destroy_codec_options(&options);
PyMem_Free(collection_name);
buffer_free(buffer);
return NULL;
}
iterator = PyObject_GetIter(docs);
if (iterator == NULL) {
PyObject* InvalidOperation = _error("InvalidOperation");
if (InvalidOperation) {
PyErr_SetString(InvalidOperation, "input is not iterable");
Py_DECREF(InvalidOperation);
}
destroy_codec_options(&options);
buffer_free(buffer);
PyMem_Free(collection_name);
return NULL;
}
while ((doc = PyIter_Next(iterator)) != NULL) {
before = buffer_get_position(buffer);
if (!write_dict(state->_cbson, buffer, doc, check_keys,
&options, 1)) {
Py_DECREF(doc);
Py_DECREF(iterator);
destroy_codec_options(&options);
buffer_free(buffer);
PyMem_Free(collection_name);
return NULL;
}
Py_DECREF(doc);
cur_size = buffer_get_position(buffer) - before;
max_size = (cur_size > max_size) ? cur_size : max_size;
}
Py_DECREF(iterator);
if (PyErr_Occurred()) {
destroy_codec_options(&options);
buffer_free(buffer);
PyMem_Free(collection_name);
return NULL;
}
if (!max_size) {
PyObject* InvalidOperation = _error("InvalidOperation");
if (InvalidOperation) {
PyErr_SetString(InvalidOperation, "cannot do an empty bulk insert");
Py_DECREF(InvalidOperation);
}
destroy_codec_options(&options);
buffer_free(buffer);
PyMem_Free(collection_name);
return NULL;
}
message_length = buffer_get_position(buffer) - length_location;
memcpy(buffer_get_buffer(buffer) + length_location, &message_length, 4);
if (safe) {
if (!add_last_error(self, buffer, request_id, collection_name,
collection_name_length, &options, last_error_args)) {
destroy_codec_options(&options);
buffer_free(buffer);
PyMem_Free(collection_name);
return NULL;
}
}
PyMem_Free(collection_name);
/* objectify buffer */
result = Py_BuildValue("i" BYTES_FORMAT_STRING "i", request_id,
buffer_get_buffer(buffer),
buffer_get_position(buffer),
max_size);
destroy_codec_options(&options);
buffer_free(buffer);
return result;
}
