int agwrite(Agraph_t * g, FILE * fp) { printdict_t *p; char *t0, *t1; /* write the graph header */ t0 = (AG_IS_STRICT(g)) ? "strict " : ""; t1 = (AG_IS_DIRECTED(g)) ? "digraph" : "graph"; if (strncmp(g->name, "_anonymous", 10)) fprintf(fp, "%s%s %s {\n", t0, t1, agcanonical(g->name)); else fprintf(fp, "%s%s {\n", t0, t1); /* write the top level attribute defs */ write_dict(g->univ->globattr, fp); write_dict(g->univ->nodeattr, fp); write_dict(g->univ->edgeattr, fp); /* write the graph contents */ p = new_printdict_t(g); write_subg(g, fp, (Agraph_t *) 0, 0, p); fprintf(fp, "}\n"); free_printdict_t(p); return ferror(fp); }
void Tagger::Clear() { /* Write c_new arrays */ if (odictfile != NULL) { if (Option(verbose)) printf("Writing dictionary (%d entries)\n", dict.size); write_dict(odictfile, &dict, Option(training) || Option(reestimate)); fclose(odictfile); } if (otranfile != NULL) { if (Option(verbose)) printf("Writing transitions\n"); if (Option(training) || Option(reestimate)) write_ascii_trans(otranfile, &c_newtrans); else write_ascii_trans(otranfile, &trans); fclose(otranfile); } if (Option(unknown_morph)) { if (obadwordname[0] != 0) fclose(features->obadwordfile); } free_dict( &dict ); if ( Option(bdbm)) /* Berkeley DB: close it */ dbp->close(dbp,(u_int32_t)0); }
int agwrite(Agraph_t * g, FILE * fp) { printdict_t *p; if (AG.fwrite == NULL) { AG.fwrite = fwrite; /* init to system version of fwrite() */ } if (AG.ferror == NULL) { #ifdef ferror #undef ferror /* if ferror is a macro, then use our wrapper function, but * undef the macro first so it doesn't subst in "AG.ferror" */ AG.ferror = agferror; /* init to ferror macro wrapper function */ #else AG.ferror = ferror; /* init to system version of ferror() */ #endif } /* write the graph header */ agputs((AG_IS_STRICT(g)) ? "strict " : "", fp); agputs((AG_IS_DIRECTED(g)) ? "digraph" : "graph", fp); if (strncmp(g->name, "_anonymous", 10)) { agputc(' ', fp); agputs(agcanonical(g->name), fp); } agputs(" {\n", fp); /* write the top level attribute defs */ write_dict(g->univ->globattr, fp); write_dict(g->univ->nodeattr, fp); write_dict(g->univ->edgeattr, fp); /* write the graph contents */ p = new_printdict_t(g); write_subg(g, fp, (Agraph_t *) 0, 0, p); agputs("}\n", fp); free_printdict_t(p); return AG.ferror(fp); }
int dict_main(int argc, char *argv[]) { args_t *args = (args_t*) calloc(1,sizeof(args_t)); args->header = 1; static const struct option loptions[] = { {"help", no_argument, NULL, 'h'}, {"no-header", no_argument, NULL, 'H'}, {"assembly", required_argument, NULL, 'a'}, {"species", required_argument, NULL, 's'}, {"uri", required_argument, NULL, 'u'}, {"output", required_argument, NULL, 'o'}, {NULL, 0, NULL, 0} }; int c; while ( (c=getopt_long(argc,argv,"?hHa:s:u:o:",loptions,NULL))>0 ) { switch (c) { case 'a': args->assembly = optarg; break; case 's': args->species = optarg; break; case 'u': args->uri = optarg; break; case 'o': args->output_fname = optarg; break; case 'H': args->header = 0; break; case 'h': return dict_usage(); default: return dict_usage(); } } char *fname = NULL; if ( optind>=argc ) { if ( !isatty(fileno((FILE *)stdin)) ) fname = "-"; // reading from stdin else return dict_usage(); } else fname = argv[optind]; write_dict(fname, args); free(args); return 0; }
static PyObject* _cbson_query_message(PyObject* self, PyObject* args) { /* NOTE just using a random number as the request_id */ struct module_state *state = GETSTATE(self); int request_id = rand(); unsigned int options; char* collection_name = NULL; int collection_name_length; int begin, cur_size, max_size = 0; int num_to_skip; int num_to_return; PyObject* query; PyObject* field_selector = Py_None; unsigned char uuid_subtype = 3; buffer_t buffer; int length_location, message_length; PyObject* result; if (!PyArg_ParseTuple(args, "Iet#iiO|Ob", &options, "utf-8", &collection_name, &collection_name_length, &num_to_skip, &num_to_return, &query, &field_selector, &uuid_subtype)) { return NULL; } buffer = buffer_new(); if (!buffer) { PyErr_NoMemory(); PyMem_Free(collection_name); return NULL; } // save space for message length length_location = buffer_save_space(buffer, 4); if (length_location == -1) { PyMem_Free(collection_name); PyErr_NoMemory(); return NULL; } if (!buffer_write_bytes(buffer, (const char*)&request_id, 4) || !buffer_write_bytes(buffer, "\x00\x00\x00\x00\xd4\x07\x00\x00", 8) || !buffer_write_bytes(buffer, (const char*)&options, 4) || !buffer_write_bytes(buffer, collection_name, collection_name_length + 1) || !buffer_write_bytes(buffer, (const char*)&num_to_skip, 4) || !buffer_write_bytes(buffer, (const char*)&num_to_return, 4)) { buffer_free(buffer); PyMem_Free(collection_name); return NULL; } begin = buffer_get_position(buffer); if (!write_dict(state->_cbson, buffer, query, 0, uuid_subtype, 1)) { buffer_free(buffer); PyMem_Free(collection_name); return NULL; } max_size = buffer_get_position(buffer) - begin; if (field_selector != Py_None) { begin = buffer_get_position(buffer); if (!write_dict(state->_cbson, buffer, field_selector, 0, uuid_subtype, 1)) { buffer_free(buffer); PyMem_Free(collection_name); return NULL; } cur_size = buffer_get_position(buffer) - begin; max_size = (cur_size > max_size) ? cur_size : max_size; } PyMem_Free(collection_name); message_length = buffer_get_position(buffer) - length_location; memcpy(buffer_get_buffer(buffer) + length_location, &message_length, 4); /* objectify buffer */ result = Py_BuildValue("i" BYTES_FORMAT_STRING "i", request_id, buffer_get_buffer(buffer), buffer_get_position(buffer), max_size); buffer_free(buffer); return result; }
static PyObject* _cbson_update_message(PyObject* self, PyObject* args) { /* NOTE just using a random number as the request_id */ struct module_state *state = GETSTATE(self); int request_id = rand(); char* collection_name = NULL; int collection_name_length; int before, cur_size, max_size = 0; PyObject* doc; PyObject* spec; unsigned char multi; unsigned char upsert; unsigned char safe; unsigned char check_keys; unsigned char uuid_subtype; PyObject* last_error_args; int options; buffer_t buffer; int length_location, message_length; PyObject* result; if (!PyArg_ParseTuple(args, "et#bbOObObb", "utf-8", &collection_name, &collection_name_length, &upsert, &multi, &spec, &doc, &safe, &last_error_args, &check_keys, &uuid_subtype)) { return NULL; } options = 0; if (upsert) { options += 1; } if (multi) { options += 2; } buffer = buffer_new(); if (!buffer) { PyErr_NoMemory(); PyMem_Free(collection_name); return NULL; } // save space for message length length_location = buffer_save_space(buffer, 4); if (length_location == -1) { PyMem_Free(collection_name); PyErr_NoMemory(); return NULL; } if (!buffer_write_bytes(buffer, (const char*)&request_id, 4) || !buffer_write_bytes(buffer, "\x00\x00\x00\x00" "\xd1\x07\x00\x00" "\x00\x00\x00\x00", 12) || !buffer_write_bytes(buffer, collection_name, collection_name_length + 1) || !buffer_write_bytes(buffer, (const char*)&options, 4)) { buffer_free(buffer); PyMem_Free(collection_name); return NULL; } before = buffer_get_position(buffer); if (!write_dict(state->_cbson, buffer, spec, 0, uuid_subtype, 1)) { buffer_free(buffer); PyMem_Free(collection_name); return NULL; } max_size = buffer_get_position(buffer) - before; before = buffer_get_position(buffer); if (!write_dict(state->_cbson, buffer, doc, check_keys, uuid_subtype, 1)) { buffer_free(buffer); PyMem_Free(collection_name); return NULL; } cur_size = buffer_get_position(buffer) - before; max_size = (cur_size > max_size) ? cur_size : max_size; message_length = buffer_get_position(buffer) - length_location; memcpy(buffer_get_buffer(buffer) + length_location, &message_length, 4); if (safe) { if (!add_last_error(self, buffer, request_id, collection_name, collection_name_length, last_error_args)) { buffer_free(buffer); PyMem_Free(collection_name); return NULL; } } PyMem_Free(collection_name); /* objectify buffer */ result = Py_BuildValue("i" BYTES_FORMAT_STRING "i", request_id, buffer_get_buffer(buffer), buffer_get_position(buffer), max_size); buffer_free(buffer); return result; }
static PyObject* _cbson_insert_message(PyObject* self, PyObject* args) { /* NOTE just using a random number as the request_id */ struct module_state *state = GETSTATE(self); int request_id = rand(); char* collection_name = NULL; int collection_name_length; PyObject* docs; PyObject* doc; PyObject* iterator; int before, cur_size, max_size = 0; int options = 0; unsigned char check_keys; unsigned char safe; unsigned char continue_on_error; unsigned char uuid_subtype; PyObject* last_error_args; buffer_t buffer; int length_location, message_length; PyObject* result; if (!PyArg_ParseTuple(args, "et#ObbObb", "utf-8", &collection_name, &collection_name_length, &docs, &check_keys, &safe, &last_error_args, &continue_on_error, &uuid_subtype)) { return NULL; } if (continue_on_error) { options += 1; } buffer = buffer_new(); if (!buffer) { PyErr_NoMemory(); PyMem_Free(collection_name); return NULL; } // save space for message length length_location = buffer_save_space(buffer, 4); if (length_location == -1) { PyMem_Free(collection_name); PyErr_NoMemory(); return NULL; } if (!buffer_write_bytes(buffer, (const char*)&request_id, 4) || !buffer_write_bytes(buffer, "\x00\x00\x00\x00" "\xd2\x07\x00\x00", 8) || !buffer_write_bytes(buffer, (const char*)&options, 4) || !buffer_write_bytes(buffer, collection_name, collection_name_length + 1)) { PyMem_Free(collection_name); buffer_free(buffer); return NULL; } iterator = PyObject_GetIter(docs); if (iterator == NULL) { PyObject* InvalidOperation = _error("InvalidOperation"); if (InvalidOperation) { PyErr_SetString(InvalidOperation, "input is not iterable"); Py_DECREF(InvalidOperation); } buffer_free(buffer); PyMem_Free(collection_name); return NULL; } while ((doc = PyIter_Next(iterator)) != NULL) { before = buffer_get_position(buffer); if (!write_dict(state->_cbson, buffer, doc, check_keys, uuid_subtype, 1)) { Py_DECREF(doc); Py_DECREF(iterator); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } Py_DECREF(doc); cur_size = buffer_get_position(buffer) - before; max_size = (cur_size > max_size) ? cur_size : max_size; } Py_DECREF(iterator); if (PyErr_Occurred()) { buffer_free(buffer); PyMem_Free(collection_name); return NULL; } if (!max_size) { PyObject* InvalidOperation = _error("InvalidOperation"); if (InvalidOperation) { PyErr_SetString(InvalidOperation, "cannot do an empty bulk insert"); Py_DECREF(InvalidOperation); } buffer_free(buffer); PyMem_Free(collection_name); return NULL; } message_length = buffer_get_position(buffer) - length_location; memcpy(buffer_get_buffer(buffer) + length_location, &message_length, 4); if (safe) { if (!add_last_error(self, buffer, request_id, collection_name, collection_name_length, last_error_args)) { buffer_free(buffer); PyMem_Free(collection_name); return NULL; } } PyMem_Free(collection_name); /* objectify buffer */ result = Py_BuildValue("i" BYTES_FORMAT_STRING "i", request_id, buffer_get_buffer(buffer), buffer_get_position(buffer), max_size); buffer_free(buffer); return result; }
void plist_to_bin(plist_t plist, char **plist_bin, uint32_t * length) { GPtrArray *objects = NULL; GHashTable *ref_table = NULL; struct serialize_s ser_s; uint8_t offset_size = 0; uint8_t dict_param_size = 0; uint64_t num_objects = 0; uint64_t root_object = 0; uint64_t offset_table_index = 0; GByteArray *bplist_buff = NULL; uint64_t i = 0; uint8_t *buff = NULL; uint64_t *offsets = NULL; uint8_t pad[6] = { 0, 0, 0, 0, 0, 0 }; uint8_t trailer[BPLIST_TRL_SIZE]; //for string glong len = 0; int type = 0; glong items_read = 0; glong items_written = 0; GError *error = NULL; gunichar2 *unicodestr = NULL; //check for valid input if (!plist || !plist_bin || *plist_bin || !length) return; //list of objects objects = g_ptr_array_new(); //hashtable to write only once same nodes ref_table = g_hash_table_new(plist_data_hash, plist_data_compare); //serialize plist ser_s.objects = objects; ser_s.ref_table = ref_table; serialize_plist(plist, &ser_s); //now stream to output buffer offset_size = 0; //unknown yet dict_param_size = get_needed_bytes(objects->len); num_objects = objects->len; root_object = 0; //root is first in list offset_table_index = 0; //unknown yet //setup a dynamic bytes array to store bplist in bplist_buff = g_byte_array_new(); //set magic number and version g_byte_array_append(bplist_buff, BPLIST_MAGIC, BPLIST_MAGIC_SIZE); g_byte_array_append(bplist_buff, BPLIST_VERSION, BPLIST_VERSION_SIZE); //write objects and table offsets = (uint64_t *) malloc(num_objects * sizeof(uint64_t)); for (i = 0; i < num_objects; i++) { plist_data_t data = plist_get_data(g_ptr_array_index(objects, i)); offsets[i] = bplist_buff->len; switch (data->type) { case PLIST_BOOLEAN: buff = (uint8_t *) malloc(sizeof(uint8_t)); buff[0] = data->boolval ? BPLIST_TRUE : BPLIST_FALSE; g_byte_array_append(bplist_buff, buff, sizeof(uint8_t)); free(buff); break; case PLIST_UINT: write_int(bplist_buff, data->intval); break; case PLIST_REAL: write_real(bplist_buff, data->realval); break; case PLIST_KEY: case PLIST_STRING: len = strlen(data->strval); if ( is_ascii_string(data->strval, len) ) { write_string(bplist_buff, data->strval); } else { unicodestr = g_utf8_to_utf16(data->strval, len, &items_read, &items_written, &error); write_unicode(bplist_buff, unicodestr, items_written); g_free(unicodestr); } break; case PLIST_DATA: write_data(bplist_buff, data->buff, data->length); case PLIST_ARRAY: write_array(bplist_buff, g_ptr_array_index(objects, i), ref_table, dict_param_size); break; case PLIST_DICT: write_dict(bplist_buff, g_ptr_array_index(objects, i), ref_table, dict_param_size); break; case PLIST_DATE: write_date(bplist_buff, data->timeval.tv_sec + (double) data->timeval.tv_usec / G_USEC_PER_SEC); break; default: break; } } //free intermediate objects g_hash_table_foreach_remove(ref_table, free_index, NULL); g_ptr_array_free(objects, TRUE); g_hash_table_destroy(ref_table); //write offsets offset_size = get_needed_bytes(bplist_buff->len); offset_table_index = bplist_buff->len; for (i = 0; i < num_objects; i++) { uint8_t *offsetbuff = (uint8_t *) malloc(offset_size); #if G_BYTE_ORDER == G_BIG_ENDIAN offsets[i] = offsets[i] << ((sizeof(uint64_t) - offset_size) * 8); #endif memcpy(offsetbuff, &offsets[i], offset_size); byte_convert(offsetbuff, offset_size); g_byte_array_append(bplist_buff, offsetbuff, offset_size); free(offsetbuff); } //experimental pad to reflect apple's files g_byte_array_append(bplist_buff, pad, 6); //setup trailer num_objects = GUINT64_FROM_BE(num_objects); root_object = GUINT64_FROM_BE(root_object); offset_table_index = GUINT64_FROM_BE(offset_table_index); memcpy(trailer + BPLIST_TRL_OFFSIZE_IDX, &offset_size, sizeof(uint8_t)); memcpy(trailer + BPLIST_TRL_PARMSIZE_IDX, &dict_param_size, sizeof(uint8_t)); memcpy(trailer + BPLIST_TRL_NUMOBJ_IDX, &num_objects, sizeof(uint64_t)); memcpy(trailer + BPLIST_TRL_ROOTOBJ_IDX, &root_object, sizeof(uint64_t)); memcpy(trailer + BPLIST_TRL_OFFTAB_IDX, &offset_table_index, sizeof(uint64_t)); g_byte_array_append(bplist_buff, trailer, BPLIST_TRL_SIZE); //duplicate buffer *plist_bin = (char *) malloc(bplist_buff->len); memcpy(*plist_bin, bplist_buff->data, bplist_buff->len); *length = bplist_buff->len; g_byte_array_free(bplist_buff, TRUE); free(offsets); }
static void write_core(pic_state *pic, pic_value obj, pic_value port, struct writer_control *p) { pic_value labels = p->labels; int i; /* shared objects */ if (is_shared_object(pic, obj, p)) { if (pic_weak_has(pic, labels, obj)) { pic_fprintf(pic, port, "#%d#", pic_int(pic, pic_weak_ref(pic, labels, obj))); return; } i = p->cnt++; pic_fprintf(pic, port, "#%d=", i); pic_weak_set(pic, labels, obj, pic_int_value(pic, i)); } switch (pic_type(pic, obj)) { case PIC_TYPE_UNDEF: pic_fprintf(pic, port, "#undefined"); break; case PIC_TYPE_NIL: pic_fprintf(pic, port, "()"); break; case PIC_TYPE_TRUE: pic_fprintf(pic, port, "#t"); break; case PIC_TYPE_FALSE: pic_fprintf(pic, port, "#f"); break; case PIC_TYPE_ID: pic_fprintf(pic, port, "#<identifier %s>", pic_str(pic, pic_id_name(pic, obj))); break; case PIC_TYPE_EOF: pic_fprintf(pic, port, "#.(eof-object)"); break; case PIC_TYPE_INT: pic_fprintf(pic, port, "%d", pic_int(pic, obj)); break; case PIC_TYPE_SYMBOL: pic_fprintf(pic, port, "%s", pic_sym(pic, obj)); break; case PIC_TYPE_FLOAT: write_float(pic, obj, port); break; case PIC_TYPE_BLOB: write_blob(pic, obj, port); break; case PIC_TYPE_CHAR: write_char(pic, obj, port, p); break; case PIC_TYPE_STRING: write_str(pic, obj, port, p); break; case PIC_TYPE_PAIR: write_pair(pic, obj, port, p); break; case PIC_TYPE_VECTOR: write_vec(pic, obj, port, p); break; case PIC_TYPE_DICT: write_dict(pic, obj, port, p); break; default: pic_fprintf(pic, port, "#<%s %p>", pic_typename(pic, pic_type(pic, obj)), pic_obj_ptr(obj)); break; } if (p->op == OP_WRITE) { if (is_shared_object(pic, obj, p)) { pic_weak_del(pic, labels, obj); } } }
static PyObject* _cbson_do_batched_write_command(PyObject* self, PyObject* args) { struct module_state *state = GETSTATE(self); long max_bson_size; long max_cmd_size; long max_write_batch_size; long idx_offset = 0; int idx = 0; int cmd_len_loc; int lst_len_loc; int ns_len; int ordered; char *ns = NULL; PyObject* max_bson_size_obj; PyObject* max_write_batch_size_obj; PyObject* command; PyObject* doc; PyObject* docs; PyObject* ctx; PyObject* iterator; PyObject* result; PyObject* results; PyObject* to_publish = NULL; unsigned char op; unsigned char check_keys; codec_options_t options; unsigned char empty = 1; unsigned char errors = 0; buffer_t buffer; if (!PyArg_ParseTuple(args, "et#bOObO&O", "utf-8", &ns, &ns_len, &op, &command, &docs, &check_keys, convert_codec_options, &options, &ctx)) { return NULL; } max_bson_size_obj = PyObject_GetAttrString(ctx, "max_bson_size"); #if PY_MAJOR_VERSION >= 3 max_bson_size = PyLong_AsLong(max_bson_size_obj); #else max_bson_size = PyInt_AsLong(max_bson_size_obj); #endif Py_XDECREF(max_bson_size_obj); if (max_bson_size == -1) { destroy_codec_options(&options); PyMem_Free(ns); return NULL; } /* * Max BSON object size + 16k - 2 bytes for ending NUL bytes * XXX: This should come from the server - SERVER-10643 */ max_cmd_size = max_bson_size + 16382; max_write_batch_size_obj = PyObject_GetAttrString(ctx, "max_write_batch_size"); #if PY_MAJOR_VERSION >= 3 max_write_batch_size = PyLong_AsLong(max_write_batch_size_obj); #else max_write_batch_size = PyInt_AsLong(max_write_batch_size_obj); #endif Py_XDECREF(max_write_batch_size_obj); if (max_write_batch_size == -1) { destroy_codec_options(&options); PyMem_Free(ns); return NULL; } /* Default to True */ ordered = !((PyDict_GetItemString(command, "ordered")) == Py_False); if (!(results = PyList_New(0))) { destroy_codec_options(&options); PyMem_Free(ns); return NULL; } if (!(to_publish = PyList_New(0))) { destroy_codec_options(&options); PyMem_Free(ns); Py_DECREF(results); return NULL; } if (!(buffer = _command_buffer_new(ns, ns_len))) { destroy_codec_options(&options); PyMem_Free(ns); Py_DECREF(results); Py_DECREF(to_publish); return NULL; } PyMem_Free(ns); /* Position of command document length */ cmd_len_loc = buffer_get_position(buffer); if (!write_dict(state->_cbson, buffer, command, 0, &options, 0)) { goto cmdfail; } /* Write type byte for array */ *(buffer_get_buffer(buffer) + (buffer_get_position(buffer) - 1)) = 0x4; switch (op) { case _INSERT: { if (!buffer_write_bytes(buffer, "documents\x00", 10)) goto cmdfail; break; } case _UPDATE: { /* MongoDB does key validation for update. */ check_keys = 0; if (!buffer_write_bytes(buffer, "updates\x00", 8)) goto cmdfail; break; } case _DELETE: { /* Never check keys in a delete command. */ check_keys = 0; if (!buffer_write_bytes(buffer, "deletes\x00", 8)) goto cmdfail; break; } default: { PyObject* InvalidOperation = _error("InvalidOperation"); if (InvalidOperation) { PyErr_SetString(InvalidOperation, "Unknown command"); Py_DECREF(InvalidOperation); } goto cmdfail; } } /* Save space for list document */ lst_len_loc = buffer_save_space(buffer, 4); if (lst_len_loc == -1) { PyErr_NoMemory(); goto cmdfail; } iterator = PyObject_GetIter(docs); if (iterator == NULL) { PyObject* InvalidOperation = _error("InvalidOperation"); if (InvalidOperation) { PyErr_SetString(InvalidOperation, "input is not iterable"); Py_DECREF(InvalidOperation); } goto cmdfail; } while ((doc = PyIter_Next(iterator)) != NULL) { int sub_doc_begin = buffer_get_position(buffer); int cur_doc_begin; int cur_size; int enough_data = 0; int enough_documents = 0; char key[16]; empty = 0; INT2STRING(key, idx); if (!buffer_write_bytes(buffer, "\x03", 1) || !buffer_write_bytes(buffer, key, (int)strlen(key) + 1)) { goto cmditerfail; } cur_doc_begin = buffer_get_position(buffer); if (!write_dict(state->_cbson, buffer, doc, check_keys, &options, 1)) { goto cmditerfail; } /* We have enough data, maybe send this batch. */ enough_data = (buffer_get_position(buffer) > max_cmd_size); enough_documents = (idx >= max_write_batch_size); if (enough_data || enough_documents) { buffer_t new_buffer; cur_size = buffer_get_position(buffer) - cur_doc_begin; /* This single document is too large for the command. */ if (!idx) { if (op == _INSERT) { _set_document_too_large(cur_size, max_bson_size); } else { PyObject* DocumentTooLarge = _error("DocumentTooLarge"); if (DocumentTooLarge) { /* * There's nothing intelligent we can say * about size for update and remove. */ PyErr_SetString(DocumentTooLarge, "command document too large"); Py_DECREF(DocumentTooLarge); } } goto cmditerfail; } if (!(new_buffer = buffer_new())) { PyErr_NoMemory(); goto cmditerfail; } /* New buffer including the current overflow document */ if (!buffer_write_bytes(new_buffer, (const char*)buffer_get_buffer(buffer), lst_len_loc + 5) || !buffer_write_bytes(new_buffer, "0\x00", 2) || !buffer_write_bytes(new_buffer, (const char*)buffer_get_buffer(buffer) + cur_doc_begin, cur_size)) { buffer_free(new_buffer); goto cmditerfail; } /* * Roll the existing buffer back to the beginning * of the last document encoded. */ buffer_update_position(buffer, sub_doc_begin); if (!buffer_write_bytes(buffer, "\x00\x00", 2)) { buffer_free(new_buffer); goto cmditerfail; } result = _send_write_command(ctx, buffer, lst_len_loc, cmd_len_loc, &errors, to_publish); buffer_free(buffer); buffer = new_buffer; if (!result) goto cmditerfail; #if PY_MAJOR_VERSION >= 3 result = Py_BuildValue("NN", PyLong_FromLong(idx_offset), result); #else result = Py_BuildValue("NN", PyInt_FromLong(idx_offset), result); #endif if (!result) goto cmditerfail; if (PyList_Append(results, result) < 0) { Py_DECREF(result); goto cmditerfail; } Py_DECREF(result); if (errors && ordered) { destroy_codec_options(&options); Py_DECREF(iterator); buffer_free(buffer); return results; } idx_offset += idx; idx = 0; Py_DECREF(to_publish); if (!(to_publish = PyList_New(0))) { goto cmditerfail; } } if (PyList_Append(to_publish, doc) < 0) { goto cmditerfail; } Py_CLEAR(doc); idx += 1; } Py_DECREF(iterator); if (PyErr_Occurred()) { goto cmdfail; } if (empty) { PyObject* InvalidOperation = _error("InvalidOperation"); if (InvalidOperation) { PyErr_SetString(InvalidOperation, "cannot do an empty bulk write"); Py_DECREF(InvalidOperation); } goto cmdfail; } if (!buffer_write_bytes(buffer, "\x00\x00", 2)) goto cmdfail; result = _send_write_command(ctx, buffer, lst_len_loc, cmd_len_loc, &errors, to_publish); if (!result) goto cmdfail; #if PY_MAJOR_VERSION >= 3 result = Py_BuildValue("NN", PyLong_FromLong(idx_offset), result); #else result = Py_BuildValue("NN", PyInt_FromLong(idx_offset), result); #endif if (!result) goto cmdfail; buffer_free(buffer); if (PyList_Append(results, result) < 0) { Py_DECREF(result); goto cmdfail; } Py_DECREF(result); Py_DECREF(to_publish); destroy_codec_options(&options); return results; cmditerfail: Py_XDECREF(doc); Py_DECREF(iterator); cmdfail: destroy_codec_options(&options); Py_DECREF(results); Py_XDECREF(to_publish); buffer_free(buffer); return NULL; }
static PyObject* _cbson_do_batched_insert(PyObject* self, PyObject* args) { struct module_state *state = GETSTATE(self); /* NOTE just using a random number as the request_id */ int request_id = rand(); int send_safe, flags = 0; int length_location, message_length; int collection_name_length; char* collection_name = NULL; PyObject* docs; PyObject* doc; PyObject* iterator; PyObject* ctx; PyObject* last_error_args; PyObject* result; PyObject* max_bson_size_obj; PyObject* max_message_size_obj; PyObject* to_publish = NULL; unsigned char check_keys; unsigned char safe; unsigned char continue_on_error; codec_options_t options; unsigned char empty = 1; long max_bson_size; long max_message_size; buffer_t buffer; PyObject *exc_type = NULL, *exc_value = NULL, *exc_trace = NULL; if (!PyArg_ParseTuple(args, "et#ObbObO&O", "utf-8", &collection_name, &collection_name_length, &docs, &check_keys, &safe, &last_error_args, &continue_on_error, convert_codec_options, &options, &ctx)) { return NULL; } if (continue_on_error) { flags += 1; } /* * If we are doing unacknowledged writes *and* continue_on_error * is True it's pointless (and slower) to send GLE. */ send_safe = (safe || !continue_on_error); max_bson_size_obj = PyObject_GetAttrString(ctx, "max_bson_size"); #if PY_MAJOR_VERSION >= 3 max_bson_size = PyLong_AsLong(max_bson_size_obj); #else max_bson_size = PyInt_AsLong(max_bson_size_obj); #endif Py_XDECREF(max_bson_size_obj); if (max_bson_size == -1) { destroy_codec_options(&options); PyMem_Free(collection_name); return NULL; } max_message_size_obj = PyObject_GetAttrString(ctx, "max_message_size"); #if PY_MAJOR_VERSION >= 3 max_message_size = PyLong_AsLong(max_message_size_obj); #else max_message_size = PyInt_AsLong(max_message_size_obj); #endif Py_XDECREF(max_message_size_obj); if (max_message_size == -1) { destroy_codec_options(&options); PyMem_Free(collection_name); return NULL; } buffer = buffer_new(); if (!buffer) { destroy_codec_options(&options); PyErr_NoMemory(); PyMem_Free(collection_name); return NULL; } length_location = init_insert_buffer(buffer, request_id, flags, collection_name, collection_name_length); if (length_location == -1) { goto insertfail; } if (!(to_publish = PyList_New(0))) { goto insertfail; } iterator = PyObject_GetIter(docs); if (iterator == NULL) { PyObject* InvalidOperation = _error("InvalidOperation"); if (InvalidOperation) { PyErr_SetString(InvalidOperation, "input is not iterable"); Py_DECREF(InvalidOperation); } goto insertfail; } while ((doc = PyIter_Next(iterator)) != NULL) { int before = buffer_get_position(buffer); int cur_size; if (!write_dict(state->_cbson, buffer, doc, check_keys, &options, 1)) { goto iterfail; } cur_size = buffer_get_position(buffer) - before; if (cur_size > max_bson_size) { /* If we've encoded anything send it before raising. */ if (!empty) { buffer_update_position(buffer, before); message_length = buffer_get_position(buffer) - length_location; memcpy(buffer_get_buffer(buffer) + length_location, &message_length, 4); result = _send_insert(self, ctx, last_error_args, buffer, collection_name, collection_name_length, request_id, send_safe, &options, to_publish); if (!result) goto iterfail; Py_DECREF(result); } _set_document_too_large(cur_size, max_bson_size); goto iterfail; } empty = 0; /* We have enough data, send this batch. */ if (buffer_get_position(buffer) > max_message_size) { int new_request_id = rand(); int message_start; buffer_t new_buffer = buffer_new(); if (!new_buffer) { PyErr_NoMemory(); goto iterfail; } message_start = init_insert_buffer(new_buffer, new_request_id, flags, collection_name, collection_name_length); if (message_start == -1) { buffer_free(new_buffer); goto iterfail; } /* Copy the overflow encoded document into the new buffer. */ if (!buffer_write_bytes(new_buffer, (const char*)buffer_get_buffer(buffer) + before, cur_size)) { buffer_free(new_buffer); goto iterfail; } /* Roll back to the beginning of this document. */ buffer_update_position(buffer, before); message_length = buffer_get_position(buffer) - length_location; memcpy(buffer_get_buffer(buffer) + length_location, &message_length, 4); result = _send_insert(self, ctx, last_error_args, buffer, collection_name, collection_name_length, request_id, send_safe, &options, to_publish); buffer_free(buffer); buffer = new_buffer; request_id = new_request_id; length_location = message_start; Py_DECREF(to_publish); if (!(to_publish = PyList_New(0))) { goto insertfail; } if (!result) { PyObject *etype = NULL, *evalue = NULL, *etrace = NULL; PyObject* OperationFailure; PyErr_Fetch(&etype, &evalue, &etrace); OperationFailure = _error("OperationFailure"); if (OperationFailure) { if (PyErr_GivenExceptionMatches(etype, OperationFailure)) { if (!safe || continue_on_error) { Py_DECREF(OperationFailure); if (!safe) { /* We're doing unacknowledged writes and * continue_on_error is False. Just return. */ Py_DECREF(etype); Py_XDECREF(evalue); Py_XDECREF(etrace); Py_DECREF(to_publish); Py_DECREF(iterator); Py_DECREF(doc); buffer_free(buffer); PyMem_Free(collection_name); Py_RETURN_NONE; } /* continue_on_error is True, store the error * details to re-raise after the final batch */ Py_XDECREF(exc_type); Py_XDECREF(exc_value); Py_XDECREF(exc_trace); exc_type = etype; exc_value = evalue; exc_trace = etrace; if (PyList_Append(to_publish, doc) < 0) { goto iterfail; } Py_CLEAR(doc); continue; } } Py_DECREF(OperationFailure); } /* This isn't OperationFailure, we couldn't * import OperationFailure, or we are doing * acknowledged writes. Re-raise immediately. */ PyErr_Restore(etype, evalue, etrace); goto iterfail; } else { Py_DECREF(result); } } if (PyList_Append(to_publish, doc) < 0) { goto iterfail; } Py_CLEAR(doc); } Py_DECREF(iterator); if (PyErr_Occurred()) { goto insertfail; } if (empty) { PyObject* InvalidOperation = _error("InvalidOperation"); if (InvalidOperation) { PyErr_SetString(InvalidOperation, "cannot do an empty bulk insert"); Py_DECREF(InvalidOperation); } goto insertfail; } message_length = buffer_get_position(buffer) - length_location; memcpy(buffer_get_buffer(buffer) + length_location, &message_length, 4); /* Send the last (or only) batch */ result = _send_insert(self, ctx, last_error_args, buffer, collection_name, collection_name_length, request_id, safe, &options, to_publish); Py_DECREF(to_publish); PyMem_Free(collection_name); buffer_free(buffer); if (!result) { Py_XDECREF(exc_type); Py_XDECREF(exc_value); Py_XDECREF(exc_trace); return NULL; } else { Py_DECREF(result); } if (exc_type) { /* Re-raise any previously stored exception * due to continue_on_error being True */ PyErr_Restore(exc_type, exc_value, exc_trace); return NULL; } Py_RETURN_NONE; iterfail: Py_XDECREF(doc); Py_DECREF(iterator); insertfail: Py_XDECREF(exc_type); Py_XDECREF(exc_value); Py_XDECREF(exc_trace); Py_XDECREF(to_publish); buffer_free(buffer); PyMem_Free(collection_name); return NULL; }
static PyObject* _cbson_insert_message(PyObject* self, PyObject* args) { /* Used by the Bulk API to insert into pre-2.6 servers. Collection.insert * uses _cbson_do_batched_insert. */ struct module_state *state = GETSTATE(self); /* NOTE just using a random number as the request_id */ int request_id = rand(); char* collection_name = NULL; int collection_name_length; PyObject* docs; PyObject* doc; PyObject* iterator; int before, cur_size, max_size = 0; int flags = 0; unsigned char check_keys; unsigned char safe; unsigned char continue_on_error; codec_options_t options; PyObject* last_error_args; buffer_t buffer; int length_location, message_length; PyObject* result; if (!PyArg_ParseTuple(args, "et#ObbObO&", "utf-8", &collection_name, &collection_name_length, &docs, &check_keys, &safe, &last_error_args, &continue_on_error, convert_codec_options, &options)) { return NULL; } if (continue_on_error) { flags += 1; } buffer = buffer_new(); if (!buffer) { PyErr_NoMemory(); destroy_codec_options(&options); PyMem_Free(collection_name); return NULL; } length_location = init_insert_buffer(buffer, request_id, flags, collection_name, collection_name_length); if (length_location == -1) { destroy_codec_options(&options); PyMem_Free(collection_name); buffer_free(buffer); return NULL; } iterator = PyObject_GetIter(docs); if (iterator == NULL) { PyObject* InvalidOperation = _error("InvalidOperation"); if (InvalidOperation) { PyErr_SetString(InvalidOperation, "input is not iterable"); Py_DECREF(InvalidOperation); } destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } while ((doc = PyIter_Next(iterator)) != NULL) { before = buffer_get_position(buffer); if (!write_dict(state->_cbson, buffer, doc, check_keys, &options, 1)) { Py_DECREF(doc); Py_DECREF(iterator); destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } Py_DECREF(doc); cur_size = buffer_get_position(buffer) - before; max_size = (cur_size > max_size) ? cur_size : max_size; } Py_DECREF(iterator); if (PyErr_Occurred()) { destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } if (!max_size) { PyObject* InvalidOperation = _error("InvalidOperation"); if (InvalidOperation) { PyErr_SetString(InvalidOperation, "cannot do an empty bulk insert"); Py_DECREF(InvalidOperation); } destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } message_length = buffer_get_position(buffer) - length_location; memcpy(buffer_get_buffer(buffer) + length_location, &message_length, 4); if (safe) { if (!add_last_error(self, buffer, request_id, collection_name, collection_name_length, &options, last_error_args)) { destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } } PyMem_Free(collection_name); /* objectify buffer */ result = Py_BuildValue("i" BYTES_FORMAT_STRING "i", request_id, buffer_get_buffer(buffer), buffer_get_position(buffer), max_size); destroy_codec_options(&options); buffer_free(buffer); return result; }