int as_particle_get_flat_size(as_bin *b, size_t *flat_size) { if (!b) return (-1); as_particle *p = as_bin_get_particle(b); uint8_t type = as_bin_get_particle_type(b); if (type == AS_PARTICLE_TYPE_NULL) return (-1); #ifdef EXTRA_CHECKS // check the incoming type if (type < AS_PARTICLE_TYPE_NULL || type >= AS_PARTICLE_TYPE_MAX) { cf_info(AS_PARTICLE, "particle set: bad particle type %d, error", (int)type); return(-1); } #endif uint32_t size = g_particle_get_flat_table[type](p); if (size == 0) return(-1); *flat_size = size; return(0); }
uint32_t as_particle_get_size_in_memory(as_bin *b, as_particle *particle) { uint8_t type = as_bin_get_particle_type(b); switch(type) { case AS_PARTICLE_TYPE_INTEGER: case AS_PARTICLE_TYPE_NULL: case AS_PARTICLE_TYPE_FLOAT: case AS_PARTICLE_TYPE_DIGEST: return 0; case AS_PARTICLE_TYPE_STRING: return sizeof(as_particle_string) + ((as_particle_string *)particle)->sz; case AS_PARTICLE_TYPE_APPEND: return 0; // can't do append without knowing the append structure which requires integration case AS_PARTICLE_TYPE_BLOB: case AS_PARTICLE_TYPE_JAVA_BLOB: case AS_PARTICLE_TYPE_CSHARP_BLOB: case AS_PARTICLE_TYPE_PYTHON_BLOB: case AS_PARTICLE_TYPE_RUBY_BLOB: case AS_PARTICLE_TYPE_PHP_BLOB: case AS_PARTICLE_TYPE_ERLANG_BLOB: return sizeof(as_particle_blob) + ((as_particle_blob *)particle)->sz; default: break; } return 0; }
//In case of memcache append/prepend, appending string to a string is allowed currently. //We have the capability to append Integers and Blobs,but we have restricted it currently, //to avoid confusions in use cases. int as_particle_append_prepend_data(as_bin *b, as_particle_type type, byte *data, uint32_t data_len, bool data_in_memory, bool is_append, bool memcache_compliant) { as_particle_type old_type = as_bin_get_particle_type(b); as_particle_type new_type; switch(old_type) { case AS_PARTICLE_TYPE_STRING: particle_append_prepend_data_impl(b, old_type, type, &new_type, data, data_len, data_in_memory, is_append, memcache_compliant); break; case AS_PARTICLE_TYPE_BLOB: if(!memcache_compliant) { particle_append_prepend_data_impl(b, old_type, type, &new_type, data, data_len, data_in_memory, is_append, memcache_compliant); } else { return -1; } break; case AS_PARTICLE_TYPE_INTEGER: return -1; break; default: return -1; } if (new_type == AS_PARTICLE_TYPE_INTEGER) { as_bin_state_set(b, AS_BIN_STATE_INUSE_INTEGER); } else { // We do not support append prepend to hidden bins as_bin_state_set(b, AS_BIN_STATE_INUSE_OTHER); } return 0; }
/* ** as_particle_pointer ** get the pointer being held by the particle ** no ref count. don't do anything to the memory */ int as_particle_p_get(as_bin *b, byte **buf, uint32_t *sz) { if (!b) return (-1); as_particle *p = as_bin_get_particle(b); uint8_t type = as_bin_get_particle_type(b); int rv = g_particle_getter_p_table[type](p, (void **)buf, sz); return(rv); }
void as_particle_destroy(as_bin *b, bool data_in_memory) { if (as_bin_is_integer(b)) { b->particle = 0; // this is the same field as the integer in the union } else if (b->particle) { if (data_in_memory) { g_particle_destructor_table[as_bin_get_particle_type(b)](b->particle); } b->particle = 0; } }
uint32_t as_bin_get_particle_size(as_bin *b) { if (! as_bin_inuse(b)) return (0); as_particle *p = as_bin_get_particle(b); uint8_t type = as_bin_get_particle_type(b); uint32_t sz = 0; (void)g_particle_getter_table[type](p, 0, &sz); return(sz); }
// NOTE: tojson is IGNORED int _as_particle_tobuf(as_bin *b, byte *buf, uint32_t *sz, bool tojson) { if (!b) return (-1); as_particle *p = as_bin_get_particle(b); uint8_t type = as_bin_get_particle_type(b); #ifdef EXTRA_CHECKS // check the incoming type if (type < AS_PARTICLE_TYPE_NULL || type >= AS_PARTICLE_TYPE_MAX) { cf_info(AS_PARTICLE, "particle set: bad particle type %d, error", (int)type); return(-1); } #endif int rv = g_particle_getter_table[type](p, buf, sz); return(rv); }
/* as_particle_compare_frombuf * Compares two particles and returns equal or not */ int as_particle_compare_frombuf(as_bin *b, as_particle_type type, byte *buf, uint32_t sz) { #ifdef EXTRA_CHECKS // check the incoming type if (type < AS_PARTICLE_TYPE_NULL || type >= AS_PARTICLE_TYPE_MAX) { cf_info(AS_PARTICLE, "particle compare: bad particle type %d, error", (int)type); return(-1); } #endif // check if the types match if (as_bin_is_integer(b)) { if (type == AS_PARTICLE_TYPE_INTEGER) return (g_particle_compare_table[type](&b->iparticle, buf, sz)); else return (-1); } else if (b->particle && type == as_bin_get_particle_type(b)) return (g_particle_compare_table[type](b->particle, buf, sz)); else return (-1); }
bool write_sindex_update(as_namespace* ns, const char* set_name, cf_digest* keyd, as_bin* old_bins, uint32_t n_old_bins, as_bin* new_bins, uint32_t n_new_bins) { int n_populated = 0; bool not_just_created[n_new_bins]; memset(not_just_created, 0, sizeof(not_just_created)); // Maximum number of sindexes which can be changed in one transaction is // 2 * ns->sindex_cnt. SINDEX_GRLOCK(); SINDEX_BINS_SETUP(sbins, 2 * ns->sindex_cnt); as_sindex* si_arr[2 * ns->sindex_cnt]; int si_arr_index = 0; // Reserve matching SIs. for (int i = 0; i < n_old_bins; i++) { si_arr_index += as_sindex_arr_lookup_by_set_binid_lockfree(ns, set_name, old_bins[i].id, &si_arr[si_arr_index]); } for (int i = 0; i < n_new_bins; i++) { si_arr_index += as_sindex_arr_lookup_by_set_binid_lockfree(ns, set_name, new_bins[i].id, &si_arr[si_arr_index]); } // For every old bin, find the corresponding new bin (if any) and adjust the // secondary index if the bin was modified. If no corresponding new bin is // found, it means the old bin was deleted - also adjust the secondary index // accordingly. for (int32_t i_old = 0; i_old < (int32_t)n_old_bins; i_old++) { as_bin* b_old = &old_bins[i_old]; bool found = false; // Loop over new bins. Start at old bin index (if possible) and go down, // wrapping around to do the higher indexes last. This will find a match // (if any) very quickly - instantly, unless there were bins deleted. bool any_new = n_new_bins != 0; int32_t n_new_minus_1 = (int32_t)n_new_bins - 1; int32_t i_new = n_new_minus_1 < i_old ? n_new_minus_1 : i_old; while (any_new) { as_bin* b_new = &new_bins[i_new]; if (b_old->id == b_new->id) { if (as_bin_get_particle_type(b_old) != as_bin_get_particle_type(b_new) || b_old->particle != b_new->particle) { n_populated += as_sindex_sbins_populate( &sbins[n_populated], ns, set_name, b_old, b_new); } found = true; not_just_created[i_new] = true; break; } if (--i_new < 0 && (i_new = n_new_minus_1) <= i_old) { break; } if (i_new == i_old) { break; } } if (! found) { n_populated += as_sindex_sbins_from_bin(ns, set_name, b_old, &sbins[n_populated], AS_SINDEX_OP_DELETE); } } // Now find the new bins that are just-created bins. We've marked the others // in the loop above, so any left are just-created. for (uint32_t i_new = 0; i_new < n_new_bins; i_new++) { if (not_just_created[i_new]) { continue; } n_populated += as_sindex_sbins_from_bin(ns, set_name, &new_bins[i_new], &sbins[n_populated], AS_SINDEX_OP_INSERT); } SINDEX_GRUNLOCK(); if (n_populated != 0) { as_sindex_update_by_sbin(ns, set_name, sbins, n_populated, keyd); as_sindex_sbin_freeall(sbins, n_populated); } as_sindex_release_arr(si_arr, si_arr_index); return n_populated != 0; }
int as_msg_make_response_bufbuilder(as_record *r, as_storage_rd *rd, cf_buf_builder **bb_r, bool nobindata, char *nsname, bool use_sets, bool include_key, cf_vector *binlist) { // Sanity checks. Either rd should be there or nobindata and nsname should be present. if (!(rd || (nobindata && nsname))) { cf_detail(AS_PROTO, "Neither storage record nor nobindata is set. Skipping the record."); return 0; } // figure out the size of the entire buffer int set_name_len = 0; const char *set_name = NULL; int ns_len = rd ? strlen(rd->ns->name) : strlen(nsname); if (use_sets && as_index_get_set_id(r) != INVALID_SET_ID) { as_namespace *ns = NULL; if (rd) { ns = rd->ns; } else if (nsname) { ns = as_namespace_get_byname(nsname); } if (!ns) { cf_info(AS_PROTO, "Cannot get namespace, needed to get set information. Skipping record."); return -1; } set_name = as_index_get_set_name(r, ns); if (set_name) { set_name_len = strlen(set_name); } } uint8_t* key = NULL; uint32_t key_size = 0; if (include_key && as_index_is_flag_set(r, AS_INDEX_FLAG_KEY_STORED)) { if (! as_storage_record_get_key(rd)) { cf_info(AS_PROTO, "can't get key - skipping record"); return -1; } key = rd->key; key_size = rd->key_size; } uint16_t n_fields = 2; int msg_sz = sizeof(as_msg); msg_sz += sizeof(as_msg_field) + sizeof(cf_digest); msg_sz += sizeof(as_msg_field) + ns_len; if (set_name) { n_fields++; msg_sz += sizeof(as_msg_field) + set_name_len; } if (key) { n_fields++; msg_sz += sizeof(as_msg_field) + key_size; } int list_bins = 0; int in_use_bins = 0; if (rd) { in_use_bins = as_bin_inuse_count(rd); } if (nobindata == false) { if(binlist) { int binlist_sz = cf_vector_size(binlist); for(uint16_t i = 0; i < binlist_sz; i++) { char binname[AS_ID_BIN_SZ]; cf_vector_get(binlist, i, (void*)&binname); cf_debug(AS_PROTO, " Binname projected inside is |%s| \n", binname); as_bin *p_bin = as_bin_get (rd, (uint8_t*)binname, strlen(binname)); if (!p_bin) { cf_debug(AS_PROTO, "To be projected bin |%s| not found \n", binname); continue; } cf_debug(AS_PROTO, "Adding bin |%s| to projected bins |%s| \n", binname); list_bins++; msg_sz += sizeof(as_msg_op); msg_sz += rd->ns->single_bin ? 0 : strlen(binname); uint32_t psz; if (as_bin_is_hidden(p_bin)) { psz = 0; } else { as_particle_tobuf(p_bin, 0, &psz); // get size } msg_sz += psz; } } else { msg_sz += sizeof(as_msg_op) * in_use_bins; // the bin headers for (uint16_t i = 0; i < in_use_bins; i++) { as_bin *p_bin = &rd->bins[i]; msg_sz += rd->ns->single_bin ? 0 : strlen(as_bin_get_name_from_id(rd->ns, p_bin->id)); uint32_t psz; if (as_bin_is_hidden(p_bin)) { psz = 0; } else { as_particle_tobuf(p_bin, 0, &psz); // get size } msg_sz += psz; } } } uint8_t *b; cf_buf_builder_reserve(bb_r, msg_sz, &b); // set up the header uint8_t *buf = b; as_msg *msgp = (as_msg *) buf; msgp->header_sz = sizeof(as_msg); msgp->info1 = (nobindata ? AS_MSG_INFO1_GET_NOBINDATA : 0); msgp->info2 = 0; msgp->info3 = 0; msgp->unused = 0; msgp->result_code = 0; msgp->generation = r->generation; msgp->record_ttl = r->void_time; msgp->transaction_ttl = 0; msgp->n_fields = n_fields; if (rd) { if (binlist) msgp->n_ops = list_bins; else msgp->n_ops = in_use_bins; } else { msgp->n_ops = 0; } as_msg_swap_header(msgp); buf += sizeof(as_msg); as_msg_field *mf = (as_msg_field *) buf; mf->field_sz = sizeof(cf_digest) + 1; mf->type = AS_MSG_FIELD_TYPE_DIGEST_RIPE; if (rd) { memcpy(mf->data, &rd->keyd, sizeof(cf_digest)); } else { memcpy(mf->data, &r->key, sizeof(cf_digest)); } as_msg_swap_field(mf); buf += sizeof(as_msg_field) + sizeof(cf_digest); mf = (as_msg_field *) buf; mf->field_sz = ns_len + 1; mf->type = AS_MSG_FIELD_TYPE_NAMESPACE; if (rd) { memcpy(mf->data, rd->ns->name, ns_len); } else { memcpy(mf->data, nsname, ns_len); } as_msg_swap_field(mf); buf += sizeof(as_msg_field) + ns_len; if (set_name) { mf = (as_msg_field *) buf; mf->field_sz = set_name_len + 1; mf->type = AS_MSG_FIELD_TYPE_SET; memcpy(mf->data, set_name, set_name_len); as_msg_swap_field(mf); buf += sizeof(as_msg_field) + set_name_len; } if (key) { mf = (as_msg_field *) buf; mf->field_sz = key_size + 1; mf->type = AS_MSG_FIELD_TYPE_KEY; memcpy(mf->data, key, key_size); as_msg_swap_field(mf); buf += sizeof(as_msg_field) + key_size; } if (nobindata) { goto Out; } if(binlist) { int binlist_sz = cf_vector_size(binlist); for(uint16_t i = 0; i < binlist_sz; i++) { char binname[AS_ID_BIN_SZ]; cf_vector_get(binlist, i, (void*)&binname); cf_debug(AS_PROTO, " Binname projected inside is |%s| \n", binname); as_bin *p_bin = as_bin_get (rd, (uint8_t*)binname, strlen(binname)); if (!p_bin) // should it be checked before ??? continue; as_msg_op *op = (as_msg_op *)buf; buf += sizeof(as_msg_op); op->op = AS_MSG_OP_READ; op->name_sz = as_bin_memcpy_name(rd->ns, op->name, p_bin); buf += op->name_sz; // Since there are two variable bits, the size is everything after // the data bytes - and this is only the head, we're patching up // the rest in a minute. op->op_sz = 4 + op->name_sz; if (as_bin_inuse(p_bin)) { op->particle_type = as_particle_type_convert(as_bin_get_particle_type(p_bin)); op->version = as_bin_get_version(p_bin, rd->ns->single_bin); uint32_t psz = msg_sz - (buf - b); // size remaining in buffer, for safety if (as_bin_is_hidden(p_bin)) { op->particle_type = AS_PARTICLE_TYPE_NULL; psz = 0; } else { if (0 != as_particle_tobuf(p_bin, buf, &psz)) { cf_warning(AS_PROTO, "particle to buf: could not copy data!"); } } buf += psz; op->op_sz += psz; } else { cf_debug(AS_PROTO, "Whoops !! bin not in use"); op->particle_type = AS_PARTICLE_TYPE_NULL; } as_msg_swap_op(op); } } else { // over all bins, copy into the buffer for (uint16_t i = 0; i < in_use_bins; i++) { as_msg_op *op = (as_msg_op *)buf; buf += sizeof(as_msg_op); op->op = AS_MSG_OP_READ; op->name_sz = as_bin_memcpy_name(rd->ns, op->name, &rd->bins[i]); buf += op->name_sz; // Since there are two variable bits, the size is everything after // the data bytes - and this is only the head, we're patching up // the rest in a minute. op->op_sz = 4 + op->name_sz; if (as_bin_inuse(&rd->bins[i])) { op->particle_type = as_particle_type_convert(as_bin_get_particle_type(&rd->bins[i])); op->version = as_bin_get_version(&rd->bins[i], rd->ns->single_bin); uint32_t psz = msg_sz - (buf - b); // size remaining in buffer, for safety if (as_bin_is_hidden(&rd->bins[i])) { op->particle_type = AS_PARTICLE_TYPE_NULL; psz = 0; } else { if (0 != as_particle_tobuf(&rd->bins[i], buf, &psz)) { cf_warning(AS_PROTO, "particle to buf: could not copy data!"); } } buf += psz; op->op_sz += psz; } else { op->particle_type = AS_PARTICLE_TYPE_NULL; } as_msg_swap_op(op); } } Out: return(0); }
cl_msg * as_msg_make_response_msg( uint32_t result_code, uint32_t generation, uint32_t void_time, as_msg_op **ops, as_bin **bins, uint16_t bin_count, as_namespace *ns, cl_msg *msgp_in, size_t *msg_sz_in, uint64_t trid, const char *setname) { int setname_len = 0; // figure out the size of the entire buffer int msg_sz = sizeof(cl_msg); msg_sz += sizeof(as_msg_op) * bin_count; // the bin headers for (uint16_t i = 0; i < bin_count; i++) { if (bins[i]) { msg_sz += ns->single_bin ? 0 : strlen(as_bin_get_name_from_id(ns, bins[i]->id)); uint32_t psz; if (as_bin_is_hidden(bins[i])) { psz = 0; } else { bool tojson = (as_bin_get_particle_type(bins[i]) == AS_PARTICLE_TYPE_LUA_BLOB); _as_particle_tobuf(bins[i], 0, &psz, tojson); // get size } msg_sz += psz; } else if (ops[i]) // no bin, only op, no particle size msg_sz += ops[i]->name_sz; else cf_warning(AS_PROTO, "internal error!"); } //If a transaction-id is sent by the client, we should send it back in a field if (trid != 0) { msg_sz += (sizeof(as_msg_field) + sizeof(trid)); } // If setname is present, we will send it as a field. Account for its space overhead. if (setname != 0) { setname_len = strlen(setname); msg_sz += (sizeof(as_msg_field) + setname_len); } // most cases are small messages - try to stack alloc if we can byte *b; if ((0 == msgp_in) || (*msg_sz_in < msg_sz)) { b = cf_malloc(msg_sz); if (!b) return(0); } else { b = (byte *) msgp_in; } *msg_sz_in = msg_sz; // set up the header byte *buf = b; // current buffer pointer cl_msg *msgp = (cl_msg *) buf; msgp->proto.version = PROTO_VERSION; msgp->proto.type = PROTO_TYPE_AS_MSG; msgp->proto.sz = msg_sz - sizeof(as_proto); as_proto_swap(&msgp->proto); as_msg *m = &msgp->msg; m->header_sz = sizeof(as_msg); m->info1 = 0; m->info2 = 0; m->info3 = 0; m->unused = 0; m->result_code = result_code; m->generation = generation; m->record_ttl = void_time; m->transaction_ttl = 0; m->n_ops = bin_count; m->n_fields = 0; // Count the number of fields that we are going to send back if (trid != 0) { m->n_fields++; } if (setname != NULL) { m->n_fields++; } as_msg_swap_header(m); buf += sizeof(cl_msg); //If we have to send back the transaction-id, we have fields to send back if (trid != 0) { as_msg_field *trfield = (as_msg_field *) buf; //Allow space for the message field header buf += sizeof(as_msg_field); //Fill the field header trfield->type = AS_MSG_FIELD_TYPE_TRID; //Copy the transaction-id as field data in network byte order (big-endian) uint64_t trid_nbo = __cpu_to_be64(trid); trfield->field_sz = sizeof(trid_nbo); memcpy(trfield->data, &trid_nbo, sizeof(trid_nbo)); as_msg_swap_field(trfield); //Allow space for the message field data buf += sizeof(trid_nbo); } // If we have to send back the setname, we have fields to send back if (setname != NULL) { as_msg_field *trfield = (as_msg_field *) buf; // Allow space for the message field header buf += sizeof(as_msg_field); // Fill the field header trfield->type = AS_MSG_FIELD_TYPE_SET; trfield->field_sz = setname_len + 1; memcpy(trfield->data, setname, setname_len); as_msg_swap_field(trfield); // Allow space for the message field data buf += setname_len; } // over all bins, copy into the buffer for (uint16_t i = 0; i < bin_count; i++) { as_msg_op *op = (as_msg_op *)buf; buf += sizeof(as_msg_op); op->op = AS_MSG_OP_READ; if (bins[i]) { op->version = as_bin_get_version(bins[i], ns->single_bin); op->name_sz = as_bin_memcpy_name(ns, op->name, bins[i]); } else { op->version = 0; memcpy(op->name, ops[i]->name, ops[i]->name_sz); op->name_sz = ops[i]->name_sz; } buf += op->name_sz; // cf_detail(AS_PROTO, "make response: bin %d %s : version %d",i,bins[i]->name,op->version); // Since there are two variable bits, the size is everything after the // data bytes - and this is only the head, we're patching up the rest // in a minute. op->op_sz = 4 + op->name_sz; if (bins[i] && as_bin_inuse(bins[i])) { op->particle_type = as_particle_type_convert(as_bin_get_particle_type(bins[i])); uint32_t psz = msg_sz - (buf - b); // size remaining in buffer, for safety if (as_bin_is_hidden(bins[i])) { op->particle_type = AS_PARTICLE_TYPE_NULL; psz = 0; // packet of size NULL } else { bool tojson = (as_bin_get_particle_type(bins[i]) == AS_PARTICLE_TYPE_LUA_BLOB); if (0 != _as_particle_tobuf(bins[i], buf, &psz, tojson)) { cf_warning(AS_PROTO, "particle to buf: could not copy data!"); } } buf += psz; op->op_sz += psz; } else { op->particle_type = AS_PARTICLE_TYPE_NULL; } as_msg_swap_op(op); } return((cl_msg *) b); }
/* as_particle_set * Set the contents of a particle, which safely destroys the old particle */ as_particle * as_particle_frombuf(as_bin *b, as_particle_type type, byte *buf, uint32_t sz, uint8_t *stack_particle, bool data_in_memory) { #ifdef EXTRA_CHECKS // check the incoming type if (type < AS_PARTICLE_TYPE_NULL || type >= AS_PARTICLE_TYPE_MAX) { cf_info(AS_PARTICLE, "particle set: bad particle type %d, error", (int)type); return(NULL); } #endif as_particle *retval = 0; if (data_in_memory) { // we have to deal with these cases // current type is integer, new type is integer // current type is not integer, new type is integer // current type is integer, new type is not integer // current type is not integer, new type is not integer if (as_bin_is_integer(b)) { if (type == AS_PARTICLE_TYPE_INTEGER) { // current type is integer, new type is integer // just copy the new integer over the existing one. return (g_particle_setter_table[type](&b->iparticle, type, buf, sz, data_in_memory)); } else { // current type is integer, new type is not integer // make this the same case as current type is not integer, new type is not integer // cleanup the integer and allocate a pointer. b->particle = 0; } } else if (as_bin_inuse(b)) { // if it's a completely new type, destruct the old one and create a new one uint8_t bin_particle_type = as_bin_get_particle_type(b); if (type != bin_particle_type) { g_particle_destructor_table[bin_particle_type](b->particle); b->particle = 0; } } else { b->particle = 0; } } switch (type) { case AS_PARTICLE_TYPE_INTEGER: // current type is not integer, new type is integer as_bin_state_set(b, AS_BIN_STATE_INUSE_INTEGER); // use the iparticle embedded in the bin retval = g_particle_setter_table[type](&b->iparticle, type, buf, sz, data_in_memory); break; case AS_PARTICLE_TYPE_NULL: // special case, used to free old particle w/o setting new one break; default: // current type is not integer, new type is not integer if (! data_in_memory) { b->particle = (as_particle *)stack_particle; } if (as_particle_type_hidden(type)) { as_bin_state_set(b, AS_BIN_STATE_INUSE_HIDDEN); } else { as_bin_state_set(b, AS_BIN_STATE_INUSE_OTHER); } b->particle = g_particle_setter_table[type](b->particle, type, buf, sz, data_in_memory); retval = b->particle; break; } return(retval); }