void asval_to_clobject(as_val * val, cl_object * obj)
{
switch(val->type) {
case AS_NIL: {
citrusleaf_object_init_null(obj);
break;
}
case AS_INTEGER: {
as_integer * v = as_integer_fromval(val);
citrusleaf_object_init_int(obj, as_integer_toint(v));
break;
}
case AS_STRING: {
as_string * v = as_string_fromval(val);
citrusleaf_object_init_str(obj, as_string_get(v));
break;
}
case AS_BYTES: {
as_bytes * v = as_bytes_fromval(val);
citrusleaf_object_init_blob2(obj, v->value, v->size, (cl_type)v->type);
break;
}
case AS_LIST:{
as_buffer buffer;
as_buffer_init(&buffer);
as_serializer ser;
as_msgpack_init(&ser);
as_serializer_serialize(&ser, val, &buffer);
as_serializer_destroy(&ser);
citrusleaf_object_init_blob_handoff(obj, buffer.data, buffer.size, CL_LIST);
break;
}
case AS_MAP: {
as_buffer buffer;
as_buffer_init(&buffer);
as_serializer ser;
as_msgpack_init(&ser);
as_serializer_serialize(&ser, val, &buffer);
as_serializer_destroy(&ser);
citrusleaf_object_init_blob_handoff(obj, buffer.data, buffer.size, CL_MAP);
break;
}
default: {
// raise an error
break;
}
}
}
size_t
as_command_value_size(as_val* val, as_buffer* buffer)
{
switch (val->type) {
case AS_NIL: {
return 0;
}
case AS_INTEGER: {
return 8;
}
case AS_STRING: {
as_string* v = as_string_fromval(val);
return as_string_len(v);
}
case AS_BYTES: {
as_bytes* v = as_bytes_fromval(val);
return v->size;
}
case AS_LIST:
case AS_MAP: {
as_serializer ser;
as_msgpack_init(&ser);
as_serializer_serialize(&ser, val, buffer);
as_serializer_destroy(&ser);
return buffer->size;
}
default: {
return 0;
}
}
}
static size_t
as_scan_command_size(const as_scan* scan, uint16_t* fields, as_buffer* argbuffer)
{
// Build Command. It's okay to share command across threads because scan does not have retries.
// If retries were allowed, the timeout field in the command would change on retry which
// would conflict with other threads.
size_t size = AS_HEADER_SIZE;
uint16_t n_fields = 0;
if (scan->ns) {
size += as_command_string_field_size(scan->ns);
n_fields++;
}
if (scan->set) {
size += as_command_string_field_size(scan->set);
n_fields++;
}
// Estimate scan options size.
size += as_command_field_size(2);
n_fields++;
// Estimate taskId size.
size += as_command_field_size(8);
n_fields++;
// Estimate background function size.
as_buffer_init(argbuffer);
if (scan->apply_each.function[0]) {
size += as_command_field_size(1);
size += as_command_string_field_size(scan->apply_each.module);
size += as_command_string_field_size(scan->apply_each.function);
if (scan->apply_each.arglist) {
// If the query has a udf w/ arglist, then serialize it.
as_serializer ser;
as_msgpack_init(&ser);
as_serializer_serialize(&ser, (as_val*)scan->apply_each.arglist, argbuffer);
as_serializer_destroy(&ser);
}
size += as_command_field_size(argbuffer->size);
n_fields += 4;
}
// Estimate size for selected bin names.
if (scan->select.size > 0) {
for (uint16_t i = 0; i < scan->select.size; i++) {
size += as_command_string_operation_size(scan->select.entries[i]);
}
}
*fields = n_fields;
return size;
}
uint32_t
map_size_from_asval(const as_val *val)
{
as_serializer s;
as_msgpack_init(&s);
uint32_t size = as_serializer_serialize_getsize(&s, (as_val *)val);
as_serializer_destroy(&s);
return (uint32_t)sizeof(map_mem) + size;
}
uint32_t
map_asval_wire_size(const as_val *val)
{
as_serializer s;
as_msgpack_init(&s);
uint32_t size = as_serializer_serialize_getsize(&s, (as_val *)val);
as_serializer_destroy(&s);
return size;
}
uint32_t
map_asval_to_wire(const as_val *val, uint8_t *wire)
{
as_serializer s;
as_msgpack_init(&s);
uint32_t size = as_serializer_serialize_presized(&s, val, wire);
as_serializer_destroy(&s);
return size;
}
static int mod_lua_map_nbytes(lua_State * l) {
as_map * map = mod_lua_checkmap(l, 1);
uint32_t nbytes = 0;
if ( map ) {
as_serializer s;
as_msgpack_init(&s);
nbytes = as_serializer_serialize_getsize(&s, (as_val *) map);
as_serializer_destroy(&s);
}
lua_pushinteger(l, nbytes);
return 1;
}
/**
* Get a value for a bin of with the given key.
*/
static as_val * scan_response_get(const as_rec * rec, const char * name) {
as_val * v = NULL;
as_serializer ser;
as_msgpack_init(&ser);
cl_scan_response_rec * r = (cl_scan_response_rec *) rec;
for (int i = 0; i < r->n_bins; i++) {
if (!strcmp(r->bins[i].bin_name, name)) {
v = citrusleaf_udf_bin_to_val(&ser, &r->bins[i]);
break;
}
}
as_serializer_destroy(&ser);
return v;
}
// Creates a message, internally calling cl_compile to pass to the server
static int scan_compile(const cl_scan * scan, uint8_t ** buf_r, size_t * buf_sz_r) {
if (!scan) return CITRUSLEAF_FAIL_CLIENT;
// Prepare udf call to send to the server
as_call call;
as_serializer ser;
as_buffer argbuffer;
as_buffer_init(&argbuffer);
if ( scan->udf.type != CL_SCAN_UDF_NONE) {
as_string file;
as_string_init(&file, (char *) scan->udf.filename, true /*ismalloc*/);
as_string func;
as_string_init(&func, (char *) scan->udf.function, true /*ismalloc*/);
if (scan->udf.arglist != NULL) {
/**
* If the query has a udf w/ arglist,
* then serialize it.
*/
as_msgpack_init(&ser);
as_serializer_serialize(&ser, (as_val *) scan->udf.arglist, &argbuffer);
}
call.file = &file;
call.func = &func;
call.args = &argbuffer;
}
// Prepare to send scan parameters
cl_scan_param_field scan_param_field;
cl_scan_params params = scan->params;
scan_param_field.scan_pct = params.pct > 100 ? 100 : params.pct;
scan_param_field.byte1 = (params.priority << 4) | (params.fail_on_cluster_change << 3);
// Prepare the msg type to be sent
uint info;
info = CL_MSG_INFO1_READ;
// Pass on to the cl_compile to create the msg
cl_compile(info /*info1*/, 0, 0, scan->ns /*namespace*/, scan->setname /*setname*/, 0 /*key*/, 0/*digest*/, NULL /*bins*/, 0/*op*/, 0/*operations*/, 0/*n_values*/, buf_r, buf_sz_r, 0 /*w_p*/, NULL /*d_ret*/, scan->job_id, &scan_param_field, scan->udf.type != CL_SCAN_UDF_NONE ? &call : NULL/*udf call*/, scan->udf.type);
if (scan->udf.arglist) {
as_serializer_destroy(&ser);
}
as_buffer_destroy(&argbuffer);
return CITRUSLEAF_OK;
}
void
map_from_asval(const as_val *val, as_particle **pp)
{
map_mem *p_map_mem = (map_mem *)*pp;
as_serializer s;
as_msgpack_init(&s);
uint32_t size = as_serializer_serialize_presized(&s, val, p_map_mem->data);
p_map_mem->type = AS_PARTICLE_TYPE_MAP;
p_map_mem->sz = size;
as_serializer_destroy(&s);
}
static void
as_command_parse_value(uint8_t* p, uint8_t type, uint32_t value_size, as_val** value)
{
// Allocate values on heap.
switch (type) {
case AS_BYTES_UNDEF: {
*value = (as_val*)&as_nil;
break;
}
case AS_BYTES_INTEGER: {
int64_t v = 0;
as_command_bytes_to_int(p, value_size, &v);
*value = (as_val*)as_integer_new(v);
break;
}
case AS_BYTES_DOUBLE: {
double v = cf_swap_from_big_float64(*(double*)p);
*value = (as_val*)as_double_new(v);
break;
}
case AS_BYTES_STRING: {
char* v = malloc(value_size + 1);
memcpy(v, p, value_size);
v[value_size] = 0;
*value = (as_val*)as_string_new_wlen(v, value_size, true);
break;
}
case AS_BYTES_LIST:
case AS_BYTES_MAP: {
as_buffer buffer;
buffer.data = p;
buffer.size = value_size;
as_serializer ser;
as_msgpack_init(&ser);
as_serializer_deserialize(&ser, &buffer, value);
as_serializer_destroy(&ser);
break;
}
default: {
void* v = malloc(value_size);
memcpy(v, p, value_size);
*value = (as_val*)as_bytes_new_wrap(v, value_size, true);
break;
}
}
}
size_t
as_command_value_size(as_val* val, as_buffer* buffer)
{
switch (val->type) {
case AS_NIL: {
return 0;
}
case AS_INTEGER: {
return 8;
}
case AS_DOUBLE: {
return 8;
}
case AS_STRING: {
as_string* v = as_string_fromval(val);
return as_string_len(v);
}
case AS_GEOJSON: {
as_geojson* v = as_geojson_fromval(val);
return
1 + // as_particle_geojson_mem::flags
2 + // as_particle_geojson_mem::ncells
(0 * 8) + // <placeholder-cellids> EMPTY!
as_geojson_len(v);
}
case AS_BYTES: {
as_bytes* v = as_bytes_fromval(val);
return v->size;
}
case AS_LIST:
case AS_MAP: {
as_serializer ser;
as_msgpack_init(&ser);
as_serializer_serialize(&ser, val, buffer);
as_serializer_destroy(&ser);
return buffer->size;
}
default: {
return 0;
}
}
}
as_val *
map_to_asval(const as_particle *p)
{
map_mem *p_map_mem = (map_mem *)p;
as_buffer buf;
as_buffer_init(&buf);
buf.data = p_map_mem->data;
buf.capacity = p_map_mem->sz;
buf.size = p_map_mem->sz;
as_serializer s;
as_msgpack_init(&s);
as_val *val;
as_serializer_deserialize(&s, &buf, &val);
as_serializer_destroy(&s);
return val;
}
uint8_t*
as_command_parse_bins(as_record* rec, uint8_t* p, uint32_t n_bins, bool deserialize)
{
as_bin* bin = rec->bins.entries;
// Parse bins
for (uint32_t i = 0; i < n_bins; i++, bin++) {
uint32_t op_size = cf_swap_from_be32(*(uint32_t*)p);
p += 5;
uint8_t type = *p;
p += 2;
uint8_t name_size = *p++;
uint8_t name_len = (name_size <= AS_BIN_NAME_MAX_LEN)? name_size : AS_BIN_NAME_MAX_LEN;
memcpy(bin->name, p, name_len);
bin->name[name_len] = 0;
p += name_size;
uint32_t value_size = (op_size - (name_size + 4));
switch (type) {
case AS_BYTES_UNDEF: {
bin->valuep = (as_bin_value*)&as_nil;
break;
}
case AS_BYTES_INTEGER: {
int64_t value;
if (as_command_bytes_to_int(p, value_size, &value) == 0) {
as_integer_init((as_integer*)&bin->value, value);
bin->valuep = &bin->value;
}
break;
}
case AS_BYTES_DOUBLE: {
double value = cf_swap_from_big_float64(*(double*)p);
as_double_init((as_double*)&bin->value, value);
bin->valuep = &bin->value;
break;
}
case AS_BYTES_STRING: {
char* value = malloc(value_size + 1);
memcpy(value, p, value_size);
value[value_size] = 0;
as_string_init_wlen((as_string*)&bin->value, (char*)value, value_size, true);
bin->valuep = &bin->value;
break;
}
case AS_BYTES_LIST:
case AS_BYTES_MAP: {
if (deserialize) {
as_val* value = 0;
as_buffer buffer;
buffer.data = p;
buffer.size = value_size;
as_serializer ser;
as_msgpack_init(&ser);
as_serializer_deserialize(&ser, &buffer, &value);
as_serializer_destroy(&ser);
bin->valuep = (as_bin_value*)value;
}
else {
void* value = malloc(value_size);
memcpy(value, p, value_size);
as_bytes_init_wrap((as_bytes*)&bin->value, value, value_size, true);
bin->value.bytes.type = (as_bytes_type)type;
bin->valuep = &bin->value;
}
break;
}
default: {
void* value = malloc(value_size);
memcpy(value, p, value_size);
as_bytes_init_wrap((as_bytes*)&bin->value, value, value_size, true);
bin->value.bytes.type = (as_bytes_type)type;
bin->valuep = &bin->value;
break;
}
}
rec->bins.size++;
p += value_size;
}
return p;
}
void clbin_to_asval(cl_bin * bin, as_serializer * ser, as_val ** val)
{
if ( val == NULL ) return;
switch( bin->object.type ) {
case CL_NULL :{
*val = (as_val *) &as_nil;
break;
}
case CL_INT : {
*val = (as_val *) as_integer_new(bin->object.u.i64);
break;
}
case CL_STR : {
// steal the pointer from the object into the val
*val = (as_val *) as_string_new(strdup(bin->object.u.str), true /*ismalloc*/);
// TODO: re-evaluate the follow zero-copy for strings from cl_bins
// *val = (as_val *) as_string_new(bin->object.u.str, true /*ismalloc*/);
// bin->object.free = NULL;
break;
}
case CL_LIST :
case CL_MAP : {
// use a temporary buffer, which doesn't need to be destroyed
as_buffer buf = {
.capacity = (uint32_t) bin->object.sz,
.size = (uint32_t) bin->object.sz,
.data = (uint8_t *) bin->object.u.blob
};
// print_buffer(&buf);
as_serializer_deserialize(ser, &buf, val);
break;
}
case CL_BLOB:
case CL_JAVA_BLOB:
case CL_CSHARP_BLOB:
case CL_PYTHON_BLOB:
case CL_RUBY_BLOB:
case CL_ERLANG_BLOB:
default : {
*val = NULL;
uint8_t * raw = malloc(sizeof(bin->object.sz));
memcpy(raw, bin->object.u.blob, bin->object.sz);
as_bytes * b = as_bytes_new_wrap(raw, (uint32_t)bin->object.sz, true /*ismalloc*/);
b->type = (as_bytes_type)bin->object.type;
*val = (as_val *) b;
break;
}
}
}
void clbin_to_asrecord(cl_bin * bin, as_record * r)
{
switch(bin->object.type) {
case CL_NULL: {
as_record_set_nil(r, bin->bin_name);
break;
}
case CL_INT: {
as_record_set_int64(r, bin->bin_name, bin->object.u.i64);
break;
}
case CL_STR: {
as_record_set_strp(r, bin->bin_name, bin->object.u.str, true);
// the following completes the handoff of the value.
bin->object.free = NULL;
break;
}
case CL_LIST:
case CL_MAP: {
as_val * val = NULL;
as_buffer buffer;
buffer.data = (uint8_t *) bin->object.u.blob;
buffer.size = (uint32_t)bin->object.sz;
as_serializer ser;
as_msgpack_init(&ser);
as_serializer_deserialize(&ser, &buffer, &val);
as_serializer_destroy(&ser);
as_record_set(r, bin->bin_name, (as_bin_value *) val);
break;
}
default: {
as_record_set_rawp(r, bin->bin_name, bin->object.u.blob, (uint32_t)bin->object.sz, true);
// the following completes the handoff of the value.
bin->object.free = NULL;
break;
}
}
}
void clbins_to_asrecord(cl_bin * bins, uint32_t nbins, as_record * r)
{
uint32_t n = nbins < r->bins.capacity ? nbins : r->bins.capacity;
for ( int i = 0; i < n; i++ ) {
clbin_to_asrecord(&bins[i], r);
}
}
/**
* Call with AS_OPERATIONS_CDT_OP only.
*/
static bool as_operations_cdt_op(as_operations *ops, const as_bin_name name, as_cdt_optype op, size_t n, ...)
{
if (op >= cdt_op_table_size) {
return false;
}
const cdt_op_table_entry *entry = &cdt_op_table[op];
if (n < entry->count - entry->opt_args || n > entry->count) {
return false;
}
va_list vl;
if (n > 0) {
va_start(vl, n);
}
as_arraylist list;
as_arraylist_inita(&list, (uint32_t)n + 1); // +1 to avoid alloca(0) undefined behavior
for (size_t i = 0; i < n; i++) {
as_cdt_paramtype type = entry->args[i];
switch (type) {
case AS_CDT_PARAM_PAYLOAD: {
as_val *arg = va_arg(vl, as_val *);
if (as_arraylist_append(&list, arg) != AS_ARRAYLIST_OK) {
va_end(vl);
as_arraylist_destroy(&list);
return false;
}
break;
}
case AS_CDT_PARAM_COUNT: {
uint64_t arg = va_arg(vl, uint64_t);
if (as_arraylist_append(&list, (as_val *)as_integer_new(arg)) != AS_ARRAYLIST_OK) {
va_end(vl);
as_arraylist_destroy(&list);
return false;
}
break;
}
case AS_CDT_PARAM_INDEX: {
int64_t arg = va_arg(vl, int64_t);
if (as_arraylist_append(&list, (as_val *)as_integer_new(arg)) != AS_ARRAYLIST_OK) {
va_end(vl);
as_arraylist_destroy(&list);
return false;
}
break;
}
default:
break;
}
}
if (n > 0) {
va_end(vl);
}
as_serializer ser;
as_msgpack_init(&ser);
uint32_t list_size = as_serializer_serialize_getsize(&ser, (as_val *) &list);
as_bytes *bytes = as_bytes_new(sizeof(uint16_t) + list_size);
uint8_t *list_write = as_bytes_get(bytes);
uint16_t *list_write_op = (uint16_t *)list_write;
*list_write_op = cf_swap_to_be16(op);
list_write += sizeof(uint16_t);
as_serializer_serialize_presized(&ser, (const as_val *) &list, list_write);
as_serializer_destroy(&ser);
as_arraylist_destroy(&list);
bytes->size = bytes->capacity;
// as_bytes->type default to AS_BYTES_BLOB
if (entry->rw_type == CDT_RW_TYPE_MODIFY) {
return as_operations_add_cdt_modify(ops, name, (as_bin_value *) bytes);
}
return as_operations_add_cdt_read(ops, name, (as_bin_value *) bytes);
}
static void
as_command_parse_value(uint8_t* p, uint8_t type, uint32_t value_size, as_val** value)
{
// Allocate values on heap.
switch (type) {
case AS_BYTES_UNDEF: {
*value = (as_val*)&as_nil;
break;
}
case AS_BYTES_INTEGER: {
int64_t v = 0;
as_command_bytes_to_int(p, value_size, &v);
*value = (as_val*)as_integer_new(v);
break;
}
case AS_BYTES_DOUBLE: {
double v = cf_swap_from_big_float64(*(double*)p);
*value = (as_val*)as_double_new(v);
break;
}
case AS_BYTES_STRING: {
char* v = cf_malloc(value_size + 1);
memcpy(v, p, value_size);
v[value_size] = 0;
*value = (as_val*)as_string_new_wlen(v, value_size, true);
break;
}
case AS_BYTES_GEOJSON: {
uint8_t * ptr = p;
// skip flags
ptr++;
// ncells
uint16_t ncells = cf_swap_from_be16(* (uint16_t *) ptr);
ptr += sizeof(uint16_t);
// skip any cells
ptr += sizeof(uint64_t) * ncells;
// Use the json bytes.
size_t jsonsz = value_size - 1 - 2 - (ncells * sizeof(uint64_t));
char* v = cf_malloc(jsonsz + 1);
memcpy(v, ptr, jsonsz);
v[jsonsz] = 0;
*value = (as_val*) as_geojson_new_wlen(v, jsonsz, true);
break;
}
case AS_BYTES_LIST:
case AS_BYTES_MAP: {
as_buffer buffer;
buffer.data = p;
buffer.size = value_size;
as_serializer ser;
as_msgpack_init(&ser);
as_serializer_deserialize(&ser, &buffer, value);
as_serializer_destroy(&ser);
break;
}
default: {
void* v = cf_malloc(value_size);
memcpy(v, p, value_size);
*value = (as_val*)as_bytes_new_wrap(v, value_size, true);
break;
}
}
}
/*
* Internal function: udf_aerospike_setbin
*
* Parameters:
* r -- udf_record to be manipulated
* bname -- name of the bin to be deleted
* val -- value to be updated with
*
* Return value:
* 0 on success
* -1 on failure
*
* Description:
* The function sets the bin with the name
* passed in as parameter to the value, passed as the third parameter.
* Before updating the bin, it is checked if the value can fit in the storage
*
* Synchronization : object lock acquired by the transaction thread executing UDF.
* Partition reservation takes place just before the transaction starts executing
* ( look for as_partition_reserve_udf in thr_tsvc.c )
*
* Callers:
* udf_aerospike__apply_update_atomic
* In this function, if it fails at the time of update, the record is set
* to rollback all the updates till this point. The case where it fails in
* rollback is not handled.
*
* Side Notes:
* i. write_to_device will be set to true on a successful bin update.
* If all the updates from udf_aerospike__apply_update_atomic (including this) are
* successful, the record will be written to disk and reopened so that the rest of
* sets of updates can be applied.
*
* ii. If put in sindex fails, we do not handle it.
*
* TODO make sure anything goes into setbin only if the bin value is
* changed
*/
static const int
udf_aerospike_setbin(udf_record * urecord, const char * bname, const as_val * val, bool is_hidden)
{
if (bname == NULL || bname[0] == 0 ) {
cf_warning(AS_UDF, "no bin name supplied");
return -1;
}
uint8_t type = as_val_type(val);
if (is_hidden &&
((type != AS_MAP) && (type != AS_LIST))) {
cf_warning(AS_UDF, "Hidden %d Type Not allowed", type);
return -3;
}
size_t blen = strlen(bname);
as_storage_rd * rd = urecord->rd;
as_transaction *tr = urecord->tr;
as_index_ref * index = urecord->r_ref;
as_bin * b = as_bin_get(rd, (byte *)bname, blen);
if ( !b && (blen > (AS_ID_BIN_SZ - 1 )
|| !as_bin_name_within_quota(rd->ns, (byte *)bname, blen)) ) {
// Can't write bin
cf_warning(AS_UDF, "bin name %s too big. Bin not added", bname);
return -1;
}
if ( !b ) {
// See if there's a free one, the hope is you will always find the bin because
// you have already allocated bin space before calling this function.
b = as_bin_create(index->r, rd, (byte *)bname, blen, 0);
if (!b) {
cf_warning(AS_UDF, "ERROR: udf_aerospike_setbin: as_bin_create: bin not found, something went really wrong!");
return -1;
}
}
SINDEX_BINS_SETUP(oldbin, 1);
SINDEX_BINS_SETUP(newbin, 1);
bool needs_sindex_delete = false;
bool needs_sindex_put = false;
bool needs_sindex_update = false;
bool has_sindex = as_sindex_ns_has_sindex(rd->ns);
if (has_sindex
&& (as_sindex_sbin_from_bin(rd->ns,
as_index_get_set_name(rd->r, rd->ns),
b, oldbin) == AS_SINDEX_OK)) {
needs_sindex_delete = true;
}
// we know we are doing an update now, make sure there is particle data,
// set to be 1 wblock size now @TODO!
uint32_t pbytes = 0;
int ret = 0;
if (!rd->ns->storage_data_in_memory && !urecord->particle_data) {
urecord->particle_data = cf_malloc(rd->ns->storage_write_block_size);
urecord->cur_particle_data = urecord->particle_data;
urecord->end_particle_data = urecord->particle_data + rd->ns->storage_write_block_size;
}
cf_detail(AS_UDF, "udf_setbin: bin %s type %d ", bname, type );
switch(type) {
case AS_STRING: {
as_string * v = as_string_fromval(val);
byte * s = (byte *) as_string_tostring(v);
size_t l = as_string_len(v);
// Save for later.
// cf_detail(AS_UDF, "udf_setbin: string: binname %s value is %s",bname,s);
if ( !as_storage_bin_can_fit(rd->ns, l) ) {
cf_warning(AS_UDF, "string: bin size too big");
ret = -1;
break;
}
if (rd->ns->storage_data_in_memory) {
as_particle_frombuf(b, AS_PARTICLE_TYPE_STRING, s, l, NULL, true);
} else {
pbytes = l + as_particle_get_base_size(AS_PARTICLE_TYPE_STRING);
if ((urecord->cur_particle_data + pbytes) < urecord->end_particle_data) {
as_particle_frombuf(b, AS_PARTICLE_TYPE_STRING, s, l,
urecord->cur_particle_data,
rd->ns->storage_data_in_memory);
urecord->cur_particle_data += pbytes;
} else {
cf_warning(AS_UDF, "string: bin data size too big: pbytes %d"
" pdata %p cur_part+pbytes %p pend %p", pbytes,
urecord->particle_data, urecord->cur_particle_data + pbytes,
urecord->end_particle_data);
ret = -1;
break;
}
}
break;
}
case AS_BYTES: {
as_bytes * v = as_bytes_fromval(val);
uint8_t * s = as_bytes_get(v);
size_t l = as_bytes_size(v);
if ( !as_storage_bin_can_fit(rd->ns, l) ) {
cf_warning(AS_UDF, "bytes: bin size too big");
ret = -1;
break;
}
if (rd->ns->storage_data_in_memory) {
as_particle_frombuf(b, AS_PARTICLE_TYPE_BLOB, s, l, NULL, true);
} else {
pbytes = l + as_particle_get_base_size(AS_PARTICLE_TYPE_BLOB);
if ((urecord->cur_particle_data + pbytes) < urecord->end_particle_data) {
as_particle_frombuf(b, AS_PARTICLE_TYPE_BLOB, s, l, urecord->cur_particle_data,
rd->ns->storage_data_in_memory);
urecord->cur_particle_data += pbytes;
} else {
cf_warning(AS_UDF, "bytes: bin data size too big pbytes %d"
" pdata %p cur_part+pbytes %p pend %p", pbytes,
urecord->particle_data, urecord->cur_particle_data + pbytes,
urecord->end_particle_data);
ret = -1;
break;
}
}
break;
}
case AS_BOOLEAN: {
as_boolean * v = as_boolean_fromval(val);
bool d = as_boolean_get(v);
int64_t i = __be64_to_cpup((void *)&d);
if ( !as_storage_bin_can_fit(rd->ns, 8) ) {
cf_warning(AS_UDF, "bool: bin size too big");
ret = -1;
break;
}
if (rd->ns->storage_data_in_memory) {
as_particle_frombuf(b, AS_PARTICLE_TYPE_INTEGER, (uint8_t *) &i, 8, NULL, true);
} else {
pbytes = 8 + as_particle_get_base_size(AS_PARTICLE_TYPE_INTEGER);
if ((urecord->cur_particle_data + pbytes) < urecord->end_particle_data) {
as_particle_frombuf(b, AS_PARTICLE_TYPE_INTEGER,
(uint8_t *) &i, 8,
urecord->cur_particle_data,
rd->ns->storage_data_in_memory);
urecord->cur_particle_data += pbytes;
} else {
cf_warning(AS_UDF, "bool: bin data size too big: pbytes %d %p %p %p",
pbytes, urecord->particle_data, urecord->cur_particle_data,
urecord->end_particle_data);
ret = -1;
break;
}
}
break;
}
case AS_INTEGER: {
as_integer * v = as_integer_fromval(val);
int64_t i = as_integer_get(v);
int64_t j = __be64_to_cpup((void *)&i);
if ( !as_storage_bin_can_fit(rd->ns, 8) ) {
cf_warning(AS_UDF, "int: bin size too big");
ret = -1;
break;
}
if (rd->ns->storage_data_in_memory) {
as_particle_frombuf(b, AS_PARTICLE_TYPE_INTEGER, (uint8_t *) &j, 8, NULL, true);
} else {
pbytes = 8 + as_particle_get_base_size(AS_PARTICLE_TYPE_INTEGER);
if ((urecord->cur_particle_data + pbytes) < urecord->end_particle_data) {
as_particle_frombuf(b, AS_PARTICLE_TYPE_INTEGER,
(uint8_t *) &j, 8, urecord->cur_particle_data,
rd->ns->storage_data_in_memory);
urecord->cur_particle_data += pbytes;
} else {
cf_warning(AS_UDF, "int: bin data size too big: pbytes %d %p %p %p",
pbytes, urecord->particle_data, urecord->cur_particle_data,
urecord->end_particle_data);
ret = -1;
break;
}
}
break;
}
// @LDT : Possibly include AS_LDT in this list. We need the LDT
// bins to be updated by LDT lua calls, and that path takes us thru here.
// However, we ALSO need to be able to set the particle type for the
// bins -- so that requires extra processing here to take the LDT flags
// and set the appropriate bin flags in the particle data.
case AS_MAP:
case AS_LIST: {
as_buffer buf;
as_buffer_init(&buf);
as_serializer s;
as_msgpack_init(&s);
int rsp = 0;
as_serializer_serialize(&s, (as_val *) val, &buf);
if ( !as_storage_bin_can_fit(rd->ns, buf.size) ) {
cf_warning(AS_UDF, "map-list: bin size too big");
ret = -1;
// Clean Up and jump out.
as_serializer_destroy(&s);
as_buffer_destroy(&buf);
break; // can't continue if value too big.
}
uint8_t ptype;
if(is_hidden) {
ptype = as_particle_type_convert_to_hidden(to_particle_type(type));
} else {
ptype = to_particle_type(type);
}
if (rd->ns->storage_data_in_memory) {
as_particle_frombuf(b, ptype, (uint8_t *) buf.data, buf.size, NULL, true);
}
else {
pbytes = buf.size + as_particle_get_base_size(ptype);
if ((urecord->cur_particle_data + pbytes) < urecord->end_particle_data) {
as_particle_frombuf(b, ptype, (uint8_t *) buf.data, buf.size,
urecord->cur_particle_data, rd->ns->storage_data_in_memory);
urecord->cur_particle_data += pbytes;
} else {
cf_warning(AS_UDF, "map-list: bin data size too big: pbytes %d %p %p %p",
pbytes, urecord->particle_data, urecord->cur_particle_data,
urecord->end_particle_data);
rsp = -1;
}
}
as_serializer_destroy(&s);
as_buffer_destroy(&buf);
if (rsp) {
ret = rsp;
break;
}
break;
}
default: {
cf_warning(AS_UDF, "unrecognized object type %d, skipping", as_val_type(val) );
break;
}
}
// If something fail bailout
if (ret) {
as_sindex_sbin_freeall(oldbin, 1);
as_sindex_sbin_freeall(newbin, 1);
return ret;
}
// Update sindex if required
if (has_sindex) {
if (as_sindex_sbin_from_bin(rd->ns,
as_index_get_set_name(rd->r, rd->ns), b, newbin) == AS_SINDEX_OK) {
if (!as_sindex_sbin_match(newbin, oldbin)) {
needs_sindex_put = true;
} else {
needs_sindex_update = true;
}
}
if (needs_sindex_update) {
tr->flag |= AS_TRANSACTION_FLAG_SINDEX_TOUCHED;
as_sindex_delete_by_sbin(rd->ns,
as_index_get_set_name(rd->r, rd->ns), 1, oldbin, rd);
as_sindex_put_by_sbin(rd->ns,
as_index_get_set_name(rd->r, rd->ns), 1, newbin, rd);
} else {
if (needs_sindex_delete) {
tr->flag |= AS_TRANSACTION_FLAG_SINDEX_TOUCHED;
as_sindex_delete_by_sbin(rd->ns,
as_index_get_set_name(rd->r, rd->ns), 1, oldbin, rd);
}
if (needs_sindex_put) {
tr->flag |= AS_TRANSACTION_FLAG_SINDEX_TOUCHED;
as_sindex_put_by_sbin(rd->ns,
as_index_get_set_name(rd->r, rd->ns), 1, newbin, rd);
}
}
as_sindex_sbin_freeall(oldbin, 1);
as_sindex_sbin_freeall(newbin, 1);
}
return ret;
} // end udf_aerospike_setbin()
/**
* Lookup a record by key, then apply the UDF.
*
* ~~~~~~~~~~{.c}
* as_key key;
* as_key_init(&key, "ns", "set", "key");
*
* as_arraylist args;
* as_arraylist_init(&args, 2, 0);
* as_arraylist_append_int64(&args, 1);
* as_arraylist_append_int64(&args, 2);
*
* as_val * res = NULL;
*
* if ( aerospike_key_apply(&as, &err, NULL, &key, "math", "add", &args, &res) != AEROSPIKE_OK ) {
* fprintf(stderr, "error(%d) %s at [%s:%d]", err.code, err.message, err.file, err.line);
* }
* else {
* as_val_destroy(res);
* }
*
* as_arraylist_destroy(&args);
* ~~~~~~~~~~
*
*
* @param as The aerospike instance to use for this operation.
* @param err The as_error to be populated if an error occurs.
* @param policy The policy to use for this operation. If NULL, then the default policy will be used.
* @param key The key of the record.
* @param module The module containing the function to execute.
* @param function The function to execute.
* @param arglist The arguments for the function.
* @param result The return value from the function.
*
* @return AEROSPIKE_OK if successful. Otherwise an error.
*/
as_status aerospike_key_apply(
aerospike * as, as_error * err, const as_policy_apply * policy,
const as_key * key,
const char * module, const char * function, as_list * arglist,
as_val ** result)
{
// we want to reset the error so, we have a clean state
as_error_reset(err);
// resolve policies
as_policy_apply p;
as_policy_apply_resolve(&p, &as->config.policies, policy);
cl_write_parameters wp;
cl_write_parameters_set_default(&wp);
wp.timeout_ms = p.timeout == UINT32_MAX ? 0 : p.timeout;
cl_object okey;
asval_to_clobject((as_val *) key->valuep, &okey);
as_serializer ser;
as_msgpack_init(&ser);
as_string file;
as_string_init(&file, (char *) module, true /*ismalloc*/);
as_string func;
as_string_init(&func, (char *) function, true /*ismalloc*/);
as_buffer args;
as_buffer_init(&args);
as_serializer_serialize(&ser, (as_val *) arglist, &args);
as_call call = {
.file = &file,
.func = &func,
.args = &args
};
uint64_t trid = 0;
cl_bin * bins = 0;
int n_bins = 0;
cl_rv rc = CITRUSLEAF_OK;
switch ( p.key ) {
case AS_POLICY_KEY_DIGEST: {
as_digest * digest = as_key_digest((as_key *) key);
rc = do_the_full_monte(
as->cluster, 0, CL_MSG_INFO2_WRITE, 0,
key->ns, key->set, 0, (cf_digest *) digest->value, &bins, CL_OP_WRITE, 0, &n_bins,
NULL, &wp, &trid, NULL, &call, NULL
);
break;
}
case AS_POLICY_KEY_SEND: {
cl_object okey;
asval_to_clobject((as_val *) key->valuep, &okey);
as_digest * digest = as_key_digest((as_key *) key);
rc = do_the_full_monte(
as->cluster, 0, CL_MSG_INFO2_WRITE, 0,
key->ns, key->set, &okey, (cf_digest*)digest->value, &bins, CL_OP_WRITE, 0, &n_bins,
NULL, &wp, &trid, NULL, &call, NULL
);
break;
}
default: {
// ERROR CASE
break;
}
}
as_buffer_destroy(&args);
if (! (rc == CITRUSLEAF_OK || rc == CITRUSLEAF_FAIL_UDF_BAD_RESPONSE)) {
as_error_fromrc(err, rc);
}
else {
// Begin processing the data returned from the server,
// IFF `result` argument is not NULL.
// The reason is if `result` is NULL, then it implies the end user
// does not care about the data returned from the server.
if ( n_bins == 1 ) {
cl_bin * bin = &bins[0];
if ( strcmp(bin->bin_name,"SUCCESS") == 0 ) {
if ( result ) {
as_val * val = NULL;
clbin_to_asval(bin, &ser, &val);
*result = val;
}
}
else if ( strcmp(bin->bin_name,"FAILURE") == 0 ) {
as_val * val = NULL;
clbin_to_asval(bin, &ser, &val);
if ( val->type == AS_STRING ) {
as_string * s = as_string_fromval(val);
as_error_update(err, AEROSPIKE_ERR_UDF, as_string_tostring(s));
}
else {
as_error_update(err, AEROSPIKE_ERR_SERVER, "unexpected failure bin type");
}
as_val_destroy(val);
}
else {
as_error_update(err, AEROSPIKE_ERR_SERVER, "unexpected bin name");
}
}
else {
as_error_update(err, AEROSPIKE_ERR_SERVER, "unexpected number of bins");
}
}
if ( bins ) {
citrusleaf_bins_free(bins, n_bins);
free(bins);
}
as_serializer_destroy(&ser);
return err->code;
}
int as_msg_make_response_bufbuilder(as_record *r, as_storage_rd *rd,
cf_buf_builder **bb_r, bool nobindata, char *nsname, bool use_sets,
bool include_key, bool skip_empty_records, cf_vector *binlist)
{
// Sanity checks. Either rd should be there or nobindata and nsname should be present.
if (!(rd || (nobindata && nsname))) {
cf_detail(AS_PROTO, "Neither storage record nor nobindata is set. Skipping the record.");
return 0;
}
// figure out the size of the entire buffer
int set_name_len = 0;
const char *set_name = NULL;
int ns_len = rd ? strlen(rd->ns->name) : strlen(nsname);
if (use_sets && as_index_get_set_id(r) != INVALID_SET_ID) {
as_namespace *ns = NULL;
if (rd) {
ns = rd->ns;
} else if (nsname) {
ns = as_namespace_get_byname(nsname);
}
if (!ns) {
cf_info(AS_PROTO, "Cannot get namespace, needed to get set information. Skipping record.");
return -1;
}
set_name = as_index_get_set_name(r, ns);
if (set_name) {
set_name_len = strlen(set_name);
}
}
uint8_t* key = NULL;
uint32_t key_size = 0;
if (include_key && as_index_is_flag_set(r, AS_INDEX_FLAG_KEY_STORED)) {
if (! as_storage_record_get_key(rd)) {
cf_info(AS_PROTO, "can't get key - skipping record");
return -1;
}
key = rd->key;
key_size = rd->key_size;
}
uint16_t n_fields = 2;
int msg_sz = sizeof(as_msg);
msg_sz += sizeof(as_msg_field) + sizeof(cf_digest);
msg_sz += sizeof(as_msg_field) + ns_len;
if (set_name) {
n_fields++;
msg_sz += sizeof(as_msg_field) + set_name_len;
}
if (key) {
n_fields++;
msg_sz += sizeof(as_msg_field) + key_size;
}
int list_bins = 0;
int num_bins = 0;
if (rd) {
num_bins = as_bin_inuse_count(rd);
}
if (binlist) {
num_bins = cf_vector_size(binlist);
}
as_val *ldtBinVal[num_bins];
as_particle_type ldtBinParticleType[num_bins];
if (nobindata == false) {
if (binlist) {
for(uint16_t i = 0; i < num_bins; i++) {
char binname[AS_ID_BIN_SZ];
cf_vector_get(binlist, i, (void*)&binname);
cf_debug(AS_PROTO, " Binname projected inside is |%s| \n", binname);
as_bin *p_bin = as_bin_get (rd, (uint8_t*)binname, strlen(binname));
if (!p_bin)
{
cf_debug(AS_PROTO, "To be projected bin |%s| not found \n", binname);
continue;
}
cf_debug(AS_PROTO, "Adding bin |%s| to projected bins |%s| \n", binname);
msg_sz += sizeof(as_msg_op);
msg_sz += rd->ns->single_bin ? 0 : strlen(binname);
uint32_t psz;
if (as_bin_is_hidden(p_bin)) {
ldtBinVal[list_bins] = as_llist_scan(rd->ns, rd->ns->partitions[as_partition_getid(rd->keyd)].sub_vp, rd, p_bin);
if (ldtBinVal[list_bins]) {
ldtBinParticleType[list_bins] = AS_PARTICLE_TYPE_HIDDEN_LIST;
}
as_serializer s;
as_msgpack_init(&s);
psz = as_serializer_serialize_getsize(&s, (as_val *)ldtBinVal[list_bins]);
as_serializer_destroy(&s);
} else {
as_particle_tobuf(p_bin, 0, &psz); // get size
}
list_bins++;
msg_sz += psz;
}
// Don't return an empty record.
if (skip_empty_records && list_bins == 0) {
return 0;
}
}
else {
msg_sz += sizeof(as_msg_op) * num_bins; // the bin headers
for (uint16_t i = 0; i < num_bins; i++) {
as_bin *p_bin = &rd->bins[i];
msg_sz += rd->ns->single_bin ? 0 : strlen(as_bin_get_name_from_id(rd->ns, p_bin->id));
uint32_t psz;
if (as_bin_is_hidden(p_bin)) {
ldtBinVal[list_bins] = as_llist_scan(rd->ns, rd->ns->partitions[as_partition_getid(rd->keyd)].sub_vp, rd, p_bin);
if (ldtBinVal[list_bins]) {
ldtBinParticleType[list_bins] = AS_PARTICLE_TYPE_HIDDEN_LIST;
}
as_serializer s;
as_msgpack_init(&s);
psz = as_serializer_serialize_getsize(&s, (as_val *)ldtBinVal[list_bins]);
as_serializer_destroy(&s);
} else {
as_particle_tobuf(p_bin, 0, &psz); // get size
}
msg_sz += psz;
list_bins++;
}
}
}
uint8_t *b;
cf_buf_builder_reserve(bb_r, msg_sz, &b);
// set up the header
uint8_t *buf = b;
as_msg *msgp = (as_msg *) buf;
msgp->header_sz = sizeof(as_msg);
msgp->info1 = (nobindata ? AS_MSG_INFO1_GET_NOBINDATA : 0);
msgp->info2 = 0;
msgp->info3 = 0;
msgp->unused = 0;
msgp->result_code = 0;
msgp->generation = r->generation;
msgp->record_ttl = r->void_time;
msgp->transaction_ttl = 0;
msgp->n_fields = n_fields;
if (rd) {
if (binlist)
msgp->n_ops = list_bins;
else
msgp->n_ops = num_bins;
} else {
msgp->n_ops = 0;
}
as_msg_swap_header(msgp);
buf += sizeof(as_msg);
as_msg_field *mf = (as_msg_field *) buf;
mf->field_sz = sizeof(cf_digest) + 1;
mf->type = AS_MSG_FIELD_TYPE_DIGEST_RIPE;
if (rd) {
memcpy(mf->data, &rd->keyd, sizeof(cf_digest));
} else {
memcpy(mf->data, &r->key, sizeof(cf_digest));
}
as_msg_swap_field(mf);
buf += sizeof(as_msg_field) + sizeof(cf_digest);
mf = (as_msg_field *) buf;
mf->field_sz = ns_len + 1;
mf->type = AS_MSG_FIELD_TYPE_NAMESPACE;
if (rd) {
memcpy(mf->data, rd->ns->name, ns_len);
} else {
memcpy(mf->data, nsname, ns_len);
}
as_msg_swap_field(mf);
buf += sizeof(as_msg_field) + ns_len;
if (set_name) {
mf = (as_msg_field *) buf;
mf->field_sz = set_name_len + 1;
mf->type = AS_MSG_FIELD_TYPE_SET;
memcpy(mf->data, set_name, set_name_len);
as_msg_swap_field(mf);
buf += sizeof(as_msg_field) + set_name_len;
}
if (key) {
mf = (as_msg_field *) buf;
mf->field_sz = key_size + 1;
mf->type = AS_MSG_FIELD_TYPE_KEY;
memcpy(mf->data, key, key_size);
as_msg_swap_field(mf);
buf += sizeof(as_msg_field) + key_size;
}
if (nobindata) {
goto Out;
}
list_bins = 0;
if (binlist) {
for (uint16_t i = 0; i < num_bins; i++) {
char binname[AS_ID_BIN_SZ];
cf_vector_get(binlist, i, (void*)&binname);
cf_debug(AS_PROTO, " Binname projected inside is |%s| \n", binname);
as_bin *p_bin = as_bin_get (rd, (uint8_t*)binname, strlen(binname));
if (!p_bin) // should it be checked before ???
continue;
as_msg_op *op = (as_msg_op *)buf;
buf += sizeof(as_msg_op);
op->op = AS_MSG_OP_READ;
op->name_sz = as_bin_memcpy_name(rd->ns, op->name, p_bin);
buf += op->name_sz;
// Since there are two variable bits, the size is everything after
// the data bytes - and this is only the head, we're patching up
// the rest in a minute.
op->op_sz = 4 + op->name_sz;
if (as_bin_inuse(p_bin)) {
op->particle_type = as_particle_type_convert(as_bin_get_particle_type(p_bin));
op->version = as_bin_get_version(p_bin, rd->ns->single_bin);
uint32_t psz = msg_sz - (buf - b); // size remaining in buffer, for safety
if (as_bin_is_hidden(p_bin)) {
if (!ldtBinVal[list_bins]) {
op->particle_type = AS_PARTICLE_TYPE_NULL;
psz = 0;
} else {
op->particle_type = ldtBinParticleType[list_bins];
as_buffer abuf;
as_buffer_init(&abuf);
as_serializer s;
as_msgpack_init(&s);
as_serializer_serialize(&s, (as_val *)ldtBinVal[list_bins], &abuf);
psz = abuf.size;
memcpy(buf, abuf.data, abuf.size);
as_serializer_destroy(&s);
as_buffer_destroy(&abuf);
as_val_destroy(ldtBinVal[list_bins]);
}
} else {
if (0 != as_particle_tobuf(p_bin, buf, &psz)) {
cf_warning(AS_PROTO, "particle to buf: could not copy data!");
}
}
list_bins++;
buf += psz;
op->op_sz += psz;
}
else {
cf_debug(AS_PROTO, "Whoops !! bin not in use");
op->particle_type = AS_PARTICLE_TYPE_NULL;
}
as_msg_swap_op(op);
}
}
else {
// over all bins, copy into the buffer
for (uint16_t i = 0; i < num_bins; i++) {
as_msg_op *op = (as_msg_op *)buf;
buf += sizeof(as_msg_op);
op->op = AS_MSG_OP_READ;
op->name_sz = as_bin_memcpy_name(rd->ns, op->name, &rd->bins[i]);
buf += op->name_sz;
// Since there are two variable bits, the size is everything after
// the data bytes - and this is only the head, we're patching up
// the rest in a minute.
op->op_sz = 4 + op->name_sz;
if (as_bin_inuse(&rd->bins[i])) {
op->particle_type = as_particle_type_convert(as_bin_get_particle_type(&rd->bins[i]));
op->version = as_bin_get_version(&rd->bins[i], rd->ns->single_bin);
uint32_t psz = msg_sz - (buf - b); // size remaining in buffer, for safety
if (as_bin_is_hidden(&rd->bins[i])) {
if (!ldtBinVal[list_bins]) {
op->particle_type = AS_PARTICLE_TYPE_NULL;
psz = 0;
} else {
op->particle_type = ldtBinParticleType[list_bins];
as_buffer abuf;
as_buffer_init(&abuf);
as_serializer s;
as_msgpack_init(&s);
as_serializer_serialize(&s, (as_val *)ldtBinVal[list_bins], &abuf);
psz = abuf.size;
memcpy(buf, abuf.data, abuf.size);
as_serializer_destroy(&s);
as_buffer_destroy(&abuf);
as_val_destroy(ldtBinVal[list_bins]);
}
} else {
if (0 != as_particle_tobuf(&rd->bins[i], buf, &psz)) {
cf_warning(AS_PROTO, "particle to buf: could not copy data!");
}
}
list_bins++;
buf += psz;
op->op_sz += psz;
}
else {
op->particle_type = AS_PARTICLE_TYPE_NULL;
}
as_msg_swap_op(op);
}
}
Out:
return(0);
}