uint32_t
blob_asval_to_wire(const as_val *val, uint8_t *wire)
{
as_bytes *bytes = as_bytes_fromval(val);
uint32_t size = as_bytes_size(bytes);
memcpy(wire, as_bytes_get(bytes), size);
return size;
}
void
blob_from_asval(const as_val *val, as_particle **pp)
{
blob_mem *p_blob_mem = (blob_mem *)*pp;
as_bytes *bytes = as_bytes_fromval(val);
p_blob_mem->type = (uint8_t)blob_bytes_type_to_particle_type(bytes->type);
p_blob_mem->sz = as_bytes_size(bytes);
memcpy(p_blob_mem->data, as_bytes_get(bytes), p_blob_mem->sz);
}
/*
* Internal function: udf_aerospike_setbin
*
* Parameters:
* r -- udf_record to be manipulated
* bname -- name of the bin to be deleted
* val -- value to be updated with
*
* Return value:
* 0 on success
* -1 on failure
*
* Description:
* The function sets the bin with the name
* passed in as parameter to the value, passed as the third parameter.
* Before updating the bin, it is checked if the value can fit in the storage
*
* Synchronization : object lock acquired by the transaction thread executing UDF.
* Partition reservation takes place just before the transaction starts executing
* ( look for as_partition_reserve_udf in thr_tsvc.c )
*
* Callers:
* udf_aerospike__apply_update_atomic
* In this function, if it fails at the time of update, the record is set
* to rollback all the updates till this point. The case where it fails in
* rollback is not handled.
*
* Side Notes:
* i. write_to_device will be set to true on a successful bin update.
* If all the updates from udf_aerospike__apply_update_atomic (including this) are
* successful, the record will be written to disk and reopened so that the rest of
* sets of updates can be applied.
*
* ii. If put in sindex fails, we do not handle it.
*
* TODO make sure anything goes into setbin only if the bin value is
* changed
*/
static const int
udf_aerospike_setbin(udf_record * urecord, const char * bname, const as_val * val, bool is_hidden)
{
if (bname == NULL || bname[0] == 0 ) {
cf_warning(AS_UDF, "no bin name supplied");
return -1;
}
uint8_t type = as_val_type(val);
if (is_hidden &&
((type != AS_MAP) && (type != AS_LIST))) {
cf_warning(AS_UDF, "Hidden %d Type Not allowed", type);
return -3;
}
size_t blen = strlen(bname);
as_storage_rd * rd = urecord->rd;
as_transaction *tr = urecord->tr;
as_index_ref * index = urecord->r_ref;
as_bin * b = as_bin_get(rd, (byte *)bname, blen);
if ( !b && (blen > (AS_ID_BIN_SZ - 1 )
|| !as_bin_name_within_quota(rd->ns, (byte *)bname, blen)) ) {
// Can't write bin
cf_warning(AS_UDF, "bin name %s too big. Bin not added", bname);
return -1;
}
if ( !b ) {
// See if there's a free one, the hope is you will always find the bin because
// you have already allocated bin space before calling this function.
b = as_bin_create(index->r, rd, (byte *)bname, blen, 0);
if (!b) {
cf_warning(AS_UDF, "ERROR: udf_aerospike_setbin: as_bin_create: bin not found, something went really wrong!");
return -1;
}
}
SINDEX_BINS_SETUP(oldbin, 1);
SINDEX_BINS_SETUP(newbin, 1);
bool needs_sindex_delete = false;
bool needs_sindex_put = false;
bool needs_sindex_update = false;
bool has_sindex = as_sindex_ns_has_sindex(rd->ns);
if (has_sindex
&& (as_sindex_sbin_from_bin(rd->ns,
as_index_get_set_name(rd->r, rd->ns),
b, oldbin) == AS_SINDEX_OK)) {
needs_sindex_delete = true;
}
// we know we are doing an update now, make sure there is particle data,
// set to be 1 wblock size now @TODO!
uint32_t pbytes = 0;
int ret = 0;
if (!rd->ns->storage_data_in_memory && !urecord->particle_data) {
urecord->particle_data = cf_malloc(rd->ns->storage_write_block_size);
urecord->cur_particle_data = urecord->particle_data;
urecord->end_particle_data = urecord->particle_data + rd->ns->storage_write_block_size;
}
cf_detail(AS_UDF, "udf_setbin: bin %s type %d ", bname, type );
switch(type) {
case AS_STRING: {
as_string * v = as_string_fromval(val);
byte * s = (byte *) as_string_tostring(v);
size_t l = as_string_len(v);
// Save for later.
// cf_detail(AS_UDF, "udf_setbin: string: binname %s value is %s",bname,s);
if ( !as_storage_bin_can_fit(rd->ns, l) ) {
cf_warning(AS_UDF, "string: bin size too big");
ret = -1;
break;
}
if (rd->ns->storage_data_in_memory) {
as_particle_frombuf(b, AS_PARTICLE_TYPE_STRING, s, l, NULL, true);
} else {
pbytes = l + as_particle_get_base_size(AS_PARTICLE_TYPE_STRING);
if ((urecord->cur_particle_data + pbytes) < urecord->end_particle_data) {
as_particle_frombuf(b, AS_PARTICLE_TYPE_STRING, s, l,
urecord->cur_particle_data,
rd->ns->storage_data_in_memory);
urecord->cur_particle_data += pbytes;
} else {
cf_warning(AS_UDF, "string: bin data size too big: pbytes %d"
" pdata %p cur_part+pbytes %p pend %p", pbytes,
urecord->particle_data, urecord->cur_particle_data + pbytes,
urecord->end_particle_data);
ret = -1;
break;
}
}
break;
}
case AS_BYTES: {
as_bytes * v = as_bytes_fromval(val);
uint8_t * s = as_bytes_get(v);
size_t l = as_bytes_size(v);
if ( !as_storage_bin_can_fit(rd->ns, l) ) {
cf_warning(AS_UDF, "bytes: bin size too big");
ret = -1;
break;
}
if (rd->ns->storage_data_in_memory) {
as_particle_frombuf(b, AS_PARTICLE_TYPE_BLOB, s, l, NULL, true);
} else {
pbytes = l + as_particle_get_base_size(AS_PARTICLE_TYPE_BLOB);
if ((urecord->cur_particle_data + pbytes) < urecord->end_particle_data) {
as_particle_frombuf(b, AS_PARTICLE_TYPE_BLOB, s, l, urecord->cur_particle_data,
rd->ns->storage_data_in_memory);
urecord->cur_particle_data += pbytes;
} else {
cf_warning(AS_UDF, "bytes: bin data size too big pbytes %d"
" pdata %p cur_part+pbytes %p pend %p", pbytes,
urecord->particle_data, urecord->cur_particle_data + pbytes,
urecord->end_particle_data);
ret = -1;
break;
}
}
break;
}
case AS_BOOLEAN: {
as_boolean * v = as_boolean_fromval(val);
bool d = as_boolean_get(v);
int64_t i = __be64_to_cpup((void *)&d);
if ( !as_storage_bin_can_fit(rd->ns, 8) ) {
cf_warning(AS_UDF, "bool: bin size too big");
ret = -1;
break;
}
if (rd->ns->storage_data_in_memory) {
as_particle_frombuf(b, AS_PARTICLE_TYPE_INTEGER, (uint8_t *) &i, 8, NULL, true);
} else {
pbytes = 8 + as_particle_get_base_size(AS_PARTICLE_TYPE_INTEGER);
if ((urecord->cur_particle_data + pbytes) < urecord->end_particle_data) {
as_particle_frombuf(b, AS_PARTICLE_TYPE_INTEGER,
(uint8_t *) &i, 8,
urecord->cur_particle_data,
rd->ns->storage_data_in_memory);
urecord->cur_particle_data += pbytes;
} else {
cf_warning(AS_UDF, "bool: bin data size too big: pbytes %d %p %p %p",
pbytes, urecord->particle_data, urecord->cur_particle_data,
urecord->end_particle_data);
ret = -1;
break;
}
}
break;
}
case AS_INTEGER: {
as_integer * v = as_integer_fromval(val);
int64_t i = as_integer_get(v);
int64_t j = __be64_to_cpup((void *)&i);
if ( !as_storage_bin_can_fit(rd->ns, 8) ) {
cf_warning(AS_UDF, "int: bin size too big");
ret = -1;
break;
}
if (rd->ns->storage_data_in_memory) {
as_particle_frombuf(b, AS_PARTICLE_TYPE_INTEGER, (uint8_t *) &j, 8, NULL, true);
} else {
pbytes = 8 + as_particle_get_base_size(AS_PARTICLE_TYPE_INTEGER);
if ((urecord->cur_particle_data + pbytes) < urecord->end_particle_data) {
as_particle_frombuf(b, AS_PARTICLE_TYPE_INTEGER,
(uint8_t *) &j, 8, urecord->cur_particle_data,
rd->ns->storage_data_in_memory);
urecord->cur_particle_data += pbytes;
} else {
cf_warning(AS_UDF, "int: bin data size too big: pbytes %d %p %p %p",
pbytes, urecord->particle_data, urecord->cur_particle_data,
urecord->end_particle_data);
ret = -1;
break;
}
}
break;
}
// @LDT : Possibly include AS_LDT in this list. We need the LDT
// bins to be updated by LDT lua calls, and that path takes us thru here.
// However, we ALSO need to be able to set the particle type for the
// bins -- so that requires extra processing here to take the LDT flags
// and set the appropriate bin flags in the particle data.
case AS_MAP:
case AS_LIST: {
as_buffer buf;
as_buffer_init(&buf);
as_serializer s;
as_msgpack_init(&s);
int rsp = 0;
as_serializer_serialize(&s, (as_val *) val, &buf);
if ( !as_storage_bin_can_fit(rd->ns, buf.size) ) {
cf_warning(AS_UDF, "map-list: bin size too big");
ret = -1;
// Clean Up and jump out.
as_serializer_destroy(&s);
as_buffer_destroy(&buf);
break; // can't continue if value too big.
}
uint8_t ptype;
if(is_hidden) {
ptype = as_particle_type_convert_to_hidden(to_particle_type(type));
} else {
ptype = to_particle_type(type);
}
if (rd->ns->storage_data_in_memory) {
as_particle_frombuf(b, ptype, (uint8_t *) buf.data, buf.size, NULL, true);
}
else {
pbytes = buf.size + as_particle_get_base_size(ptype);
if ((urecord->cur_particle_data + pbytes) < urecord->end_particle_data) {
as_particle_frombuf(b, ptype, (uint8_t *) buf.data, buf.size,
urecord->cur_particle_data, rd->ns->storage_data_in_memory);
urecord->cur_particle_data += pbytes;
} else {
cf_warning(AS_UDF, "map-list: bin data size too big: pbytes %d %p %p %p",
pbytes, urecord->particle_data, urecord->cur_particle_data,
urecord->end_particle_data);
rsp = -1;
}
}
as_serializer_destroy(&s);
as_buffer_destroy(&buf);
if (rsp) {
ret = rsp;
break;
}
break;
}
default: {
cf_warning(AS_UDF, "unrecognized object type %d, skipping", as_val_type(val) );
break;
}
}
// If something fail bailout
if (ret) {
as_sindex_sbin_freeall(oldbin, 1);
as_sindex_sbin_freeall(newbin, 1);
return ret;
}
// Update sindex if required
if (has_sindex) {
if (as_sindex_sbin_from_bin(rd->ns,
as_index_get_set_name(rd->r, rd->ns), b, newbin) == AS_SINDEX_OK) {
if (!as_sindex_sbin_match(newbin, oldbin)) {
needs_sindex_put = true;
} else {
needs_sindex_update = true;
}
}
if (needs_sindex_update) {
tr->flag |= AS_TRANSACTION_FLAG_SINDEX_TOUCHED;
as_sindex_delete_by_sbin(rd->ns,
as_index_get_set_name(rd->r, rd->ns), 1, oldbin, rd);
as_sindex_put_by_sbin(rd->ns,
as_index_get_set_name(rd->r, rd->ns), 1, newbin, rd);
} else {
if (needs_sindex_delete) {
tr->flag |= AS_TRANSACTION_FLAG_SINDEX_TOUCHED;
as_sindex_delete_by_sbin(rd->ns,
as_index_get_set_name(rd->r, rd->ns), 1, oldbin, rd);
}
if (needs_sindex_put) {
tr->flag |= AS_TRANSACTION_FLAG_SINDEX_TOUCHED;
as_sindex_put_by_sbin(rd->ns,
as_index_get_set_name(rd->r, rd->ns), 1, newbin, rd);
}
}
as_sindex_sbin_freeall(oldbin, 1);
as_sindex_sbin_freeall(newbin, 1);
}
return ret;
} // end udf_aerospike_setbin()
/*
* Internal Function: Entry function from UDF code path to send
* success result to the caller. Performs
* value translation.
*/
void
send_result(as_result * res, udf_call * call, void *udata)
{
// The following "no-op" line serves to quiet the compiler warning of an
// otherwise unused variable.
udata = udata;
as_val * v = res->value;
if ( res->is_success ) {
if ( cf_context_at_severity(AS_UDF, CF_DETAIL) ) {
char * str = as_val_tostring(v);
cf_detail(AS_UDF, "SUCCESS: %s", str);
cf_free(str);
}
if ( v != NULL ) {
switch( as_val_type(v) ) {
case AS_NIL:
{
send_success(call, AS_PARTICLE_TYPE_NULL, NULL, 0);
break;
}
case AS_BOOLEAN:
{
as_boolean * b = as_boolean_fromval(v);
int64_t bi = as_boolean_tobool(b) == true ? 1 : 0;
send_success(call, AS_PARTICLE_TYPE_INTEGER, &bi, 8);
break;
}
case AS_INTEGER:
{
as_integer * i = as_integer_fromval(v);
int64_t ri = as_integer_toint(i);
send_success(call, AS_PARTICLE_TYPE_INTEGER, &ri, 8);
break;
}
case AS_STRING:
{
// this looks bad but it just pulls the pointer
// out of the object
as_string * s = as_string_fromval(v);
char * rs = (char *) as_string_tostring(s);
send_success(call, AS_PARTICLE_TYPE_STRING, rs, as_string_len(s));
break;
}
case AS_BYTES:
{
as_bytes * b = as_bytes_fromval(v);
uint8_t * rs = as_bytes_get(b);
send_success(call, AS_PARTICLE_TYPE_BLOB, rs, as_bytes_size(b));
break;
}
case AS_MAP:
case AS_LIST:
{
as_buffer buf;
as_buffer_init(&buf);
as_serializer s;
as_msgpack_init(&s);
int res = as_serializer_serialize(&s, v, &buf);
if (res != 0) {
const char * error = "Complex Data Type Serialization failure";
cf_warning(AS_UDF, "%s (%d)", (char *)error, res);
as_buffer_destroy(&buf);
send_cdt_failure(call, AS_PARTICLE_TYPE_STRING, (char *)error, strlen(error));
}
else {
// Do not use this until after cf_detail_binary() can accept larger buffers.
// cf_detail_binary(AS_UDF, buf.data, buf.size, CF_DISPLAY_HEX_COLUMNS,
// "serialized %d bytes: ", buf.size);
send_success(call, to_particle_type(as_val_type(v)), buf.data, buf.size);
// Not needed stack allocated - unless serialize has internal state
// as_serializer_destroy(&s);
as_buffer_destroy(&buf);
}
break;
}
default:
{
cf_debug(AS_UDF, "SUCCESS: VAL TYPE UNDEFINED %d\n", as_val_type(v));
send_success(call, AS_PARTICLE_TYPE_STRING, NULL, 0);
break;
}
}
} else {
send_success(call, AS_PARTICLE_TYPE_NULL, NULL, 0);
}
} else { // Else -- NOT success
as_string * s = as_string_fromval(v);
char * rs = (char *) as_string_tostring(s);
cf_debug(AS_UDF, "FAILURE when calling %s %s %s", call->filename, call->function, rs);
send_udf_failure(call, AS_PARTICLE_TYPE_STRING, rs, as_string_len(s));
}
}
/*
*******************************************************************************************************
* Checks as_bytes->type.
* Deserializes as_bytes into Py_Object (retval) using deserialization logic
* based on as_bytes->type.
*
* @param bytes The as_bytes to be deserialized.
* @param retval The return zval to be populated with the
* deserialized value of the input as_bytes.
* @param error_p The as_error to be populated by the function
* with encountered error if any.
*******************************************************************************************************
*/
extern PyObject * deserialize_based_on_as_bytes_type(AerospikeClient * self,
as_bytes *bytes,
PyObject **retval,
as_error *error_p)
{
switch(as_bytes_get_type(bytes)) {
case AS_BYTES_PYTHON: {
PyObject* sysmodules = PyImport_GetModuleDict();
PyObject* cpickle_module = NULL;
if(PyMapping_HasKeyString(sysmodules, "cPickle")) {
cpickle_module = PyMapping_GetItemString(sysmodules, "cPickle");
} else {
cpickle_module = PyImport_ImportModule("cPickle");
}
PyObject* initresult = NULL;
if(!cpickle_module) {
/* insert error handling here! and exit this function */
as_error_update(error_p, AEROSPIKE_ERR_CLIENT, "Unable to load cpickle module");
goto CLEANUP;
} else {
char* bytes_val_p = (char*)bytes->value;
PyObject *py_value = PyStr_FromStringAndSize(bytes_val_p, as_bytes_size(bytes));
PyObject *py_funcname = PyStr_FromString("loads");
Py_INCREF(cpickle_module);
initresult = PyObject_CallMethodObjArgs(cpickle_module, py_funcname, py_value, NULL);
Py_DECREF(cpickle_module);
Py_DECREF(py_funcname);
Py_DECREF(py_value);
if(!initresult) {
/* more error handling &c */
as_error_update(error_p, AEROSPIKE_ERR_CLIENT, "Unable to call loads function");
goto CLEANUP;
} else {
*retval = initresult;
}
}
Py_XDECREF(cpickle_module);
}
break;
case AS_BYTES_BLOB: {
if (self->user_deserializer_call_info.callback) {
execute_user_callback(&self->user_deserializer_call_info, &bytes, retval, false, error_p);
if(AEROSPIKE_OK != (error_p->code)) {
uint32_t bval_size = as_bytes_size(bytes);
PyObject *py_val = PyByteArray_FromStringAndSize((char *) as_bytes_get(bytes), bval_size);
if (!py_val) {
as_error_update(error_p, AEROSPIKE_ERR_CLIENT, "Unable to deserialize bytes");
goto CLEANUP;
}
*retval = py_val;
as_error_update(error_p, AEROSPIKE_OK, NULL);
}
} else {
if (is_user_deserializer_registered) {
execute_user_callback(&user_deserializer_call_info, &bytes, retval, false, error_p);
if(AEROSPIKE_OK != (error_p->code)) {
uint32_t bval_size = as_bytes_size(bytes);
PyObject *py_val = PyByteArray_FromStringAndSize((char *) as_bytes_get(bytes), bval_size);
if (!py_val) {
as_error_update(error_p, AEROSPIKE_ERR_CLIENT, "Unable to deserialize bytes");
goto CLEANUP;
}
as_error_update(error_p, AEROSPIKE_OK, NULL);
*retval = py_val;
}
} else {
uint32_t bval_size = as_bytes_size(bytes);
PyObject *py_val = PyByteArray_FromStringAndSize((char *) as_bytes_get(bytes), bval_size);
if (!py_val) {
as_error_update(error_p, AEROSPIKE_ERR_CLIENT, "Unable to deserialize bytes");
goto CLEANUP;
}
*retval = py_val;
}
}
}
break;
case AS_BYTES_LDT: {
Py_INCREF(Py_None);
*retval = Py_None;
}
break;
default:
as_error_update(error_p, AEROSPIKE_ERR,
"Unable to deserialize bytes");
goto CLEANUP;
}
CLEANUP:
if ( error_p->code != AEROSPIKE_OK ) {
PyObject * py_err = NULL;
error_to_pyobject(error_p, &py_err);
PyObject *exception_type = raise_exception(error_p);
PyErr_SetObject(exception_type, py_err);
Py_DECREF(py_err);
return NULL;
}
return PyLong_FromLong(0);
}
/**
* Call with AS_OPERATIONS_CDT_OP only.
*/
static bool as_operations_cdt_op(as_operations *ops, const as_bin_name name, as_cdt_optype op, size_t n, ...)
{
if (op >= cdt_op_table_size) {
return false;
}
const cdt_op_table_entry *entry = &cdt_op_table[op];
if (n < entry->count - entry->opt_args || n > entry->count) {
return false;
}
va_list vl;
if (n > 0) {
va_start(vl, n);
}
as_arraylist list;
as_arraylist_inita(&list, (uint32_t)n + 1); // +1 to avoid alloca(0) undefined behavior
for (size_t i = 0; i < n; i++) {
as_cdt_paramtype type = entry->args[i];
switch (type) {
case AS_CDT_PARAM_PAYLOAD: {
as_val *arg = va_arg(vl, as_val *);
if (as_arraylist_append(&list, arg) != AS_ARRAYLIST_OK) {
va_end(vl);
as_arraylist_destroy(&list);
return false;
}
break;
}
case AS_CDT_PARAM_COUNT: {
uint64_t arg = va_arg(vl, uint64_t);
if (as_arraylist_append(&list, (as_val *)as_integer_new(arg)) != AS_ARRAYLIST_OK) {
va_end(vl);
as_arraylist_destroy(&list);
return false;
}
break;
}
case AS_CDT_PARAM_INDEX: {
int64_t arg = va_arg(vl, int64_t);
if (as_arraylist_append(&list, (as_val *)as_integer_new(arg)) != AS_ARRAYLIST_OK) {
va_end(vl);
as_arraylist_destroy(&list);
return false;
}
break;
}
default:
break;
}
}
if (n > 0) {
va_end(vl);
}
as_serializer ser;
as_msgpack_init(&ser);
uint32_t list_size = as_serializer_serialize_getsize(&ser, (as_val *) &list);
as_bytes *bytes = as_bytes_new(sizeof(uint16_t) + list_size);
uint8_t *list_write = as_bytes_get(bytes);
uint16_t *list_write_op = (uint16_t *)list_write;
*list_write_op = cf_swap_to_be16(op);
list_write += sizeof(uint16_t);
as_serializer_serialize_presized(&ser, (const as_val *) &list, list_write);
as_serializer_destroy(&ser);
as_arraylist_destroy(&list);
bytes->size = bytes->capacity;
// as_bytes->type default to AS_BYTES_BLOB
if (entry->rw_type == CDT_RW_TYPE_MODIFY) {
return as_operations_add_cdt_modify(ops, name, (as_bin_value *) bytes);
}
return as_operations_add_cdt_read(ops, name, (as_bin_value *) bytes);
}
