static as_node*
as_cluster_find_node(as_cluster* cluster, in_addr_t addr, in_port_t port)
{
as_nodes* nodes = (as_nodes*)cluster->nodes;
as_node* node;
as_vector* addresses;
as_address* address;
struct sockaddr_in* sockaddr;
in_port_t port_be = cf_swap_to_be16(port);
for (uint32_t i = 0; i < nodes->size; i++) {
node = nodes->array[i];
addresses = &node->addresses;
for (uint32_t j = 0; j < addresses->size; j++) {
address = as_vector_get(addresses, j);
sockaddr = &address->addr;
if (sockaddr->sin_addr.s_addr == addr && sockaddr->sin_port == port_be) {
return node;
}
}
}
return 0;
}
bool
as_lookup(as_cluster* cluster, char* hostname, uint16_t port, bool enable_warning, as_vector* /*<struct sockaddr_in>*/ addresses)
{
// Check if there is an alternate address that should be used for this hostname.
if (cluster && cluster->ip_map) {
as_addr_map* entry = cluster->ip_map;
for (uint32_t i = 0; i < cluster->ip_map_size; i++) {
if (strcmp(entry->orig, hostname) == 0) {
// Found mapping for this address. Use alternate.
cf_debug("Using %s instead of %s", entry->alt, hostname);
hostname = entry->alt;
break;
}
entry++;
}
}
// Lookup TCP addresses.
struct addrinfo hints;
memset(&hints, 0, sizeof(hints));
hints.ai_family = AF_INET;
struct addrinfo* results = 0;
int ret = getaddrinfo(hostname, 0, &hints, &results);
if (ret) {
if (enable_warning) {
cf_warn("Invalid hostname %s: %s", hostname, gai_strerror(ret));
}
return false;
}
// Add addresses to vector if it exists.
if (addresses) {
uint16_t port_be = cf_swap_to_be16(port);
for (struct addrinfo* r = results; r; r = r->ai_next) {
struct sockaddr_in* addr = (struct sockaddr_in*)r->ai_addr;
addr->sin_port = port_be;
as_vector_append_unique(addresses, addr);
}
}
freeaddrinfo(results);
return true;
}
// TODO - old pickle - remove in "six months".
// Flatten record's bins into "pickle" format for fabric.
uint8_t *
as_record_pickle(as_storage_rd *rd, size_t *len_r)
{
as_namespace *ns = rd->ns;
uint32_t sz = 2; // always 2 bytes for number of bins
uint16_t n_bins_in_use;
for (n_bins_in_use = 0; n_bins_in_use < rd->n_bins; n_bins_in_use++) {
as_bin *b = &rd->bins[n_bins_in_use];
if (! as_bin_inuse(b)) {
break;
}
sz += 1; // for bin name length
sz += ns->single_bin ?
0 : strlen(as_bin_get_name_from_id(ns, b->id)); // for bin name
sz += 1; // was for version - currently not used
sz += as_bin_particle_pickled_size(b);
}
uint8_t *pickle = cf_malloc(sz);
uint8_t *buf = pickle;
(*(uint16_t *)buf) = cf_swap_to_be16(n_bins_in_use); // number of bins
buf += 2;
for (uint16_t i = 0; i < n_bins_in_use; i++) {
as_bin *b = &rd->bins[i];
// Copy bin name, skipping a byte for name length.
uint8_t name_len = (uint8_t)as_bin_memcpy_name(ns, buf + 1, b);
*buf++ = name_len; // fill in bin name length
buf += name_len; // skip past bin name
*buf++ = 0; // was version - currently not used
buf += as_bin_particle_to_pickled(b, buf);
}
*len_r = sz;
return pickle;
}
uint32_t
geojson_asval_to_wire(const as_val *val, uint8_t *wire)
{
as_geojson *pg = as_geojson_fromval(val);
size_t jsz = as_geojson_len(pg);
uint8_t *p8 = wire;
*p8++ = 0; // flags
uint16_t *p16 = (uint16_t *)p8;
*p16++ = cf_swap_to_be16(0); // no cells on output to client
p8 = (uint8_t *)p16;
memcpy(p8, as_geojson_get(pg), jsz);
return geojson_size(0, jsz);
}
uint32_t
geojson_to_wire(const as_particle *p, uint8_t *wire)
{
// Use blob routine first.
uint32_t sz = blob_to_wire(p, wire);
// Swap ncells.
uint16_t *p_ncells = (uint16_t *)(wire + sizeof(uint8_t));
uint16_t ncells = *p_ncells;
*p_ncells = cf_swap_to_be16(*p_ncells);
++p_ncells;
// Swap the cells.
uint64_t *p_cell_begin = (uint64_t *)p_ncells;
uint64_t *p_cell_end = p_cell_begin + ncells;
for (uint64_t *p_cell = p_cell_begin; p_cell < p_cell_end; ++p_cell) {
*p_cell = cf_swap_to_be64(*p_cell);
}
return sz;
}
uint8_t*
as_command_write_header(uint8_t* cmd, uint8_t read_attr, uint8_t write_attr,
as_policy_commit_level commit_level, as_policy_consistency_level consistency,
as_policy_exists exists, as_policy_gen gen_policy, uint32_t gen, uint32_t ttl,
uint32_t timeout_ms, uint16_t n_fields, uint16_t n_bins)
{
uint32_t generation = 0;
uint8_t info_attr = 0;
switch (exists) {
case AS_POLICY_EXISTS_IGNORE:
break;
case AS_POLICY_EXISTS_UPDATE:
info_attr |= AS_MSG_INFO3_UPDATE_ONLY;
break;
case AS_POLICY_EXISTS_CREATE_OR_REPLACE:
info_attr |= AS_MSG_INFO3_CREATE_OR_REPLACE;
break;
case AS_POLICY_EXISTS_REPLACE:
info_attr |= AS_MSG_INFO3_REPLACE_ONLY;
break;
case AS_POLICY_EXISTS_CREATE:
write_attr |= AS_MSG_INFO2_CREATE_ONLY;
break;
}
switch (gen_policy) {
case AS_POLICY_GEN_IGNORE:
break;
case AS_POLICY_GEN_EQ:
generation = gen;
write_attr |= AS_MSG_INFO2_GENERATION;
break;
case AS_POLICY_GEN_GT:
generation = gen;
write_attr |= AS_MSG_INFO2_GENERATION_GT;
break;
default:
break;
}
if (commit_level == AS_POLICY_COMMIT_LEVEL_MASTER) {
info_attr |= AS_MSG_INFO3_COMMIT_MASTER;
}
if (consistency == AS_POLICY_CONSISTENCY_LEVEL_ALL) {
read_attr |= AS_MSG_INFO1_CONSISTENCY_ALL;
}
cmd[8] = 22;
cmd[9] = read_attr;
cmd[10] = write_attr;
cmd[11] = info_attr;
*(uint16_t*)&cmd[12] = 0;
*(uint32_t*)&cmd[14] = cf_swap_to_be32(generation);
*(uint32_t*)&cmd[18] = cf_swap_to_be32(ttl);
*(uint32_t*)&cmd[22] = cf_swap_to_be32(timeout_ms);
*(uint16_t*)&cmd[26] = cf_swap_to_be16(n_fields);
*(uint16_t*)&cmd[28] = cf_swap_to_be16(n_bins);
return cmd + AS_HEADER_SIZE;
}
int
cl_lookup(cl_cluster *asc, char *hostname, short port, cf_vector *sockaddr_in_v)
{
// do the gethostbyname to find the IP address
size_t hstbuflen = 1024;
uint8_t stack_hstbuf[hstbuflen];
void *tmphstbuf = stack_hstbuf;
int rv, herr, addrmapsz;
struct hostent *hp;
cl_addrmap *map;
int retry = 0;
//Find if there is an alternate address that should be used for this hostname.
if (asc && (asc->host_addr_map_v.len > 0)) {
addrmapsz = asc->host_addr_map_v.len;
for (int i=0; i<addrmapsz; i++) {
map = cf_vector_pointer_get(&asc->host_addr_map_v, i);
if (map && strcmp(map->orig, hostname) == 0) {
//found a mapping for this address. Use the alternate one.
cf_debug("Using %s instead of %s", map->alt, hostname);
hostname = map->alt;
break;
}
}
}
do {
#ifdef __APPLE__
// on OSX, gethostbyname is thread safe and there is no '_r' version
hp = gethostbyname2(hostname, AF_INET);
rv = 0;
if(hp == NULL){
herr = h_errno; // I'm hoping this is thread-safe too, in the Mac world...
}
#else
struct hostent hostbuf;
rv = gethostbyname2_r(hostname, AF_INET, &hostbuf, tmphstbuf, hstbuflen,
&hp, &herr);
#endif
/* TRY_AGAIN for a maximun of 3 times, after which throw an error */
if(retry > 2) {
cf_error("gethostbyname of %s - maxmimum retries failed", hostname);
retry = 0;
return -1;
}
if (hp == NULL) {
hostname = hostname ? hostname : "NONAME";
switch(herr) {
case HOST_NOT_FOUND:
cf_error("gethostbyname says no host at %s", hostname);
break;
case NO_ADDRESS:
cf_error("gethostbyname of %s says invalid address (errno %d)", hostname, herr);
break;
case NO_RECOVERY:
cf_error("gethostbyname of %s says form error (errno %d)", hostname, herr);
break;
case TRY_AGAIN:
cf_error("gethostbyname of %s returned TRY_AGAIN, try again (rv=%d)", hostname, rv);
retry++;
continue;
default:
cf_error("gethostbyname of %s returned an unknown error (errno %d)", hostname, herr);
break;
}
if (tmphstbuf != stack_hstbuf) free(tmphstbuf);
return(-1);
}
else if (rv != 0) {
if (rv == ERANGE) {
hstbuflen *= 2;
if (tmphstbuf == stack_hstbuf)
tmphstbuf = malloc(hstbuflen);
else
tmphstbuf = realloc (tmphstbuf, hstbuflen);
if (!tmphstbuf) {
cf_error("malloc fail");
return(-1);
}
}
else if (rv == EAGAIN || herr == TRY_AGAIN) {
cf_error("gethostbyname returned EAGAIN, try again");
retry++;
}
else if (rv == ETIMEDOUT) {
cf_error("gethostbyname for %s timed out", hostname ? (hostname): "NONAME");
if (tmphstbuf != stack_hstbuf) free(tmphstbuf);
return(-1);
}
else {
cf_error("gethostbyname returned an unknown error %d %d (errno %d)",rv,herr, errno);
if (tmphstbuf != stack_hstbuf) free(tmphstbuf);
return(-1);
}
}
} while ((rv != 0) || (hp == NULL));
#ifdef DEBUG
cf_debug("host lookup: %s canonical: %s addrtype %d length: %d",
hostname, hp->h_name, hp->h_addrtype, hp->h_length);
for (int i=0;hp->h_aliases[i];i++) {
cf_debug(" alias %d: %s",i, hp->h_aliases[i]);
}
for (int i=0;hp->h_addr_list[i];i++) {
// todo: print something about the actual address
cf_debug(" address %d: %x",i,*(uint32_t *) hp->h_addr_list[i]);
}
#endif
if (hp->h_addrtype != AF_INET) {
cf_error("unknown address type %d", hp->h_addrtype);
if (tmphstbuf != stack_hstbuf) free(tmphstbuf);
return(-1);
}
// sockaddr_in_v is passed as NULL from caller which needs
// to only check if lookup succeeds. If reach here it is
// a successful lookup.
if (sockaddr_in_v == NULL) {
goto ret_success;
}
// Move into vector
for (int i=0;hp->h_addr_list[i];i++) {
struct sockaddr_in addr;
memset(&addr,0,sizeof(addr));
addr.sin_family = hp->h_addrtype;
addr.sin_addr.s_addr = *(uint32_t *) hp->h_addr_list[i];
addr.sin_port = (in_port_t)cf_swap_to_be16(port);
cf_vector_append_unique(sockaddr_in_v, &addr);
}
ret_success:
if (tmphstbuf != stack_hstbuf) free(tmphstbuf);
return(0);
}
uint8_t*
as_command_write_bin(uint8_t* begin, uint8_t operation_type, const as_bin* bin, as_buffer* buffer)
{
uint8_t* p = begin + AS_OPERATION_HEADER_SIZE;
const char* name = bin->name;
// Copy string, but do not transfer null byte.
while (*name) {
*p++ = *name++;
}
uint8_t name_len = p - begin - AS_OPERATION_HEADER_SIZE;
as_val* val = (as_val*)bin->valuep;
uint32_t val_len;
uint8_t val_type;
switch (val->type) {
default:
case AS_NIL: {
val_len = 0;
val_type = AS_BYTES_UNDEF;
break;
}
case AS_INTEGER: {
as_integer* v = as_integer_fromval(val);
*(uint64_t*)p = cf_swap_to_be64(v->value);
p += 8;
val_len = 8;
val_type = AS_BYTES_INTEGER;
break;
}
case AS_DOUBLE: {
as_double* v = as_double_fromval(val);
*(double*)p = cf_swap_to_big_float64(v->value);
p += 8;
val_len = 8;
val_type = AS_BYTES_DOUBLE;
break;
}
case AS_STRING: {
as_string* v = as_string_fromval(val);
// v->len should have been already set by as_command_value_size().
memcpy(p, v->value, v->len);
p += v->len;
val_len = (uint32_t)v->len;
val_type = AS_BYTES_STRING;
break;
}
case AS_GEOJSON: {
// We send a cellid placeholder so we can fill in points
// in place on the server w/o changing object size.
as_geojson* v = as_geojson_fromval(val);
// v->len should have been already set by as_command_value_size().
// as_particle_geojson_mem::flags
*p++ = 0;
// as_particle_geojson_mem::ncells
*(uint16_t *) p = cf_swap_to_be16(0);
p += sizeof(uint16_t);
// placeholder cellid
// THIS LOOP EXECUTES 0 TIMES (still, it belongs here ...)
for (int ii = 0; ii < 0; ++ii) {
*(uint64_t *) p = cf_swap_to_be64(0);
p += sizeof(uint64_t);
}
// json data itself
memcpy(p, v->value, v->len);
p += v->len;
val_len = (uint32_t)(1 + 2 + (0 * 8) + v->len);
val_type = AS_BYTES_GEOJSON;
break;
}
case AS_BYTES: {
as_bytes* v = as_bytes_fromval(val);
memcpy(p, v->value, v->size);
p += v->size;
val_len = v->size;
// Note: v->type must be a blob type (AS_BYTES_BLOB, AS_BYTES_JAVA, AS_BYTES_PYTHON ...).
// Otherwise, the particle type will be reassigned to a non-blob which causes a
// mismatch between type and value.
val_type = v->type;
break;
}
case AS_LIST: {
memcpy(p, buffer->data, buffer->size);
p += buffer->size;
val_len = buffer->size;
val_type = AS_BYTES_LIST;
cf_free(buffer->data);
break;
}
case AS_MAP: {
memcpy(p, buffer->data, buffer->size);
p += buffer->size;
val_len = buffer->size;
val_type = AS_BYTES_MAP;
cf_free(buffer->data);
break;
}
}
*(uint32_t*)begin = cf_swap_to_be32(name_len + val_len + 4);
begin += 4;
*begin++ = operation_type;
*begin++ = val_type;
*begin++ = 0;
*begin++ = name_len;
return p;
}
/**
* Call with AS_OPERATIONS_CDT_OP only.
*/
static bool as_operations_cdt_op(as_operations *ops, const as_bin_name name, as_cdt_optype op, size_t n, ...)
{
if (op >= cdt_op_table_size) {
return false;
}
const cdt_op_table_entry *entry = &cdt_op_table[op];
if (n < entry->count - entry->opt_args || n > entry->count) {
return false;
}
va_list vl;
if (n > 0) {
va_start(vl, n);
}
as_arraylist list;
as_arraylist_inita(&list, (uint32_t)n + 1); // +1 to avoid alloca(0) undefined behavior
for (size_t i = 0; i < n; i++) {
as_cdt_paramtype type = entry->args[i];
switch (type) {
case AS_CDT_PARAM_PAYLOAD: {
as_val *arg = va_arg(vl, as_val *);
if (as_arraylist_append(&list, arg) != AS_ARRAYLIST_OK) {
va_end(vl);
as_arraylist_destroy(&list);
return false;
}
break;
}
case AS_CDT_PARAM_COUNT: {
uint64_t arg = va_arg(vl, uint64_t);
if (as_arraylist_append(&list, (as_val *)as_integer_new(arg)) != AS_ARRAYLIST_OK) {
va_end(vl);
as_arraylist_destroy(&list);
return false;
}
break;
}
case AS_CDT_PARAM_INDEX: {
int64_t arg = va_arg(vl, int64_t);
if (as_arraylist_append(&list, (as_val *)as_integer_new(arg)) != AS_ARRAYLIST_OK) {
va_end(vl);
as_arraylist_destroy(&list);
return false;
}
break;
}
default:
break;
}
}
if (n > 0) {
va_end(vl);
}
as_serializer ser;
as_msgpack_init(&ser);
uint32_t list_size = as_serializer_serialize_getsize(&ser, (as_val *) &list);
as_bytes *bytes = as_bytes_new(sizeof(uint16_t) + list_size);
uint8_t *list_write = as_bytes_get(bytes);
uint16_t *list_write_op = (uint16_t *)list_write;
*list_write_op = cf_swap_to_be16(op);
list_write += sizeof(uint16_t);
as_serializer_serialize_presized(&ser, (const as_val *) &list, list_write);
as_serializer_destroy(&ser);
as_arraylist_destroy(&list);
bytes->size = bytes->capacity;
// as_bytes->type default to AS_BYTES_BLOB
if (entry->rw_type == CDT_RW_TYPE_MODIFY) {
return as_operations_add_cdt_modify(ops, name, (as_bin_value *) bytes);
}
return as_operations_add_cdt_read(ops, name, (as_bin_value *) bytes);
}
