END_TEST
START_TEST (check_fs_rid_vector_sort)
{
fs_rid_vector *v = fs_rid_vector_new(0);
fail_unless(v != NULL, "fs_rid_vector is NULL");
int i=0;
for (i=100; i < 1e4; i++) {
if (i % 2) {
fs_rid_vector_append(v, i);
fs_rid_vector_append(v, i+3);
fs_rid_vector_append(v, i-10);
} else {
fs_rid_vector_append(v, i-1);
fs_rid_vector_append(v, i+5);
fs_rid_vector_append(v, i);
}
}
fs_rid_vector_sort(v);
fail_if(fs_rid_vector_length(v) != ((1e4 - 100) *3), "Length fail after sort");
for (i=100; i < 1e4; i++) {
fail_if ( !fs_rid_vector_contains(v, i), "Contain failed after sort");
}
for (i=0;i<fs_rid_vector_length(v) -1; i++) {
fail_if (v->data[i] > v->data[i+1], "Sort does not match.");
}
fs_rid_vector_free(v);
}
END_TEST
START_TEST (check_fs_rid_vector_uniq)
{
fs_rid_vector *v = fs_rid_vector_new(0);
fail_unless(v != NULL, "fs_rid_vector is NULL");
int i=0;
for (i=100; i < 500; i++) {
fs_rid_vector_append(v, i);
fs_rid_vector_append(v, i);
}
fs_rid_vector_append(v,FS_RID_NULL);
fail_if(fs_rid_vector_length(v) != (((500 - 100)*2) + 1), "fs_rid_vector_length failed");
fs_rid_vector_uniq(v, 0); //Not remove nulls
fail_if(fs_rid_vector_length(v) != ((500 - 100) + 1), "fs_rid_vector_length failed");
for (i=1; i < fs_rid_vector_length(v) - 1; i++) {
fail_if(v->data[i] == v->data[i-1], "unique values failed [%d %d]",v->data[i],v->data[i-1]);
}
fail_if(v->data[fs_rid_vector_length(v)-1] != FS_RID_NULL, "FS_RID_NULL isn't there.");
v->data[123] = FS_RID_NULL;
fs_rid_vector_uniq(v, 1); //remove nulls
fail_if(fs_rid_vector_length(v) != ((500 - 100) -1), "fs_rid_vector_length failed");
fail_if(v->data[fs_rid_vector_length(v)-1] == FS_RID_NULL, "FS_RID_NULL should not be there %llx",
v->data[fs_rid_vector_length(v)-1]);
fail_if(v->data[123] == FS_RID_NULL, "FS_RID_NULL should not be there %llx",
v->data[fs_rid_vector_length(v)-1]);
fs_rid_vector_free(v);
}
static int delete_rasqal_triple(struct update_context *ct, fs_rid_vector *vec[], rasqal_triple *triple, int row)
{
fs_rid m, s, p, o;
if (triple->origin) {
m = fs_hash_rasqal_literal(ct, triple->origin, row);
if (m == FS_RID_NULL) return 1;
} else if (ct->op->graph_uri) {
m = fs_hash_uri((char *)raptor_uri_as_string(ct->op->graph_uri));
} else {
/* m can be wildcard in the absence of GRAPH, WITH etc. */
m = FS_RID_NULL;
}
s = fs_hash_rasqal_literal(ct, triple->subject, row);
if (s == FS_RID_NULL) return 1;
p = fs_hash_rasqal_literal(ct, triple->predicate, row);
if (p == FS_RID_NULL) return 1;
o = fs_hash_rasqal_literal(ct, triple->object, row);
if (o == FS_RID_NULL) return 1;
/* as long as s, p, and o are bound, we can add this quad */
fs_rid_vector_append(vec[0], m);
fs_rid_vector_append(vec[1], s);
fs_rid_vector_append(vec[2], p);
fs_rid_vector_append(vec[3], o);
if (fs_rid_vector_length(vec[0]) > 999) {
fsp_delete_quads_all(ct->link, vec);
for (int s=0; s<4; s++) {
fs_rid_vector_truncate(vec[s], 0);
}
}
return 0;
}
END_TEST
START_TEST (check_fs_rid_vector_truncate)
{
fs_rid_vector *v = fs_rid_vector_new(0);
fail_unless(v != NULL, "fs_rid_vector is NULL");
int i=0;
for (i=100; i < 1e4; i++) {
fs_rid_vector_append(v, i);
}
fs_rid_vector_truncate(v, 100);
fail_if(fs_rid_vector_length(v) != 100);
fail_if(v->data[fs_rid_vector_length(v)-1] != 199);
fs_rid_vector_truncate(v, 1);
fail_if(v->data[fs_rid_vector_length(v)-1] != 100);
fs_rid_vector_free(v);
}
END_TEST
START_TEST (check_fs_rid_vector_grow)
{
fs_rid_vector *v = fs_rid_vector_new(0);
fail_unless(v != NULL, "fs_rid_vector is NULL");
int i=0;
for (i=100; i < 1e4; i++) {
fs_rid_vector_append(v, i);
}
fail_if(fs_rid_vector_length(v) != (1e4 - 100));
fs_rid_vector_grow(v, 2e4);
fail_if(fs_rid_vector_length(v) != 2e4);
for (i=0;i<2e4;i++) {
v->data[i] = i;
}
fail_if(v->size != 2e4);
fs_rid_vector_free(v);
}
/**
* It wraps up the bind operation to discard rows from the result that cannot be accessed.
*/
int fs_bind_cache_wrapper_intl_acl(fs_query_state *qs, fs_query *q, int all,
int flags, fs_rid_vector *rids[4],
fs_rid_vector ***result, int offset, int limit) {
int flags_copy = flags;
int ndiscarded = 0;
if (fsp_is_acl_enabled(qs->link)) {
flags = flags | FS_BIND_MODEL;
}
int ret = fs_bind_cache_wrapper_intl(qs, q, all, flags, rids, result, offset, limit);
if (fsp_is_acl_enabled(qs->link) && (*result)) {
unsigned char *rows_discarded = NULL;
/* TODO probably this can be done with one iteration of results */
fs_rid_set *inv_acl = no_access_for_user(qs->link->acl_system_info,q->apikey_rid);
ndiscarded = fs_mark_discard_rows((*result)[0], inv_acl, &rows_discarded);
if (inv_acl)
fs_rid_set_free(inv_acl);
int slots = fs_slots_n(flags_copy);
if (!(flags_copy & FS_BIND_MODEL) && (flags & FS_BIND_MODEL)) {
fs_rid_vector **result_copy = calloc(slots, sizeof(fs_rid_vector));
for (int i=0;i<slots;i++)
result_copy[i] = (*result)[i+1];
fs_rid_vector_free((*result)[0]);
free(*result);
*result = result_copy;
}
if (ndiscarded) {
fs_rid_vector **rows = *result;
int count = fs_rid_vector_length(rows[0]);
int shifts = 0;
for (int i=0; i < count; i++) {
for (int s=0;s<slots;s++)
rows[s]->data[i-shifts] = rows[s]->data[i];
if (!fs_bit_array_get(rows_discarded, i))
shifts++;
}
for (int s=0;s<slots;s++) {
rows[s]->length -= ndiscarded;
}
}
if (rows_discarded)
fs_bit_array_destroy(rows_discarded);
}
return ret - ndiscarded;
}
/**
* It discards graph rows that are not accesible by the query user
* m is the models (column 0) result of the bind
* inv_acl is the set of graphs that the query cannot access
* discarded returns a bit_array by reference with 0's in not accesible rows
*/
int fs_mark_discard_rows(fs_rid_vector *m, fs_rid_set *inv_acl, unsigned char **discarded) {
int qdiscarded = 0;
unsigned char *allow_access = NULL;
if (inv_acl) {
int count = fs_rid_vector_length(m);
for (int i=0 ; i<count ; i++) {
fs_rid rid_m = m->data[i];
if (fs_rid_set_contains(inv_acl, rid_m)) {
if (!allow_access) allow_access = fs_new_bit_array(count);
fs_bit_array_set(allow_access,i,0);
qdiscarded++;
}
}
}
if (allow_access)
*discarded = allow_access;
return qdiscarded;
}
static int update_op(struct update_context *uc)
{
fs_rid_vector *vec[4];
switch (uc->op->type) {
case RASQAL_UPDATE_TYPE_UNKNOWN:
add_message(uc, "Unknown update operation", 0);
return 1;
case RASQAL_UPDATE_TYPE_CLEAR:
fs_clear(uc, graph_arg(uc->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_CREATE:
return 0;
case RASQAL_UPDATE_TYPE_DROP:
fs_clear(uc, graph_arg(uc->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_LOAD:
fs_load(uc, graph_arg(uc->op->document_uri),
graph_arg(uc->op->graph_uri));
return 0;
#if RASQAL_VERSION >= 924
case RASQAL_UPDATE_TYPE_ADD:
fs_add(uc, graph_arg(uc->op->graph_uri),
graph_arg(uc->op->document_uri));
return 0;
case RASQAL_UPDATE_TYPE_MOVE:
fs_move(uc, graph_arg(uc->op->graph_uri),
graph_arg(uc->op->document_uri));
return 0;
case RASQAL_UPDATE_TYPE_COPY:
fs_copy(uc, graph_arg(uc->op->graph_uri),
graph_arg(uc->op->document_uri));
return 0;
#endif
case RASQAL_UPDATE_TYPE_UPDATE:
break;
}
fs_hash_freshen();
raptor_sequence *todel = NULL;
raptor_sequence *toins = NULL;
if (uc->op->delete_templates && !uc->op->where) {
int where = 0;
/* check to see if it's a DELETE WHERE { } */
for (int t=0; t<raptor_sequence_size(uc->op->delete_templates); t++) {
rasqal_triple *tr = raptor_sequence_get_at(uc->op->delete_templates, t);
if (any_vars(tr)) {
where = 1;
break;
}
}
if (where) {
fs_error(LOG_ERR, "DELETE WHERE { x } not yet supported");
add_message(uc, "DELETE WHERE { x } not yet supported, use DELETE { x } WHERE { x }", 0);
return 1;
}
}
#if RASQAL_VERSION >= 923
if (uc->op->where) {
todel = raptor_new_sequence(NULL, NULL);
toins = raptor_new_sequence(NULL, NULL);
raptor_sequence *todel_p = raptor_new_sequence(NULL, NULL);
raptor_sequence *toins_p = raptor_new_sequence(NULL, NULL);
raptor_sequence *vars = raptor_new_sequence(NULL, NULL);
fs_query *q = calloc(1, sizeof(fs_query));
uc->q = q;
q->qs = uc->qs;
q->rq = uc->rq;
q->flags = FS_BIND_DISTINCT;
#ifdef DEBUG_MERGE
q->flags |= FS_QUERY_CONSOLE_OUTPUT;
#endif
q->boolean = 1;
q->opt_level = 3;
q->soft_limit = -1;
q->segments = fsp_link_segments(uc->link);
q->link = uc->link;
q->bb[0] = fs_binding_new();
q->bt = q->bb[0];
/* hashtable to hold runtime created resources */
q->tmp_resources = g_hash_table_new_full(fs_rid_hash, fs_rid_equal, g_free, fs_free_cached_resource);
/* add column to denote join ordering */
fs_binding_create(q->bb[0], "_ord", FS_RID_NULL, 0);
if (uc->op->delete_templates) {
for (int t=0; t<raptor_sequence_size(uc->op->delete_templates); t++) {
rasqal_triple *tr = raptor_sequence_get_at(uc->op->delete_templates, t);
if (any_vars(tr)) {
fs_check_cons_slot(q, vars, tr->subject);
fs_check_cons_slot(q, vars, tr->predicate);
fs_check_cons_slot(q, vars, tr->object);
raptor_sequence_push(todel_p, tr);
} else {
raptor_sequence_push(todel, tr);
}
}
}
if (uc->op->insert_templates) {
for (int t=0; t<raptor_sequence_size(uc->op->insert_templates); t++) {
rasqal_triple *tr = raptor_sequence_get_at(uc->op->insert_templates, t);
if (any_vars(tr)) {
fs_check_cons_slot(q, vars, tr->subject);
fs_check_cons_slot(q, vars, tr->predicate);
fs_check_cons_slot(q, vars, tr->object);
raptor_sequence_push(toins_p, tr);
} else {
raptor_sequence_push(toins, tr);
}
}
}
q->num_vars = raptor_sequence_size(vars);
for (int i=0; i < q->num_vars; i++) {
rasqal_variable *v = raptor_sequence_get_at(vars, i);
fs_binding_add(q->bb[0], v, FS_RID_NULL, 1);
}
/* perform the WHERE match */
fs_query_process_pattern(q, uc->op->where, vars);
q->length = fs_binding_length(q->bb[0]);
for (int s=0; s<4; s++) {
vec[s] = fs_rid_vector_new(0);
}
for (int t=0; t<raptor_sequence_size(todel_p); t++) {
rasqal_triple *triple = raptor_sequence_get_at(todel_p, t);
for (int row=0; row < q->length; row++) {
delete_rasqal_triple(uc, vec, triple, row);
}
if (fs_rid_vector_length(vec[0]) > 1000) {
fsp_delete_quads_all(uc->link, vec);
}
}
if (fs_rid_vector_length(vec[0]) > 0) {
fsp_delete_quads_all(uc->link, vec);
}
for (int s=0; s<4; s++) {
//fs_rid_vector_print(vec[s], 0, stdout);
fs_rid_vector_free(vec[s]);
vec[s] = NULL;
}
for (int t=0; t<raptor_sequence_size(toins_p); t++) {
rasqal_triple *triple = raptor_sequence_get_at(toins_p, t);
for (int row=0; row < q->length; row++) {
insert_rasqal_triple(uc, triple, row);
}
}
/* must not free the rasqal_query */
q->rq = NULL;
fs_query_free(q);
uc->q = NULL;
} else {
todel = uc->op->delete_templates;
toins = uc->op->insert_templates;
}
#else
if (uc->op->where) {
fs_error(LOG_ERR, "DELETE/INSERT WHERE requires Rasqal 0.9.23 or newer");
add_message(uc, "DELETE/INSERT WHERE requires Rasqal 0.9.23 or newer", 0);
}
#endif
/* delete constant triples */
if (todel) {
for (int s=0; s<4; s++) {
vec[s] = fs_rid_vector_new(0);
}
for (int t=0; t<raptor_sequence_size(todel); t++) {
rasqal_triple *triple = raptor_sequence_get_at(todel, t);
if (any_vars(triple)) {
continue;
}
delete_rasqal_triple(uc, vec, triple, 0);
}
if (fs_rid_vector_length(vec[0]) > 0) {
fsp_delete_quads_all(uc->link, vec);
}
for (int s=0; s<4; s++) {
fs_rid_vector_free(vec[s]);
vec[s] = NULL;
}
}
/* insert constant triples */
if (toins) {
for (int t=0; t<raptor_sequence_size(toins); t++) {
rasqal_triple *triple = raptor_sequence_get_at(toins, t);
if (any_vars(triple)) {
continue;
}
insert_rasqal_triple(uc, triple, 0);
}
}
fs_hash_freshen();
return 0;
}
int fs_bind_cache_wrapper_intl(fs_query_state *qs, fs_query *q, int all,
int flags, fs_rid_vector *rids[4],
fs_rid_vector ***result, int offset, int limit)
{
g_static_mutex_lock(&qs->cache_mutex);
if (!qs->bind_cache) {
qs->bind_cache = calloc(CACHE_SIZE, sizeof(struct _fs_bind_cache));
}
if (fsp_acl_needs_reload(qs->link))
fs_acl_load_system_info(qs->link);
g_static_mutex_unlock(&qs->cache_mutex);
int slots = 0;
if (flags & FS_BIND_MODEL) slots++;
if (flags & FS_BIND_SUBJECT) slots++;
if (flags & FS_BIND_PREDICATE) slots++;
if (flags & FS_BIND_OBJECT) slots++;
/* check for no possible bindings */
for (int s=0; s<4; s++) {
if (rids[s]->length == 1 && rids[s]->data[0] == FS_RID_NULL) {
*result = calloc(slots, sizeof(fs_rid_vector));
for (int s=0; s<slots; s++) {
(*result)[s] = fs_rid_vector_new(0);
}
return 0;
}
}
int cachable = 0;
fs_rid cache_hash = 0;
fs_rid cache_key[4];
/* only consult the cache for optimasation levels 0-2 */
if (q && q->opt_level < 3) goto skip_cache;
if (q && q->qs && q->qs->cache_stats) q->qs->bind_hits++;
cachable = 1;
cache_hash += all + flags * 2 + offset * 256 + limit * 32768;
for (int s=0; s<4; s++) {
if (rids[s]->length == 1) {
cache_hash ^= (rids[s]->data[0] + s);
cache_key[s] = rids[s]->data[0];
} else if (rids[s]->length == 0) {
cache_key[s] = 0;
} else {
/* bind cache does not cache binds with any
slot containing multiple values */
cachable = 0;
break;
}
}
cache_hash %= (CACHE_SIZE - 1);
g_static_mutex_lock(&qs->cache_mutex);
if (cachable && qs->bind_cache[cache_hash].filled) {
int match = 1;
if (qs->bind_cache[cache_hash].all != all) match = 0;
if (qs->bind_cache[cache_hash].flags != flags) match = 0;
if (qs->bind_cache[cache_hash].offset != offset) match = 0;
if (qs->bind_cache[cache_hash].limit != limit) match = 0;
for (int s=0; s<4 && match; s++) {
if (cache_key[s] != qs->bind_cache[cache_hash].key[s]) {
match = 0;
}
}
if (match) {
*result = calloc(slots, sizeof(fs_rid_vector));
for (int s=0; s<slots; s++) {
(*result)[s] = fs_rid_vector_copy(qs->bind_cache[cache_hash].res[s]);
}
fsp_hit_limits_add(qs->link, qs->bind_cache[cache_hash].limited);
qs->bind_cache[cache_hash].hits++;
if (q && q->qs && q->qs->cache_stats) q->qs->bind_cache_success++;
g_static_mutex_unlock(&qs->cache_mutex);
return 0;
}
}
g_static_mutex_unlock(&qs->cache_mutex);
int ret;
skip_cache:;
int limited_before = fsp_hit_limits(qs->link);
if (all) {
ret = fsp_bind_limit_all(qs->link, flags, rids[0], rids[1], rids[2], rids[3], result, offset, limit);
} else {
ret = fsp_bind_limit_many(qs->link, flags, rids[0], rids[1], rids[2], rids[3], result, offset, limit);
}
int limited = fsp_hit_limits(qs->link) - limited_before;
if (ret) {
fs_error(LOG_ERR, "bind failed in '%s', %d segments gave errors",
fsp_kb_name(qs->link), ret);
exit(1);
}
int small = 1;
for (int s=0; s<slots; s++) {
if (fs_rid_vector_length((*result)[s]) > 10000) {
small = 0;
break;
}
}
if (cachable && small && slots > 0) {
g_static_mutex_lock(&qs->cache_mutex);
if (qs->bind_cache[cache_hash].filled == 1) {
for (int s=0; s<4; s++) {
fs_rid_vector_free(qs->bind_cache[cache_hash].res[s]);
qs->bind_cache[cache_hash].res[s] = NULL;
}
}
qs->bind_cache[cache_hash].filled = 1;
qs->bind_cache[cache_hash].all = all;
qs->bind_cache[cache_hash].flags = flags;
qs->bind_cache[cache_hash].offset = offset;
qs->bind_cache[cache_hash].limit = limit;
qs->bind_cache[cache_hash].limited = limited;
for (int s=0; s<4; s++) {
qs->bind_cache[cache_hash].key[s] = cache_key[s];
if (s < slots) {
qs->bind_cache[cache_hash].res[s] = fs_rid_vector_copy((*result)[s]);
} else {
qs->bind_cache[cache_hash].res[s] = NULL;
}
}
g_static_mutex_unlock(&qs->cache_mutex);
}
return ret;
}
static int update_op(struct update_context *ct)
{
fs_rid_vector *vec[4];
switch (ct->op->type) {
case RASQAL_UPDATE_TYPE_UNKNOWN:
add_message(ct, "Unknown update operation", 0);
return 1;
case RASQAL_UPDATE_TYPE_CLEAR:
fs_clear(ct, (char *)raptor_uri_as_string(ct->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_CREATE:
return 0;
case RASQAL_UPDATE_TYPE_DROP:
fs_clear(ct, (char *)raptor_uri_as_string(ct->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_LOAD:
fs_load(ct, (char *)raptor_uri_as_string(ct->op->document_uri),
(char *)raptor_uri_as_string(ct->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_UPDATE:
break;
}
fs_hash_freshen();
raptor_sequence *todel = NULL;
raptor_sequence *toins = NULL;
if (ct->op->where) {
todel = raptor_new_sequence(NULL, NULL);
toins = raptor_new_sequence(NULL, NULL);
raptor_sequence *todel_p = raptor_new_sequence(NULL, NULL);
raptor_sequence *toins_p = raptor_new_sequence(NULL, NULL);
raptor_sequence *vars = raptor_new_sequence(NULL, NULL);
fs_query *q = calloc(1, sizeof(fs_query));
ct->q = q;
q->qs = ct->qs;
q->rq = ct->rq;
q->flags = FS_BIND_DISTINCT;
q->opt_level = 3;
q->soft_limit = -1;
q->segments = fsp_link_segments(ct->link);
q->link = ct->link;
q->bb[0] = fs_binding_new();
q->bt = q->bb[0];
/* add column to denote join ordering */
fs_binding_add(q->bb[0], "_ord", FS_RID_NULL, 0);
struct pattern_data pd = { .q = q, .vars = vars, .patterns = NULL, .fixed = NULL };
if (ct->op->delete_templates) {
pd.patterns = todel_p;
pd.fixed = todel;
for (int t=0; t<raptor_sequence_size(ct->op->delete_templates); t++) {
rasqal_graph_pattern *gp = raptor_sequence_get_at(ct->op->delete_templates, t);
assign_gp(gp, NULL, &pd);
}
}
if (ct->op->insert_templates) {
pd.patterns = toins_p;
pd.fixed = toins;
for (int t=0; t<raptor_sequence_size(ct->op->insert_templates); t++) {
rasqal_graph_pattern *gp = raptor_sequence_get_at(ct->op->insert_templates, t);
assign_gp(gp, NULL, &pd);
}
}
q->num_vars = raptor_sequence_size(vars);
for (int i=0; i < q->num_vars; i++) {
rasqal_variable *v = raptor_sequence_get_at(vars, i);
fs_binding *b = fs_binding_get(q->bb[0], (char *)v->name);
if (b) {
b->need_val = 1;
} else {
fs_binding_add(q->bb[0], (char *)v->name, FS_RID_NULL, 1);
}
}
fs_query_process_pattern(q, ct->op->where, vars);
q->length = fs_binding_length(q->bb[0]);
for (int s=0; s<4; s++) {
vec[s] = fs_rid_vector_new(0);
}
for (int t=0; t<raptor_sequence_size(todel_p); t++) {
rasqal_triple *triple = raptor_sequence_get_at(todel_p, t);
for (int row=0; row < q->length; row++) {
delete_rasqal_triple(ct, vec, triple, row);
if (fs_rid_vector_length(vec[0]) > 0) {
fsp_delete_quads_all(ct->link, vec);
}
}
}
for (int s=0; s<4; s++) {
//fs_rid_vector_print(vec[s], 0, stdout);
fs_rid_vector_free(vec[s]);
}
for (int t=0; t<raptor_sequence_size(toins_p); t++) {
rasqal_triple *triple = raptor_sequence_get_at(toins_p, t);
for (int row=0; row < q->length; row++) {
insert_rasqal_triple(ct, triple, row);
}
}
/* must not free the rasqal_query */
q->rq = NULL;
fs_query_free(q);
ct->q = NULL;
} else {