END_TEST
START_TEST (check_fs_rid_vector_sort)
{
fs_rid_vector *v = fs_rid_vector_new(0);
fail_unless(v != NULL, "fs_rid_vector is NULL");
int i=0;
for (i=100; i < 1e4; i++) {
if (i % 2) {
fs_rid_vector_append(v, i);
fs_rid_vector_append(v, i+3);
fs_rid_vector_append(v, i-10);
} else {
fs_rid_vector_append(v, i-1);
fs_rid_vector_append(v, i+5);
fs_rid_vector_append(v, i);
}
}
fs_rid_vector_sort(v);
fail_if(fs_rid_vector_length(v) != ((1e4 - 100) *3), "Length fail after sort");
for (i=100; i < 1e4; i++) {
fail_if ( !fs_rid_vector_contains(v, i), "Contain failed after sort");
}
for (i=0;i<fs_rid_vector_length(v) -1; i++) {
fail_if (v->data[i] > v->data[i+1], "Sort does not match.");
}
fs_rid_vector_free(v);
}
END_TEST
START_TEST (check_fs_rid_vector_uniq)
{
fs_rid_vector *v = fs_rid_vector_new(0);
fail_unless(v != NULL, "fs_rid_vector is NULL");
int i=0;
for (i=100; i < 500; i++) {
fs_rid_vector_append(v, i);
fs_rid_vector_append(v, i);
}
fs_rid_vector_append(v,FS_RID_NULL);
fail_if(fs_rid_vector_length(v) != (((500 - 100)*2) + 1), "fs_rid_vector_length failed");
fs_rid_vector_uniq(v, 0); //Not remove nulls
fail_if(fs_rid_vector_length(v) != ((500 - 100) + 1), "fs_rid_vector_length failed");
for (i=1; i < fs_rid_vector_length(v) - 1; i++) {
fail_if(v->data[i] == v->data[i-1], "unique values failed [%d %d]",v->data[i],v->data[i-1]);
}
fail_if(v->data[fs_rid_vector_length(v)-1] != FS_RID_NULL, "FS_RID_NULL isn't there.");
v->data[123] = FS_RID_NULL;
fs_rid_vector_uniq(v, 1); //remove nulls
fail_if(fs_rid_vector_length(v) != ((500 - 100) -1), "fs_rid_vector_length failed");
fail_if(v->data[fs_rid_vector_length(v)-1] == FS_RID_NULL, "FS_RID_NULL should not be there %llx",
v->data[fs_rid_vector_length(v)-1]);
fail_if(v->data[123] == FS_RID_NULL, "FS_RID_NULL should not be there %llx",
v->data[fs_rid_vector_length(v)-1]);
fs_rid_vector_free(v);
}
static unsigned char * handle_bind_next (fs_backend *be, fs_segment segment,
unsigned int length,
unsigned char *content)
{
unsigned char *reply;
if (segment > be->segments) {
fs_error(LOG_ERR, "invalid segment number: %d", segment);
return fsp_error_new(segment, "invalid segment number");
}
if (length != 8) {
fs_error(LOG_ERR, "bind_next(%d) wrong length %u", segment, length);
return fsp_error_new(segment, "wrong length");
}
unsigned int flags;
int count;
memcpy(&flags, content, sizeof (flags));
memcpy(&count, content + 4, sizeof (count));
fs_rid_vector **bindings;
bindings = fs_bind_next(be, segment, flags, count);
int k, cols = 0;
for (k = 0; k < 4; ++k) {
if (flags & 1 << k) cols++;
}
if (bindings == NULL) {
/* NULL => no match */
reply = message_new(FS_NO_MATCH, segment, 0);
cols = 0;
} else if (cols == 0) {
/* Zero columns => match with no binding */
reply = message_new(FS_BIND_LIST, segment, 0);
} else {
/* otherwise return bindings */
reply = message_new(FS_BIND_LIST, segment, bindings[0]->length * 8 * cols);
unsigned char *data = reply + FS_HEADER;
for (k= 0; k < cols; ++k) {
memcpy(data, bindings[k]->data, bindings[k]->length * 8);
data += bindings[k]->length * 8;
}
}
for (k = 0; k < cols; ++k) {
fs_rid_vector_free(bindings[k]);
}
free(bindings);
return reply;
}
fs_value fn_datatype(fs_query *q, fs_value a)
{
#if 0
printf("datatype(");
fs_value_print(a);
printf(")\n");
#endif
if (a.valid & fs_valid_bit(FS_V_TYPE_ERROR)) {
return a;
}
if (a.valid & fs_valid_bit(FS_V_RID) && FS_IS_URI_BN(a.rid)) {
return fs_value_error(FS_ERROR_INVALID_TYPE, NULL);
}
if (a.attr == FS_RID_NULL) {
return fs_value_error(FS_ERROR_INVALID_TYPE, NULL);
}
if (FS_IS_LITERAL(a.attr) && a.attr != fs_c.empty) {
return fs_value_error(FS_ERROR_INVALID_TYPE, NULL);
} else {
if (a.attr == fs_c.xsd_string || a.attr == fs_c.empty) {
return fs_value_uri(XSD_STRING);
} else if (a.attr == fs_c.xsd_double) {
return fs_value_uri(XSD_DOUBLE);
} else if (a.attr == fs_c.xsd_float) {
return fs_value_uri(XSD_FLOAT);
} else if (a.attr == fs_c.xsd_decimal) {
return fs_value_uri(XSD_DECIMAL);
} else if (a.attr == fs_c.xsd_integer) {
return fs_value_uri(XSD_INTEGER);
} else if (a.attr == fs_c.xsd_boolean) {
return fs_value_uri(XSD_BOOLEAN);
} else if (a.attr == fs_c.xsd_datetime) {
return fs_value_uri(XSD_DATETIME);
}
}
fs_rid_vector *r = fs_rid_vector_new(1);
r->data[0] = a.attr;
fs_resource res;
if (fs_query_link(q)) {
fsp_resolve(fs_query_link(q), FS_RID_SEGMENT(a.attr,
fsp_link_segments(fs_query_link(q))), r, &res);
fs_rid_vector_free(r);
return fs_value_uri(res.lex);
}
return fs_value_uri("error:unresloved");
}
void fs_binding_free(fs_binding *b)
{
if (!b) return;
for (int i=0; b[i].name; i++) {
g_free(b[i].name);
b[i].name = NULL;
fs_rid_vector_free(b[i].vals);
b[i].vals = NULL;
}
memset(b, 0, sizeof(fs_binding));
free(b);
}
static char *get_lex(fsp_link *link, fs_rid rid)
{
if (rid == FS_RID_NULL) return g_strdup("*");
const int segments = fsp_link_segments(link);
fs_resource res;
fs_rid_vector *rv = fs_rid_vector_new(1);
rv->data[0] = rid;
fsp_resolve(link, FS_RID_SEGMENT(rid, segments), rv, &res);
fs_rid_vector_free(rv);
if (!strncmp(RDF_NAMESPACE, res.lex, strlen(RDF_NAMESPACE))) {
char *new = g_strdup_printf("rdf:%s", res.lex + strlen(RDF_NAMESPACE));
g_free(res.lex);
return new;
}
/**
* It wraps up the bind operation to discard rows from the result that cannot be accessed.
*/
int fs_bind_cache_wrapper_intl_acl(fs_query_state *qs, fs_query *q, int all,
int flags, fs_rid_vector *rids[4],
fs_rid_vector ***result, int offset, int limit) {
int flags_copy = flags;
int ndiscarded = 0;
if (fsp_is_acl_enabled(qs->link)) {
flags = flags | FS_BIND_MODEL;
}
int ret = fs_bind_cache_wrapper_intl(qs, q, all, flags, rids, result, offset, limit);
if (fsp_is_acl_enabled(qs->link) && (*result)) {
unsigned char *rows_discarded = NULL;
/* TODO probably this can be done with one iteration of results */
fs_rid_set *inv_acl = no_access_for_user(qs->link->acl_system_info,q->apikey_rid);
ndiscarded = fs_mark_discard_rows((*result)[0], inv_acl, &rows_discarded);
if (inv_acl)
fs_rid_set_free(inv_acl);
int slots = fs_slots_n(flags_copy);
if (!(flags_copy & FS_BIND_MODEL) && (flags & FS_BIND_MODEL)) {
fs_rid_vector **result_copy = calloc(slots, sizeof(fs_rid_vector));
for (int i=0;i<slots;i++)
result_copy[i] = (*result)[i+1];
fs_rid_vector_free((*result)[0]);
free(*result);
*result = result_copy;
}
if (ndiscarded) {
fs_rid_vector **rows = *result;
int count = fs_rid_vector_length(rows[0]);
int shifts = 0;
for (int i=0; i < count; i++) {
for (int s=0;s<slots;s++)
rows[s]->data[i-shifts] = rows[s]->data[i];
if (!fs_bit_array_get(rows_discarded, i))
shifts++;
}
for (int s=0;s<slots;s++) {
rows[s]->length -= ndiscarded;
}
}
if (rows_discarded)
fs_bit_array_destroy(rows_discarded);
}
return ret - ndiscarded;
}
END_TEST
START_TEST (check_fs_rid_vector_truncate)
{
fs_rid_vector *v = fs_rid_vector_new(0);
fail_unless(v != NULL, "fs_rid_vector is NULL");
int i=0;
for (i=100; i < 1e4; i++) {
fs_rid_vector_append(v, i);
}
fs_rid_vector_truncate(v, 100);
fail_if(fs_rid_vector_length(v) != 100);
fail_if(v->data[fs_rid_vector_length(v)-1] != 199);
fs_rid_vector_truncate(v, 1);
fail_if(v->data[fs_rid_vector_length(v)-1] != 100);
fs_rid_vector_free(v);
}
fs_value fn_lang(fs_query *q, fs_value a)
{
if (a.valid & fs_valid_bit(FS_V_TYPE_ERROR)) {
return a;
}
if (a.valid & fs_valid_bit(FS_V_RID) && FS_IS_URI_BN(a.rid)) {
return fs_value_error(FS_ERROR_INVALID_TYPE, NULL);
}
if (a.attr == FS_RID_NULL) {
return fs_value_error(FS_ERROR_INVALID_TYPE, NULL);
}
if (FS_IS_URI(a.attr)) {
return fs_value_plain("");
} else {
if (a.attr == fs_c.lang_en) {
return fs_value_plain("en");
} else if (a.attr == fs_c.lang_fr) {
return fs_value_plain("fr");
} else if (a.attr == fs_c.lang_de) {
return fs_value_plain("de");
} else if (a.attr == fs_c.lang_es) {
return fs_value_plain("es");
} else if (a.attr == fs_c.empty) {
return fs_value_plain("");
}
}
fs_rid_vector *r = fs_rid_vector_new(1);
r->data[0] = a.attr;
fs_resource res;
if (fs_query_link(q)) {
fsp_resolve(fs_query_link(q), FS_RID_SEGMENT(a.attr,
fsp_link_segments(fs_query_link(q))), r, &res);
fs_rid_vector_free(r);
fs_query_add_freeable(q, res.lex);
return fs_value_plain(res.lex);
}
return fs_value_plain("???");
}
END_TEST
START_TEST (check_fs_rid_vector_grow)
{
fs_rid_vector *v = fs_rid_vector_new(0);
fail_unless(v != NULL, "fs_rid_vector is NULL");
int i=0;
for (i=100; i < 1e4; i++) {
fs_rid_vector_append(v, i);
}
fail_if(fs_rid_vector_length(v) != (1e4 - 100));
fs_rid_vector_grow(v, 2e4);
fail_if(fs_rid_vector_length(v) != 2e4);
for (i=0;i<2e4;i++) {
v->data[i] = i;
}
fail_if(v->size != 2e4);
fs_rid_vector_free(v);
}
int fs_query_cache_flush(fs_query_state *qs, int verbosity)
{
/* assumption: the cache is created once only, ie it can't be pulled out from under us */
if (!qs->bind_cache) return 1;
g_static_mutex_lock(&qs->cache_mutex);
for (int i=0; i<CACHE_SIZE; i++) {
if (qs->bind_cache[i].filled) {
if (verbosity > 0) {
printf("# cache entry %d\n", i);
printf("# hits=%d, all=%s, flags=%08x, offset=%d, limit=%d\n", qs->bind_cache[i].hits, qs->bind_cache[i].all ? "true" : "false", qs->bind_cache[i].flags, qs->bind_cache[i].offset, qs->bind_cache[i].limit);
printf("# bind(%016llx, %016llx, %016llx, %016llx)\n", qs->bind_cache[i].key[0], qs->bind_cache[i].key[1], qs->bind_cache[i].key[2], qs->bind_cache[i].key[3]);
}
qs->bind_cache[i].filled = 0;
qs->bind_cache[i].all = 0;
qs->bind_cache[i].flags = 0;
qs->bind_cache[i].offset = 0;
qs->bind_cache[i].limit = 0;
for (int s=0; s<4; s++) {
qs->bind_cache[i].key[s] = 0;
fs_rid_vector_free(qs->bind_cache[i].res[s]);
qs->bind_cache[i].res[s] = 0;
}
}
}
if (verbosity > 0) {
printf("# @resolver@ cache_stats hits %u l1 %u l2 %u fails %u (%.4f perc. success)\n",
qs->cache_hits,qs->cache_success_l1,qs->cache_success_l2,qs->cache_fail,
((((double)qs->cache_success_l1)+((double)qs->cache_success_l2))/((double)qs->cache_hits))*100.0);
printf("# @resolver@ cache_stats items pre cached %u calls %u elapse %.3f\n",
qs->pre_cache_total,qs->resolve_all_calls,qs->resolve_all_elapse);
printf("# @resolver@ cache_stats resolve single calls %u elapse %.3f\n",
qs->cache_fail,qs->resolve_unique_elapse);
}
g_static_mutex_unlock(&qs->cache_mutex);
return 0;
}
int fs_clear(struct update_context *uc, char *graphuri)
{
fs_rid_vector *mvec = fs_rid_vector_new(0);
fs_rid mrid;
if (graphuri) {
mrid = fs_hash_uri(graphuri);
} else {
graphuri = FS_DEFAULT_GRAPH;
mrid = fs_c.default_graph;
}
fs_rid_vector_append(mvec, mrid);
int errors = 0;
if (fsp_delete_model_all(uc->link, mvec)) {
errors++;
add_message(uc, g_strdup_printf("Error while trying to delete %s", graphuri), 1);
} else {
add_message(uc, g_strdup_printf("Deleted <%s>", graphuri), 1);
}
fs_rid_vector_free(mvec);
return errors;
}
xmlChar *get_uri(fsp_link *link, fs_rid rid)
{
if (cache[rid & CACHE_MASK].rid == rid) {
return (xmlChar *) cache[rid & CACHE_MASK].lex;
}
fs_rid_vector onerid = { .length = 1, .size = 1, .data = &rid };
fs_resource resource;
fsp_resolve(link, FS_RID_SEGMENT(rid, segments), &onerid, &resource);
return (xmlChar *) resource.lex;
}
xmlChar *get_attr(fsp_link *link, fs_rid rid)
{
if (attr_cache[rid & CACHE_MASK].rid == rid) {
return (xmlChar *) attr_cache[rid & CACHE_MASK].lex;
}
fs_rid_vector onerid = { .length = 1, .size = 1, .data = &rid };
fs_resource resource;
fsp_resolve(link, FS_RID_SEGMENT(rid, segments), &onerid, &resource);
memcpy(&attr_cache[rid & ATTR_CACHE_MASK], &resource, sizeof(fs_resource));
return (xmlChar *) resource.lex;
}
xmlChar *get_literal(fsp_link *link, fs_rid rid, fs_rid *attr)
{
if (cache[rid & CACHE_MASK].rid == rid) {
*attr = cache[rid & CACHE_MASK].attr;
return (xmlChar *) cache[rid & CACHE_MASK].lex;
}
fs_rid_vector onerid = { .length = 1, .size = 1, .data = &rid };
fs_resource resource;
fsp_resolve(link, FS_RID_SEGMENT(rid, segments), &onerid, &resource);
*attr = resource.attr;
return (xmlChar *) resource.lex;
}
void resolve_triples(fsp_link *link, fs_rid_vector **rids)
{
int quads = rids[0]->length;
fs_rid_vector *todo[segments];
fs_segment segment;
for (segment = 0; segment < segments; ++segment) {
todo[segment] = fs_rid_vector_new(0);
}
for (int c = 0; c < 3; ++c) {
for (int k = 0; k < quads; ++k) {
const fs_rid rid = rids[c]->data[k];
if (FS_IS_BNODE(rid) || cache[rid & CACHE_MASK].rid == rid) continue;
fs_rid_vector_append(todo[FS_RID_SEGMENT(rid, segments)], rid);
cache[rid & CACHE_MASK].rid = rid; /* well, it will be soon */
}
}
int length[segments];
fs_resource *resources[segments];
for (segment = 0; segment < segments; ++segment) {
length[segment] = todo[segment]->length;
resources[segment] = calloc(length[segment], sizeof(fs_resource));
}
fsp_resolve_all(link, todo, resources);
for (segment = 0; segment < segments; ++segment) {
fs_resource *res = resources[segment];
for (int k = 0; k < length[segment]; ++k) {
free(cache[res[k].rid & CACHE_MASK].lex);
memcpy(&cache[res[k].rid & CACHE_MASK], &res[k], sizeof(fs_resource));
}
fs_rid_vector_free(todo[segment]);
free(resources[segment]);
}
}
void dump_model(fsp_link *link, fs_rid model, xmlTextWriterPtr xml)
{
fs_rid_vector none = { .length = 0, .size = 0, .data = 0 };
fs_rid_vector one = { .length = 1, .size = 1, .data = &model };
fs_rid_vector **results;
double then; /* for time keeping */
then = fs_time();
fsp_bind_first_all(link, BIND_SPO, &one, &none, &none, &none, &results, QUAD_LIMIT);
time_bind_first += (fs_time() - then);
while (results != NULL) {
long length = results[0]->length;
if (length == 0) break;
then = fs_time();
resolve_triples(link, results);
time_resolving += (fs_time() - then);
then = fs_time();
for (int k = 0; k < length; ++k) {
xmlTextWriterStartElement(xml, (xmlChar *) "triple");
for (int r = 0; r < 3; ++r) {
fs_rid rid = results[r]->data[k];
if (FS_IS_BNODE(rid)) {
unsigned long long node = FS_BNODE_NUM(rid);
xmlTextWriterWriteFormatElement(xml, (xmlChar *) "id", "%llu", node);
} else if (FS_IS_URI(rid)) {
xmlChar *uri = get_uri(link, rid);
xmlTextWriterWriteElement(xml, (xmlChar *) "uri", uri);
} else if (FS_IS_LITERAL(rid)) {
fs_rid attr;
xmlChar *lex = get_literal(link, rid, &attr);
if (attr == fs_c.empty) {
xmlTextWriterWriteElement(xml, (xmlChar *) "plainLiteral", lex);
} else if (FS_IS_URI(attr)) {
xmlChar *type = get_uri(link, attr);
xmlTextWriterStartElement(xml, (xmlChar *) "typedLiteral");
xmlTextWriterWriteString(xml, (xmlChar *) lex);
xmlTextWriterWriteAttribute(xml, (xmlChar *) "datatype", type);
xmlTextWriterEndElement(xml);
} else if (FS_IS_LITERAL(attr)) {
xmlChar *lang = get_attr(link, attr);
xmlTextWriterStartElement(xml, (xmlChar *) "plainLiteral");
xmlTextWriterWriteAttribute(xml, (xmlChar *) "xml:lang", lang);
xmlTextWriterWriteString(xml, (xmlChar *) lex);
xmlTextWriterEndElement(xml);
}
}
}
xmlTextWriterEndElement(xml);
xmlTextWriterWriteString(xml, (xmlChar *) "\n");
}
time_write_out += (fs_time() - then);
fs_rid_vector_free(results[0]);
fs_rid_vector_free(results[1]);
fs_rid_vector_free(results[2]);
free(results);
then = fs_time();
fsp_bind_next_all(link, BIND_SPO, &results, QUAD_LIMIT);
time_bind_next += (fs_time() - then);
}
fsp_bind_done_all(link);
}
void dump_trix(fsp_link *link, xmlTextWriterPtr xml)
{
fs_rid_vector **models;
fs_rid_vector none = { .length = 0, .size = 0, .data = 0 };
fsp_bind_all(link, FS_BIND_DISTINCT | FS_BIND_MODEL | FS_BIND_BY_SUBJECT, &none, &none, &none, &none, &models);
fs_rid_vector_sort(models[0]);
fs_rid_vector_uniq(models[0], 1);
long length = models[0]->length;
for (int k = 0; k < length; ++k) {
fs_rid model = models[0]->data[k];
xmlChar *model_uri = get_uri(link, model);
xmlTextWriterStartElement(xml, (xmlChar *) "graph");
if (FS_IS_URI(model)) {
xmlTextWriterWriteElement(xml, (xmlChar *) "uri", model_uri);
} else {
fs_error(LOG_WARNING, "model %lld is not a URI", model);
}
dump_model(link, model, xml);
xmlTextWriterEndElement(xml);
xmlTextWriterWriteString(xml, (xmlChar *) "\n");
printf("%5d/%ld: %4.5f %4.5f %4.5f %4.5f\n", k + 1, length, time_resolving, time_bind_first, time_bind_next, time_write_out);
}
}
void dump_file(fsp_link *link, char *filename)
{
xmlTextWriterPtr xml = xmlNewTextWriterFilename(filename, TRUE);
if (!xml) {
fs_error(LOG_ERR, "Couldn't write output file, giving up");
exit(4);
}
xmlTextWriterStartDocument(xml, NULL, NULL, NULL);
xmlTextWriterStartElement(xml, (xmlChar *) "TriX");
dump_trix(link, xml);
xmlTextWriterEndDocument(xml); /* also closes TriX */
xmlFreeTextWriter(xml);
}
int main(int argc, char *argv[])
{
char *password = fsp_argv_password(&argc, argv);
if (argc != 3) {
fprintf(stderr, "%s revision %s\n", argv[0], FS_FRONTEND_VER);
fprintf(stderr, "Usage: %s <kbname> <uri>\n", argv[0]);
exit(1);
}
fsp_link *link = fsp_open_link(argv[1], password, FS_OPEN_HINT_RO);
if (!link) {
fs_error (LOG_ERR, "couldn't connect to “%s”", argv[1]);
exit(2);
}
fs_hash_init(fsp_hash_type(link));
segments = fsp_link_segments(link);
dump_file(link, argv[2]);
fsp_close_link(link);
}
int fs_copy(struct update_context *uc, char *from, char *to)
{
fs_rid_vector *mvec = fs_rid_vector_new(0);
fs_rid_vector *empty = fs_rid_vector_new(0);
fs_rid fromrid, torid;
if (from) {
fromrid = fs_hash_uri(from);
} else {
from = FS_DEFAULT_GRAPH;
fromrid = fs_c.default_graph;
}
if (to) {
torid = fs_hash_uri(to);
} else {
to = FS_DEFAULT_GRAPH;
torid = fs_c.default_graph;
}
if (fromrid == torid) {
/*don't need to do anything */
fs_rid_vector_free(mvec);
fs_rid_vector_free(empty);
add_message(uc, g_strdup_printf("Copied <%s> to <%s>", from, to), 1);
add_message(uc, "0 triples added, 0 removed", 0);
return 0;
}
fs_rid_vector_append(mvec, fromrid);
/* search for all the triples in from */
fs_rid_vector **results;
fs_rid_vector *slot[4] = { mvec, empty, empty, empty };
/* see if there's any data in <from> */
fs_bind_cache_wrapper(uc->qs, NULL, 1, FS_BIND_BY_SUBJECT | FS_BIND_SUBJECT,
slot, &results, -1, 1);
if (!results || results[0]->length == 0) {
if (results) {
fs_rid_vector_free(results[0]);
free(results);
}
fs_rid_vector_free(mvec);
fs_rid_vector_free(empty);
add_message(uc, g_strdup_printf("<%s> is empty, not copying", from), 1);
return 1;
}
fs_rid_vector_free(results[0]);
free(results);
/* get the contents of <from> */
fs_bind_cache_wrapper(uc->qs, NULL, 1, FS_BIND_BY_SUBJECT | FS_BIND_SUBJECT | FS_BIND_PREDICATE | FS_BIND_OBJECT,
slot, &results, -1, -1);
/* map old bnodes to new ones */
map_bnodes(uc, results[0]);
map_bnodes(uc, results[1]);
map_bnodes(uc, results[2]);
/* delete <to> */
mvec->data[0] = torid;
if (fsp_delete_model_all(uc->link, mvec)) {
fs_rid_vector_free(mvec);
fs_rid_vector_free(empty);
add_message(uc, g_strdup_printf("Error while trying to delete %s", to), 1);
return 1;
}
fs_rid_vector_free(mvec);
fs_rid_vector_free(empty);
/* insert <to> */
fs_resource tores;
tores.lex = to;
tores.attr= FS_RID_NULL;
tores.rid = torid;
fsp_res_import(uc->link, FS_RID_SEGMENT(torid, uc->segments), 1, &tores);
insert_triples(uc, torid, results[0], results[1], results[2]);
add_message(uc, g_strdup_printf("Copied <%s> to <%s>", from, to), 1);
add_message(uc, g_strdup_printf("%d triples added, ?? removed", results[0]->length), 1);
for (int i=0; i<3; i++) {
fs_rid_vector_free(results[i]);
}
free(results);
return 0;
}
int fs_add(struct update_context *uc, char *from, char *to)
{
fs_rid_vector *mvec = fs_rid_vector_new(0);
fs_rid_vector *empty = fs_rid_vector_new(0);
fs_rid fromrid, torid;
if (from) {
fromrid = fs_hash_uri(from);
} else {
from = FS_DEFAULT_GRAPH;
fromrid = fs_c.default_graph;
}
if (to) {
torid = fs_hash_uri(to);
} else {
to = FS_DEFAULT_GRAPH;
torid = fs_c.default_graph;
}
if (fromrid == torid) {
/*don't need to do anything */
add_message(uc, g_strdup_printf("Added <%s> to <%s>", from, to), 1);
add_message(uc, "0 triples added, 0 removed", 0);
return 0;
}
fs_rid_vector_append(mvec, fromrid);
int errors = 0;
/* search for all the triples in from */
fs_rid_vector **results;
fs_rid_vector *slot[4] = { mvec, empty, empty, empty };
fs_bind_cache_wrapper(uc->qs, NULL, 1, FS_BIND_BY_SUBJECT | FS_BIND_SUBJECT | FS_BIND_PREDICATE | FS_BIND_OBJECT,
slot, &results, -1, -1);
fs_rid_vector_free(mvec);
fs_rid_vector_free(empty);
if (!results || results[0]->length == 0) {
/* there's nothing to add */
if (results) {
for (int i=0; i<3; i++) {
fs_rid_vector_free(results[i]);
}
free(results);
}
add_message(uc, g_strdup_printf("Added <%s> to <%s>", from, to), 1);
add_message(uc, "0 triples added, 0 removed", 0);
return 0;
}
map_bnodes(uc, results[0]);
map_bnodes(uc, results[1]);
map_bnodes(uc, results[2]);
fs_resource tores;
tores.lex = to;
tores.attr= FS_RID_NULL;
tores.rid = torid;
fsp_res_import(uc->link, FS_RID_SEGMENT(torid, uc->segments), 1, &tores);
insert_triples(uc, torid, results[0], results[1], results[2]);
add_message(uc, g_strdup_printf("Added <%s> to <%s>", from, to), 1);
add_message(uc, g_strdup_printf("%d triples added, 0 removed", results[0]->length), 1);
for (int i=0; i<3; i++) {
fs_rid_vector_free(results[i]);
}
free(results);
return errors;
}
static int update_op(struct update_context *uc)
{
fs_rid_vector *vec[4];
switch (uc->op->type) {
case RASQAL_UPDATE_TYPE_UNKNOWN:
add_message(uc, "Unknown update operation", 0);
return 1;
case RASQAL_UPDATE_TYPE_CLEAR:
fs_clear(uc, graph_arg(uc->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_CREATE:
return 0;
case RASQAL_UPDATE_TYPE_DROP:
fs_clear(uc, graph_arg(uc->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_LOAD:
fs_load(uc, graph_arg(uc->op->document_uri),
graph_arg(uc->op->graph_uri));
return 0;
#if RASQAL_VERSION >= 924
case RASQAL_UPDATE_TYPE_ADD:
fs_add(uc, graph_arg(uc->op->graph_uri),
graph_arg(uc->op->document_uri));
return 0;
case RASQAL_UPDATE_TYPE_MOVE:
fs_move(uc, graph_arg(uc->op->graph_uri),
graph_arg(uc->op->document_uri));
return 0;
case RASQAL_UPDATE_TYPE_COPY:
fs_copy(uc, graph_arg(uc->op->graph_uri),
graph_arg(uc->op->document_uri));
return 0;
#endif
case RASQAL_UPDATE_TYPE_UPDATE:
break;
}
fs_hash_freshen();
raptor_sequence *todel = NULL;
raptor_sequence *toins = NULL;
if (uc->op->delete_templates && !uc->op->where) {
int where = 0;
/* check to see if it's a DELETE WHERE { } */
for (int t=0; t<raptor_sequence_size(uc->op->delete_templates); t++) {
rasqal_triple *tr = raptor_sequence_get_at(uc->op->delete_templates, t);
if (any_vars(tr)) {
where = 1;
break;
}
}
if (where) {
fs_error(LOG_ERR, "DELETE WHERE { x } not yet supported");
add_message(uc, "DELETE WHERE { x } not yet supported, use DELETE { x } WHERE { x }", 0);
return 1;
}
}
#if RASQAL_VERSION >= 923
if (uc->op->where) {
todel = raptor_new_sequence(NULL, NULL);
toins = raptor_new_sequence(NULL, NULL);
raptor_sequence *todel_p = raptor_new_sequence(NULL, NULL);
raptor_sequence *toins_p = raptor_new_sequence(NULL, NULL);
raptor_sequence *vars = raptor_new_sequence(NULL, NULL);
fs_query *q = calloc(1, sizeof(fs_query));
uc->q = q;
q->qs = uc->qs;
q->rq = uc->rq;
q->flags = FS_BIND_DISTINCT;
#ifdef DEBUG_MERGE
q->flags |= FS_QUERY_CONSOLE_OUTPUT;
#endif
q->boolean = 1;
q->opt_level = 3;
q->soft_limit = -1;
q->segments = fsp_link_segments(uc->link);
q->link = uc->link;
q->bb[0] = fs_binding_new();
q->bt = q->bb[0];
/* hashtable to hold runtime created resources */
q->tmp_resources = g_hash_table_new_full(fs_rid_hash, fs_rid_equal, g_free, fs_free_cached_resource);
/* add column to denote join ordering */
fs_binding_create(q->bb[0], "_ord", FS_RID_NULL, 0);
if (uc->op->delete_templates) {
for (int t=0; t<raptor_sequence_size(uc->op->delete_templates); t++) {
rasqal_triple *tr = raptor_sequence_get_at(uc->op->delete_templates, t);
if (any_vars(tr)) {
fs_check_cons_slot(q, vars, tr->subject);
fs_check_cons_slot(q, vars, tr->predicate);
fs_check_cons_slot(q, vars, tr->object);
raptor_sequence_push(todel_p, tr);
} else {
raptor_sequence_push(todel, tr);
}
}
}
if (uc->op->insert_templates) {
for (int t=0; t<raptor_sequence_size(uc->op->insert_templates); t++) {
rasqal_triple *tr = raptor_sequence_get_at(uc->op->insert_templates, t);
if (any_vars(tr)) {
fs_check_cons_slot(q, vars, tr->subject);
fs_check_cons_slot(q, vars, tr->predicate);
fs_check_cons_slot(q, vars, tr->object);
raptor_sequence_push(toins_p, tr);
} else {
raptor_sequence_push(toins, tr);
}
}
}
q->num_vars = raptor_sequence_size(vars);
for (int i=0; i < q->num_vars; i++) {
rasqal_variable *v = raptor_sequence_get_at(vars, i);
fs_binding_add(q->bb[0], v, FS_RID_NULL, 1);
}
/* perform the WHERE match */
fs_query_process_pattern(q, uc->op->where, vars);
q->length = fs_binding_length(q->bb[0]);
for (int s=0; s<4; s++) {
vec[s] = fs_rid_vector_new(0);
}
for (int t=0; t<raptor_sequence_size(todel_p); t++) {
rasqal_triple *triple = raptor_sequence_get_at(todel_p, t);
for (int row=0; row < q->length; row++) {
delete_rasqal_triple(uc, vec, triple, row);
}
if (fs_rid_vector_length(vec[0]) > 1000) {
fsp_delete_quads_all(uc->link, vec);
}
}
if (fs_rid_vector_length(vec[0]) > 0) {
fsp_delete_quads_all(uc->link, vec);
}
for (int s=0; s<4; s++) {
//fs_rid_vector_print(vec[s], 0, stdout);
fs_rid_vector_free(vec[s]);
vec[s] = NULL;
}
for (int t=0; t<raptor_sequence_size(toins_p); t++) {
rasqal_triple *triple = raptor_sequence_get_at(toins_p, t);
for (int row=0; row < q->length; row++) {
insert_rasqal_triple(uc, triple, row);
}
}
/* must not free the rasqal_query */
q->rq = NULL;
fs_query_free(q);
uc->q = NULL;
} else {
todel = uc->op->delete_templates;
toins = uc->op->insert_templates;
}
#else
if (uc->op->where) {
fs_error(LOG_ERR, "DELETE/INSERT WHERE requires Rasqal 0.9.23 or newer");
add_message(uc, "DELETE/INSERT WHERE requires Rasqal 0.9.23 or newer", 0);
}
#endif
/* delete constant triples */
if (todel) {
for (int s=0; s<4; s++) {
vec[s] = fs_rid_vector_new(0);
}
for (int t=0; t<raptor_sequence_size(todel); t++) {
rasqal_triple *triple = raptor_sequence_get_at(todel, t);
if (any_vars(triple)) {
continue;
}
delete_rasqal_triple(uc, vec, triple, 0);
}
if (fs_rid_vector_length(vec[0]) > 0) {
fsp_delete_quads_all(uc->link, vec);
}
for (int s=0; s<4; s++) {
fs_rid_vector_free(vec[s]);
vec[s] = NULL;
}
}
/* insert constant triples */
if (toins) {
for (int t=0; t<raptor_sequence_size(toins); t++) {
rasqal_triple *triple = raptor_sequence_get_at(toins, t);
if (any_vars(triple)) {
continue;
}
insert_rasqal_triple(uc, triple, 0);
}
}
fs_hash_freshen();
return 0;
}
/* Read runtime.info and metadata.nt to fill in info for a kb.
* Leave ipaddr unset, caller can set if needed.
*
* Returns 0 on normal operation, -1 on error
*
* err is set to one of:
* ADM_ERR_SEE_ERRNO - check errno to find error
* ADM_ERR_GENERIC - usually std lib error where errno not set
* ADM_ERR_KB_NOT_EXISTS - KB requested does not exist
* ADM_ERR_KB_GET_INFO - KB exists, but runtime/metadata unreadable
* ADM_ERR_OK - no errors
*/
int fsab_kb_info_init(fsa_kb_info *ki, const unsigned char *kb_name, int *err)
{
fsa_error(LOG_DEBUG, "init kb info for '%s'", kb_name);
FILE *ri_file;
int len, rv;
char *path;
fs_metadata *md;
struct stat info;
ki->name = (unsigned char *)strdup((char *)kb_name);
/* check if kb exists */
len = (strlen(FS_KB_DIR)-2) + strlen((char *)kb_name) + 1;
path = (char *)malloc(len * sizeof(char));
if (path == NULL) {
errno = ENOMEM;
*err = ADM_ERR_SEE_ERRNO;
return -1;
}
/* generate full path to kb dir */
rv = sprintf(path, FS_KB_DIR, kb_name);
if (rv < 0) {
*err = ADM_ERR_GENERIC;
fsa_error(LOG_DEBUG, "sprintf failed");
free(path);
return -1;
}
rv = stat(path, &info);
free(path);
if (rv == -1) {
if (errno == ENOENT) {
/* not an error, return empty kb info, but let caller know */
fsa_error(LOG_DEBUG, "kb '%s' does not exist", kb_name);
*err = ADM_ERR_KB_NOT_EXISTS;
return 0;
}
else {
fsa_error(LOG_DEBUG,
"stat error for kb '%s': %s", kb_name, strerror(errno));
*err = ADM_ERR_SEE_ERRNO;
return -1;
}
}
/* alloc mem for string path to runtime.info */
len = (strlen(FS_RI_FILE)-2) + strlen((char *)kb_name) + 1;
path = (char *)malloc(len * sizeof(char));
if (path == NULL) {
errno = ENOMEM;
*err = ADM_ERR_SEE_ERRNO;
return -1;
}
/* generate full path to runtime.info */
rv = sprintf(path, FS_RI_FILE, kb_name);
if (rv < 0) {
*err = ADM_ERR_GENERIC;
fsa_error(LOG_DEBUG, "sprintf failed");
free(path);
return -1;
}
/* attempt to open file for reading, ignore failures, but log them */
ri_file = fopen(path, "r");
if (ri_file == NULL) {
fsa_error(LOG_ERR, "failed to read runtime info file at '%s': %s",
path, strerror(errno));
*err = ADM_ERR_KB_GET_INFO;
free(path);
}
else {
free(path);
/* check lock on file, and ignore if not locked - info is stale */
struct flock ri_lock;
int fd = fileno(ri_file);
ri_lock.l_type = F_WRLCK; /* write lock */
ri_lock.l_whence = SEEK_SET; /* l_start begins at start of file */
ri_lock.l_start = 0; /* offset from whence */
ri_lock.l_len = 0; /* until EOF */
rv = fcntl(fd, F_GETLK, &ri_lock);
if (rv == -1) {
fsa_error(LOG_CRIT, "fnctl locking error: %s", strerror(errno));
fclose(ri_file);
*err = ADM_ERR_KB_GET_INFO;
return -1;
}
if (ri_lock.l_type == F_WRLCK) {
/* file locked, so use info */
int port, pid;
ki->pid = ri_lock.l_pid;
rv = fscanf(ri_file, "%d %d", &pid, &port);
if (rv == 0 || rv == EOF) {
fsa_error(LOG_CRIT,
"bad data in runtime info file, fscanf failed");
fclose(ri_file);
*err = ADM_ERR_KB_GET_INFO;
return -1;
}
else {
/* file locked and contains running port and pid */
ki->port = port;
ki->status = KB_STATUS_RUNNING;
}
}
else if (ri_lock.l_type == F_UNLCK) {
/* file readable, but not locked */
ki->status = KB_STATUS_STOPPED;
}
fclose(ri_file);
}
/* pull data from metadata.nt */
md = fs_metadata_open((char *)kb_name);
if (md != NULL) {
ki->num_segments =
(uint16_t)atoi(fs_metadata_get_string(md, FS_MD_SEGMENTS, "0"));
fs_rid_vector *vec = fs_metadata_get_int_vector(md, FS_MD_SEGMENT_P);
fs_rid_vector_sort(vec);
/* segment ID and max segments should be 256, but allow 65536
to allow for value to be upped in #define */
ki->p_segments_len = (uint16_t)vec->length;
ki->p_segments_data =
(uint16_t *)malloc(ki->p_segments_len * sizeof(uint16_t));
for (int i = 0; i < vec->length; i++) {
ki->p_segments_data[i] = (uint16_t)vec->data[i];
}
fs_rid_vector_free(vec);
fs_metadata_close(md);
fsa_error(LOG_DEBUG, "metadata.nt read for kb %s", kb_name);
*err = ADM_ERR_OK;
}
else {
fsa_error(LOG_ERR, "unable to read metadata.nt for kb %s", kb_name);
*err = ADM_ERR_KB_GET_INFO;
}
return 0;
}
static int update_op(struct update_context *ct)
{
fs_rid_vector *vec[4];
switch (ct->op->type) {
case RASQAL_UPDATE_TYPE_UNKNOWN:
add_message(ct, "Unknown update operation", 0);
return 1;
case RASQAL_UPDATE_TYPE_CLEAR:
fs_clear(ct, (char *)raptor_uri_as_string(ct->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_CREATE:
return 0;
case RASQAL_UPDATE_TYPE_DROP:
fs_clear(ct, (char *)raptor_uri_as_string(ct->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_LOAD:
fs_load(ct, (char *)raptor_uri_as_string(ct->op->document_uri),
(char *)raptor_uri_as_string(ct->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_UPDATE:
break;
}
fs_hash_freshen();
raptor_sequence *todel = NULL;
raptor_sequence *toins = NULL;
if (ct->op->where) {
todel = raptor_new_sequence(NULL, NULL);
toins = raptor_new_sequence(NULL, NULL);
raptor_sequence *todel_p = raptor_new_sequence(NULL, NULL);
raptor_sequence *toins_p = raptor_new_sequence(NULL, NULL);
raptor_sequence *vars = raptor_new_sequence(NULL, NULL);
fs_query *q = calloc(1, sizeof(fs_query));
ct->q = q;
q->qs = ct->qs;
q->rq = ct->rq;
q->flags = FS_BIND_DISTINCT;
q->opt_level = 3;
q->soft_limit = -1;
q->segments = fsp_link_segments(ct->link);
q->link = ct->link;
q->bb[0] = fs_binding_new();
q->bt = q->bb[0];
/* add column to denote join ordering */
fs_binding_add(q->bb[0], "_ord", FS_RID_NULL, 0);
struct pattern_data pd = { .q = q, .vars = vars, .patterns = NULL, .fixed = NULL };
if (ct->op->delete_templates) {
pd.patterns = todel_p;
pd.fixed = todel;
for (int t=0; t<raptor_sequence_size(ct->op->delete_templates); t++) {
rasqal_graph_pattern *gp = raptor_sequence_get_at(ct->op->delete_templates, t);
assign_gp(gp, NULL, &pd);
}
}
if (ct->op->insert_templates) {
pd.patterns = toins_p;
pd.fixed = toins;
for (int t=0; t<raptor_sequence_size(ct->op->insert_templates); t++) {
rasqal_graph_pattern *gp = raptor_sequence_get_at(ct->op->insert_templates, t);
assign_gp(gp, NULL, &pd);
}
}
q->num_vars = raptor_sequence_size(vars);
for (int i=0; i < q->num_vars; i++) {
rasqal_variable *v = raptor_sequence_get_at(vars, i);
fs_binding *b = fs_binding_get(q->bb[0], (char *)v->name);
if (b) {
b->need_val = 1;
} else {
fs_binding_add(q->bb[0], (char *)v->name, FS_RID_NULL, 1);
}
}
fs_query_process_pattern(q, ct->op->where, vars);
q->length = fs_binding_length(q->bb[0]);
for (int s=0; s<4; s++) {
vec[s] = fs_rid_vector_new(0);
}
for (int t=0; t<raptor_sequence_size(todel_p); t++) {
rasqal_triple *triple = raptor_sequence_get_at(todel_p, t);
for (int row=0; row < q->length; row++) {
delete_rasqal_triple(ct, vec, triple, row);
if (fs_rid_vector_length(vec[0]) > 0) {
fsp_delete_quads_all(ct->link, vec);
}
}
}
for (int s=0; s<4; s++) {
//fs_rid_vector_print(vec[s], 0, stdout);
fs_rid_vector_free(vec[s]);
}
for (int t=0; t<raptor_sequence_size(toins_p); t++) {
rasqal_triple *triple = raptor_sequence_get_at(toins_p, t);
for (int row=0; row < q->length; row++) {
insert_rasqal_triple(ct, triple, row);
}
}
/* must not free the rasqal_query */
q->rq = NULL;
fs_query_free(q);
ct->q = NULL;
} else {
/**
* It loads the acl system info from the system:config graph.
* Due to link->acl_system_info manipulation this function should be
* under mutex conditions.
*/
int fs_acl_load_system_info(fsp_link *link) {
if (!fsp_acl_needs_reload(link))
return 0;
int flags = FS_BIND_SUBJECT | FS_BIND_PREDICATE | FS_BIND_OBJECT | FS_BIND_BY_SUBJECT;
fs_rid_vector *mrids = fs_rid_vector_new_from_args(1, fs_c.system_config);
fs_rid_vector *srids = fs_rid_vector_new(0);
fs_rid_vector *prids = fs_rid_vector_new_from_args(2, fs_c.fs_acl_admin, fs_c.fs_acl_access_by);
fs_rid_vector *orids = fs_rid_vector_new(0);
fs_rid_vector **result = NULL;
fsp_bind_limit_all(link, flags, mrids, srids, prids, orids, &result, -1, -1);
fs_rid_vector_free(mrids);
fs_rid_vector_free(srids);
fs_rid_vector_free(prids);
fs_rid_vector_free(orids);
int admin_users_count = 0;
fs_acl_system_info *acl_system_info = link->acl_system_info;
if (result && result[0]) {
if (!acl_system_info->acl_graph_hash || acl_system_info->admin_user_set)
link->acl_system_info = acl_system_info;
if (acl_system_info->acl_graph_hash) {
g_hash_table_steal(acl_system_info->acl_graph_hash, &fs_c.system_config);
g_hash_table_destroy(acl_system_info->acl_graph_hash);
acl_system_info->acl_graph_hash = NULL;
}
acl_system_info->acl_graph_hash = g_hash_table_new_full(fs_rid_hash,fs_rid_equal,
acl_key_destroyed, acl_value_destroyed);
if (acl_system_info->admin_user_set) {
fs_rid_set_free(acl_system_info->admin_user_set);
acl_system_info->admin_user_set = NULL;
}
acl_system_info->admin_user_set = fs_rid_set_new();
for (int row = 0; row < result[0]->length; row++) {
if(result[1]->data[row] == fs_c.fs_acl_access_by) {
/* if pred is acl_access_by then subject is the graph and object is the user rid */
gpointer users_set_ref = NULL;
fs_rid_set *users_set = NULL;
if (!(users_set_ref=g_hash_table_lookup(acl_system_info->acl_graph_hash, &result[0]->data[row]))) {
users_set = fs_rid_set_new();
fs_rid *rid_graph = malloc(sizeof(fs_rid));
*rid_graph = result[0]->data[row];
g_hash_table_insert(acl_system_info->acl_graph_hash, rid_graph, users_set);
} else
users_set = (fs_rid_set *) users_set_ref;
fs_rid_set_add(users_set, result[2]->data[row]);
} else if (result[1]->data[row] == fs_c.fs_acl_admin) {
/* if admin predicate then object contains the admin user rid id */
fs_rid_set_add(acl_system_info->admin_user_set, result[2]->data[row]);
admin_users_count++;
}
}
if (admin_users_count == 0) {
fs_error(LOG_ERR,"Added default admin user %s",FS_ACL_DEFAULT_ADMIN);
fs_rid_set_add(acl_system_info->admin_user_set, fs_c.fs_acl_default_admin);
}
/* only admin users can access system:config */
g_hash_table_insert(acl_system_info->acl_graph_hash, &fs_c.system_config, acl_system_info->admin_user_set);
}
fsp_acl_reloaded(link);
if (result) {
for (int i=0;i<3;i++) {
fs_rid_vector_free(result[i]);
}
free(result);
}
return 1;
}
/* return to = from [X] to, this is used to perform joins inside blocks, it
* saves allocations by doing most operations inplace, unlike fs_binding_join */
void fs_binding_merge(fs_query *q, int block, fs_binding *from, fs_binding *to)
{
fs_binding *inter_f = NULL; /* the intersecting column */
fs_binding *inter_t = NULL; /* the intersecting column */
for (int i=0; from[i].name; i++) {
from[i].sort = 0;
to[i].sort = 0;
}
int used = 0;
for (int i=1; from[i].name; i++) {
if (!from[i].bound || !to[i].bound) continue;
if (from[i].used) used++;
if (from[i].bound && to[i].bound) {
inter_f = from+i;
inter_t = to+i;
from[i].sort = 1;
to[i].sort = 1;
#ifdef DEBUG_MERGE
printf("@@ join on %s\n", to[i].name);
#endif
}
}
/* from and to bound variables do not intersect, we can just dump results,
under some circustances we need to do a combinatorial explosion */
if (!inter_f && (fs_binding_length(from) == 0)) {
const int length_f = fs_binding_length(from);
const int length_t = fs_binding_length(to);
for (int i=1; from[i].name; i++) {
if (to[i].bound && !from[i].bound) {
if (from[i].vals) {
fs_rid_vector_free(from[i].vals);
}
from[i].vals = fs_rid_vector_new(length_f);
for (int d=0; d<length_f; d++) {
from[i].vals->data[d] = FS_RID_NULL;
}
from[i].bound = 1;
}
if (!from[i].bound) continue;
if (!to[i].bound) {
if (to[i].vals) {
fs_rid_vector_free(to[i].vals);
}
to[i].vals = fs_rid_vector_new(length_t);
for (int d=0; d<length_t; d++) {
to[i].vals->data[d] = FS_RID_NULL;
}
}
fs_rid_vector_append_vector(to[i].vals, from[i].vals);
to[i].bound = 1;
}
#ifdef DEBUG_MERGE
printf("append all, result:\n");
fs_binding_print(to, stdout);
#endif
return;
}
/* If were running in restricted mode, truncate the binding tables */
if (q->flags & FS_QUERY_RESTRICTED) {
fs_binding_truncate(from, q->soft_limit);
fs_binding_truncate(to, q->soft_limit);
}
int length_t = fs_binding_length(to);
int length_f = fs_binding_length(from);
/* ms8: this list keeps track of the vars to replace */
GList *rep_list = NULL;
for (int i=1; to[i].name; i++) {
if (to+i == inter_t || to[i].used || to[i].bound) {
/* do nothing */
#if DEBUG_MERGE > 1
printf("@@ preserve %s\n", to[i].name);
#endif
} else if (from[i].bound && !to[i].bound) {
#if DEBUG_MERGE > 1
printf("@@ replace %s\n", from[i].name);
#endif
to[i].bound = 1;
if (to[i].vals) {
if (to[i].vals->length != length_t) {
fs_rid_vector_free(to[i].vals);
to[i].vals = fs_rid_vector_new(length_t);
}
} else {
to[i].vals = fs_rid_vector_new(length_t);
}
for (int d=0; d<length_t; d++) {
to[i].vals->data[d] = FS_RID_NULL;
}
rep_list = g_list_append(rep_list, GINT_TO_POINTER(i));
}
}
/* sort the two sets of bindings so they can be merged linearly */
if (inter_f) {
fs_binding_sort(from);
fs_binding_sort(to);
} else {
/* make sure the tables are not marked sorted */
from[0].vals->length = 0;
to[0].vals->length = 0;
}
#ifdef DEBUG_MERGE
printf("old: %d bindings\n", fs_binding_length(from));
fs_binding_print(from, stdout);
printf("new: %d bindings\n", fs_binding_length(to));
fs_binding_print(to, stdout);
#endif
int fpos = 0;
int tpos = 0;
while (fpos < length_f || tpos < length_t) {
if (q->flags & FS_QUERY_RESTRICTED &&
fs_binding_length(to) >= q->soft_limit) {
char *msg = g_strdup("some results have been dropped to prevent overunning time allocation");
q->warnings = g_slist_prepend(q->warnings, msg);
break;
}
int cmp;
cmp = binding_row_compare(q, from, to, fpos, tpos, length_f, length_t);
if (cmp == 0) {
/* both rows match */
int fp, tp = tpos;
for (fp = fpos; binding_row_compare(q, from, to, fp, tpos, length_f, length_t) == 0; fp++) {
#if DEBUG_MERGE > 1
if (fp == DEBUG_CUTOFF) {
printf("...\n");
}
#endif
for (tp = tpos; 1; tp++) {
if (binding_row_compare(q, from, to, fp, tp, length_f, length_t) == 0) {
#if DEBUG_MERGE > 1
if (fp < DEBUG_CUTOFF) {
printf("STEP %d, %d ", fp-fpos, tp-tpos);
}
#endif
if (fp == fpos) {
#if DEBUG_MERGE > 1
if (fp < DEBUG_CUTOFF) {
if (inter_f) {
printf("REPL %llx\n", inter_f->vals->data[fp]);
} else {
printf("REPL ???\n");
}
}
#endif
for (int c=1; to[c].name; c++) {
if (!from[c].bound && !to[c].bound) continue;
if (from[c].bound && table_value(from, c, fp) == FS_RID_NULL) {
continue;
}
if (from[c].bound && fp < from[c].vals->length) {
long wrow = to[0].vals->length ? to[0].vals->data[tp] : tp;
to[c].vals->data[wrow] = table_value(from, c, fp);
if (to[c].vals->length <= tp) {
to[c].vals->length = tp+1;
}
}
}
} else {
#if DEBUG_MERGE > 1
if (fp < DEBUG_CUTOFF) {
printf("ADD\n");
}
#endif
for (int c=1; to[c].name; c++) {
if (!from[c].bound && !to[c].bound) continue;
if (from[c].bound && fp < from[c].vals->length) {
fs_rid_vector_append(to[c].vals, table_value(from, c, fp));
} else {
fs_rid_vector_append(to[c].vals, table_value(to, c, tp));
}
}
}
} else {
break;
}
}
}
tpos = tp;
fpos = fp;
} else if (cmp <= -1) {
fpos++;
} else if (cmp >= 1) {
tpos++;
} else {
fs_error(LOG_CRIT, "unknown compare state %d in binding", cmp);
}
}
/* clear the _ord columns */
from[0].vals->length = 0;
to[0].vals->length = 0;
/* ms8: INIT code to clean up rows that where not replaced */
if (rep_list) {
unsigned char *to_del = fs_new_bit_array(length_t);
int to_del_count = 0;
while(rep_list) {
int col_r = GPOINTER_TO_INT(rep_list->data);
rep_list = g_list_next(rep_list);
for (int d=0; d<length_t; d++) {
if (to[col_r].vals->data[d] == FS_RID_NULL) {
fs_bit_array_set(to_del, d, 0);
to_del_count++;
}
}
}
g_list_free(rep_list);
if (to_del_count) {
int vars = 0;
for (int i=1; to[i].name; i++)
vars++;
fs_rid_vector **clean = calloc(vars, sizeof(fs_rid_vector *));
for (int i=0;i<vars;i++)
clean[i] = fs_rid_vector_new(0);
for (int d = 0;d<length_t;d++) {
if (fs_bit_array_get(to_del,d)) {
for (int i=0;i<vars;i++) {
fs_rid_vector_append(clean[i],to[i+1].vals->data[d]);
}
}
}
for (int i=1;i<=vars;i++) {
free(to[i].vals->data);
to[i].vals->data = clean[i-1]->data;
to[i].vals->length = clean[i-1]->length;
to[i].vals->size = clean[i-1]->size;
free(clean[i-1]);
}
free(clean);
}
fs_bit_array_destroy(to_del);
}
/* ms8: END code to clean up rows that where not replaced */
#ifdef DEBUG_MERGE
printf("result: %d bindings\n", fs_binding_length(to));
fs_binding_print(to, stdout);
#endif
}
int fs_bind_cache_wrapper_intl(fs_query_state *qs, fs_query *q, int all,
int flags, fs_rid_vector *rids[4],
fs_rid_vector ***result, int offset, int limit)
{
g_static_mutex_lock(&qs->cache_mutex);
if (!qs->bind_cache) {
qs->bind_cache = calloc(CACHE_SIZE, sizeof(struct _fs_bind_cache));
}
if (fsp_acl_needs_reload(qs->link))
fs_acl_load_system_info(qs->link);
g_static_mutex_unlock(&qs->cache_mutex);
int slots = 0;
if (flags & FS_BIND_MODEL) slots++;
if (flags & FS_BIND_SUBJECT) slots++;
if (flags & FS_BIND_PREDICATE) slots++;
if (flags & FS_BIND_OBJECT) slots++;
/* check for no possible bindings */
for (int s=0; s<4; s++) {
if (rids[s]->length == 1 && rids[s]->data[0] == FS_RID_NULL) {
*result = calloc(slots, sizeof(fs_rid_vector));
for (int s=0; s<slots; s++) {
(*result)[s] = fs_rid_vector_new(0);
}
return 0;
}
}
int cachable = 0;
fs_rid cache_hash = 0;
fs_rid cache_key[4];
/* only consult the cache for optimasation levels 0-2 */
if (q && q->opt_level < 3) goto skip_cache;
if (q && q->qs && q->qs->cache_stats) q->qs->bind_hits++;
cachable = 1;
cache_hash += all + flags * 2 + offset * 256 + limit * 32768;
for (int s=0; s<4; s++) {
if (rids[s]->length == 1) {
cache_hash ^= (rids[s]->data[0] + s);
cache_key[s] = rids[s]->data[0];
} else if (rids[s]->length == 0) {
cache_key[s] = 0;
} else {
/* bind cache does not cache binds with any
slot containing multiple values */
cachable = 0;
break;
}
}
cache_hash %= (CACHE_SIZE - 1);
g_static_mutex_lock(&qs->cache_mutex);
if (cachable && qs->bind_cache[cache_hash].filled) {
int match = 1;
if (qs->bind_cache[cache_hash].all != all) match = 0;
if (qs->bind_cache[cache_hash].flags != flags) match = 0;
if (qs->bind_cache[cache_hash].offset != offset) match = 0;
if (qs->bind_cache[cache_hash].limit != limit) match = 0;
for (int s=0; s<4 && match; s++) {
if (cache_key[s] != qs->bind_cache[cache_hash].key[s]) {
match = 0;
}
}
if (match) {
*result = calloc(slots, sizeof(fs_rid_vector));
for (int s=0; s<slots; s++) {
(*result)[s] = fs_rid_vector_copy(qs->bind_cache[cache_hash].res[s]);
}
fsp_hit_limits_add(qs->link, qs->bind_cache[cache_hash].limited);
qs->bind_cache[cache_hash].hits++;
if (q && q->qs && q->qs->cache_stats) q->qs->bind_cache_success++;
g_static_mutex_unlock(&qs->cache_mutex);
return 0;
}
}
g_static_mutex_unlock(&qs->cache_mutex);
int ret;
skip_cache:;
int limited_before = fsp_hit_limits(qs->link);
if (all) {
ret = fsp_bind_limit_all(qs->link, flags, rids[0], rids[1], rids[2], rids[3], result, offset, limit);
} else {
ret = fsp_bind_limit_many(qs->link, flags, rids[0], rids[1], rids[2], rids[3], result, offset, limit);
}
int limited = fsp_hit_limits(qs->link) - limited_before;
if (ret) {
fs_error(LOG_ERR, "bind failed in '%s', %d segments gave errors",
fsp_kb_name(qs->link), ret);
exit(1);
}
int small = 1;
for (int s=0; s<slots; s++) {
if (fs_rid_vector_length((*result)[s]) > 10000) {
small = 0;
break;
}
}
if (cachable && small && slots > 0) {
g_static_mutex_lock(&qs->cache_mutex);
if (qs->bind_cache[cache_hash].filled == 1) {
for (int s=0; s<4; s++) {
fs_rid_vector_free(qs->bind_cache[cache_hash].res[s]);
qs->bind_cache[cache_hash].res[s] = NULL;
}
}
qs->bind_cache[cache_hash].filled = 1;
qs->bind_cache[cache_hash].all = all;
qs->bind_cache[cache_hash].flags = flags;
qs->bind_cache[cache_hash].offset = offset;
qs->bind_cache[cache_hash].limit = limit;
qs->bind_cache[cache_hash].limited = limited;
for (int s=0; s<4; s++) {
qs->bind_cache[cache_hash].key[s] = cache_key[s];
if (s < slots) {
qs->bind_cache[cache_hash].res[s] = fs_rid_vector_copy((*result)[s]);
} else {
qs->bind_cache[cache_hash].res[s] = NULL;
}
}
g_static_mutex_unlock(&qs->cache_mutex);
}
return ret;
}
int main(int argc, char *argv[])
{
int verbosity = 0;
int dryrun = 0;
char *password = NULL;
char *format = "auto";
FILE *msg = stderr;
char *optstring = "am:M:vnf:";
int c, opt_index = 0, help = 0;
int files = 0, adding = 0;
char *kb_name = NULL;
char *model[argc], *uri[argc];
char *model_default = NULL;
password = fsp_argv_password(&argc, argv);
static struct option long_options[] = {
{ "add", 0, 0, 'a' },
{ "model", 1, 0, 'm' },
{ "model-default", 1, 0, 'M' },
{ "verbose", 0, 0, 'v' },
{ "dryrun", 0, 0, 'n' },
{ "no-resources", 0, 0, 'R' },
{ "no-quads", 0, 0, 'Q' },
{ "format", 1, 0, 'f' },
{ "help", 0, 0, 'h' },
{ "version", 0, 0, 'V' },
{ 0, 0, 0, 0 }
};
for (int i= 0; i < argc; ++i) {
model[i] = NULL;
}
int help_return = 1;
while ((c = getopt_long (argc, argv, optstring, long_options, &opt_index)) != -1) {
if (c == 'm') {
model[files++] = optarg;
} else if (c == 'M') {
model_default = optarg;
} else if (c == 'v') {
verbosity++;
} else if (c == 'a') {
adding = 1;
} else if (c == 'n') {
dryrun |= FS_DRYRUN_DELETE | FS_DRYRUN_RESOURCES | FS_DRYRUN_QUADS;
} else if (c == 'R') {
dryrun |= FS_DRYRUN_RESOURCES;
} else if (c == 'Q') {
dryrun |= FS_DRYRUN_QUADS;
} else if (c == 'f') {
format = optarg;
} else if (c == 'h') {
help = 1;
help_return = 0;
} else if (c == 'V') {
printf("%s, built for 4store %s\n", argv[0], GIT_REV);
exit(0);
} else {
help = 1;
}
}
if (verbosity > 0) {
if (dryrun & FS_DRYRUN_DELETE) {
printf("warning: not deleting old model\n");
}
if (dryrun & FS_DRYRUN_RESOURCES) {
printf("warning: not importing resource nodes\n");
}
if (dryrun & FS_DRYRUN_QUADS) {
printf("warning: not importing quad graph\n");
}
}
files = 0;
for (int k = optind; k < argc; ++k) {
if (!kb_name) {
kb_name = argv[k];
} else {
if (strchr(argv[k], ':')) {
uri[files] = g_strdup(argv[k]);
} else {
uri[files] = (char *)raptor_uri_filename_to_uri_string(argv[k]);
}
if (!model[files]) {
if (!model_default) {
model[files] = uri[files];
} else {
model[files] = model_default;
}
}
files++;
}
}
raptor_world *rw = raptor_new_world();
if (help || !kb_name || files == 0) {
fprintf(stdout, "%s revision %s\n", argv[0], FS_FRONTEND_VER);
fprintf(stdout, "Usage: %s <kbname> <rdf file/URI> ...\n", argv[0]);
fprintf(stdout, " -v --verbose increase verbosity (can repeat)\n");
fprintf(stdout, " -a --add add data to models instead of replacing\n");
fprintf(stdout, " -m --model specify a model URI for the next RDF file\n");
fprintf(stdout, " -M --model-default specify a model URI for all RDF files\n");
fprintf(stdout, " -f --format specify an RDF syntax for the import\n");
fprintf(stdout, "\n available formats are:\n");
for (unsigned int i=0; 1; i++) {
const raptor_syntax_description *desc =
raptor_world_get_parser_description(rw, i);
if (!desc) {
break;
}
fprintf(stdout, " %12s - %s\n", desc->names[0], desc->label);
}
exit(help_return);
}
fsp_syslog_enable();
fsplink = fsp_open_link(kb_name, password, FS_OPEN_HINT_RW);
if (!fsplink) {
fs_error (LOG_ERR, "couldn't connect to “%s”", kb_name);
exit(2);
}
const char *features = fsp_link_features(fsplink);
int has_o_index = !(strstr(features, "no-o-index")); /* tweak */
fs_hash_init(fsp_hash_type(fsplink));
const int segments = fsp_link_segments(fsplink);
int total_triples = 0;
fs_import_timing timing[segments];
for (int seg = 0; seg < segments; seg++) {
fsp_get_import_times(fsplink, seg, &timing[seg]);
}
gettimeofday(&then, 0);
if (fsp_start_import_all(fsplink)) {
fs_error(LOG_ERR, "aborting import");
exit(3);
}
#if 0
printf("press enter\n");
char foo;
read(0, &foo, 1);
#endif
fs_rid_vector *mvec = fs_rid_vector_new(0);
for (int f= 0; f < files; ++f) {
fs_rid muri = fs_hash_uri(model[f]);
fs_rid_vector_append(mvec, muri);
}
if (!adding) {
if (verbosity) {
printf("removing old data\n");
fflush(stdout);
}
if (!(dryrun & FS_DRYRUN_DELETE)) {
if (fsp_delete_model_all(fsplink, mvec)) {
fs_error(LOG_ERR, "model delete failed");
return 1;
}
for (int i=0; i<mvec->length; i++) {
if (mvec->data[i] == fs_c.system_config) {
fs_import_reread_config();
}
}
}
fsp_new_model_all(fsplink, mvec);
}
fs_rid_vector_free(mvec);
gettimeofday(&then_last, 0);
for (int f = 0; f < files; ++f) {
if (verbosity) {
printf("Reading <%s>\n", uri[f]);
if (strcmp(uri[f], model[f])) {
printf(" into <%s>\n", model[f]);
}
fflush(stdout);
}
fs_import(fsplink, model[f], uri[f], format, verbosity, dryrun, has_o_index, msg, &total_triples);
if (verbosity) {
fflush(stdout);
}
}
double sthen = fs_time();
int ret = fs_import_commit(fsplink, verbosity, dryrun, has_o_index, msg, &total_triples);
if (verbosity > 0) {
printf("Updating index\n");
fflush(stdout);
}
fsp_stop_import_all(fsplink);
if (verbosity > 0) {
printf("Index update took %f seconds\n", fs_time()-sthen);
}
if (!ret) {
gettimeofday(&now, 0);
double diff = (now.tv_sec - then.tv_sec) +
(now.tv_usec - then.tv_usec) * 0.000001;
if (verbosity && total_triples > 0) {
printf("Imported %d triples, average %d triples/s\n", total_triples,
(int)((double)total_triples/diff));
fflush(stdout);
}
}
if (verbosity > 1) {
printf("seg add_q\tadd_r\t\tcommit_q\tcommit_r\tremove\t\trebuild\t\twrite\n");
long long *tics = fsp_profile_write(fsplink);
for (int seg = 0; seg < segments; seg++) {
fs_import_timing newtimes;
fsp_get_import_times(fsplink, seg, &newtimes);
printf("%2d: %f\t%f\t%f\t%f\t%f\t%f\t%f\n", seg,
newtimes.add_s - timing[seg].add_s,
newtimes.add_r - timing[seg].add_r,
newtimes.commit_q - timing[seg].commit_q,
newtimes.commit_r - timing[seg].commit_r,
newtimes.remove - timing[seg].remove,
newtimes.rebuild - timing[seg].rebuild,
tics[seg] * 0.001);
}
}
fsp_close_link(fsplink);
raptor_free_world(rw);
return 0;
}
static unsigned char * handle_bind_first (fs_backend *be, fs_segment segment,
unsigned int length,
unsigned char *content)
{
unsigned char *reply;
if (segment > be->segments) {
fs_error(LOG_ERR, "invalid segment number: %d", segment);
return fsp_error_new(segment, "invalid segment number");
}
if (length < 32) {
fs_error(LOG_ERR, "bind_first(%d) much too short", segment);
return fsp_error_new(segment, "much too short");
}
fs_rid_vector models, subjects, predicates, objects;
unsigned int flags, value;
int count;
memcpy(&flags, content, sizeof (flags));
memcpy(&count, content + 4, sizeof (count));
memcpy(&value, content + 12, sizeof (models.length));
models.size = models.length = value / 8;
memcpy(&value, content + 16, sizeof (subjects.length));
subjects.size = subjects.length = value / 8;
memcpy(&value, content + 20, sizeof (predicates.length));
predicates.size = predicates.length = value / 8;
memcpy(&value, content + 24, sizeof (objects.length));
objects.size = objects.length = value / 8;
content += 32;
if (length < (models.size + subjects.size + predicates.size + objects.size) * 8 + 32) {
fs_error(LOG_ERR, "bind_first(%d) too short", segment);
return fsp_error_new(segment, "too short");
}
models.data = (fs_rid *) content;
content += models.length * 8;
subjects.data = (fs_rid *) content;
content += subjects.length * 8;
predicates.data = (fs_rid *) content;
content += predicates.length * 8;
objects.data = (fs_rid *) content;
fs_rid_vector **bindings;
bindings = fs_bind_first(be, segment, flags, &models, &subjects,
&predicates, &objects, count);
int k, cols = 0;
for (k = 0; k < 4; ++k) {
if (flags & 1 << k) cols++;
}
if (bindings == NULL) {
/* NULL => no match */
reply = message_new(FS_NO_MATCH, segment, 0);
cols = 0;
} else if (cols == 0) {
/* Zero columns => match with no binding */
reply = message_new(FS_BIND_LIST, segment, 0);
} else {
/* otherwise return bindings */
reply = message_new(FS_BIND_LIST, segment, bindings[0]->length * 8 * cols);
unsigned char *data = reply + FS_HEADER;
for (k= 0; k < cols; ++k) {
memcpy(data, bindings[k]->data, bindings[k]->length * 8);
data += bindings[k]->length * 8;
}
}
for (k = 0; k < cols; ++k) {
fs_rid_vector_free(bindings[k]);
}
free(bindings);
return reply;
}
