fs_binding *fs_binding_create(fs_binding *b, const char *name, fs_rid val, int projected)
{
int i;
for (i=0; i < FS_BINDING_MAX_VARS && b[i].name; i++);
if (i == FS_BINDING_MAX_VARS) {
fs_error(LOG_ERR, "variable limit of %d exceeded", FS_BINDING_MAX_VARS);
return NULL;
}
b[i].name = g_strdup(name);
if (val != FS_RID_NULL) {
if (b[i].vals) {
fs_error(LOG_WARNING, "loosing pointer to rid_vector");
}
b[i].vals = fs_rid_vector_new_from_args(1, val);
b[i].bound = 1;
} else {
if (b[i].vals) {
fs_error(LOG_WARNING, "loosing pointer to rid_vector");
}
b[i].vals = fs_rid_vector_new(0);
}
b[i].proj = projected;
b[i].need_val = projected;
return b+i;
}
END_TEST
START_TEST (check_fs_rid_vector_sort)
{
fs_rid_vector *v = fs_rid_vector_new(0);
fail_unless(v != NULL, "fs_rid_vector is NULL");
int i=0;
for (i=100; i < 1e4; i++) {
if (i % 2) {
fs_rid_vector_append(v, i);
fs_rid_vector_append(v, i+3);
fs_rid_vector_append(v, i-10);
} else {
fs_rid_vector_append(v, i-1);
fs_rid_vector_append(v, i+5);
fs_rid_vector_append(v, i);
}
}
fs_rid_vector_sort(v);
fail_if(fs_rid_vector_length(v) != ((1e4 - 100) *3), "Length fail after sort");
for (i=100; i < 1e4; i++) {
fail_if ( !fs_rid_vector_contains(v, i), "Contain failed after sort");
}
for (i=0;i<fs_rid_vector_length(v) -1; i++) {
fail_if (v->data[i] > v->data[i+1], "Sort does not match.");
}
fs_rid_vector_free(v);
}
END_TEST
START_TEST (check_fs_rid_vector_uniq)
{
fs_rid_vector *v = fs_rid_vector_new(0);
fail_unless(v != NULL, "fs_rid_vector is NULL");
int i=0;
for (i=100; i < 500; i++) {
fs_rid_vector_append(v, i);
fs_rid_vector_append(v, i);
}
fs_rid_vector_append(v,FS_RID_NULL);
fail_if(fs_rid_vector_length(v) != (((500 - 100)*2) + 1), "fs_rid_vector_length failed");
fs_rid_vector_uniq(v, 0); //Not remove nulls
fail_if(fs_rid_vector_length(v) != ((500 - 100) + 1), "fs_rid_vector_length failed");
for (i=1; i < fs_rid_vector_length(v) - 1; i++) {
fail_if(v->data[i] == v->data[i-1], "unique values failed [%d %d]",v->data[i],v->data[i-1]);
}
fail_if(v->data[fs_rid_vector_length(v)-1] != FS_RID_NULL, "FS_RID_NULL isn't there.");
v->data[123] = FS_RID_NULL;
fs_rid_vector_uniq(v, 1); //remove nulls
fail_if(fs_rid_vector_length(v) != ((500 - 100) -1), "fs_rid_vector_length failed");
fail_if(v->data[fs_rid_vector_length(v)-1] == FS_RID_NULL, "FS_RID_NULL should not be there %llx",
v->data[fs_rid_vector_length(v)-1]);
fail_if(v->data[123] == FS_RID_NULL, "FS_RID_NULL should not be there %llx",
v->data[fs_rid_vector_length(v)-1]);
fs_rid_vector_free(v);
}
fs_binding *fs_binding_copy_and_clear(fs_binding *b)
{
#ifdef DEBUG_BINDING
printf("@@ copy_and_clear()\n");
#endif
fs_binding *b2 = fs_binding_new();
memcpy(b2, b, sizeof(fs_binding) * FS_BINDING_MAX_VARS);
for (int i=0; 1; i++) {
if (!b[i].name) {
break;
}
b[i].name = g_strdup(b2[i].name);
b[i].vals = fs_rid_vector_new(b2[i].vals->size);
b[i].vals->length = 0;
/* at this point we can clear the bound flag as
shortcuts on variables bound to the empty
list are now handled by a wrapper round
fsp_bind_*() and look to the parent code
just like we had sent them to the backend */
b[i].bound = 0;
}
return b2;
}
static int calc_freq(fs_query *q, int block, GHashTable *freq, rasqal_literal *pri, rasqal_literal *sec)
{
int ret = 0;
int junk;
rasqal_variable *var;
fs_rid_vector *pv = fs_rid_vector_new(1);
fs_rid_vector *sv = fs_rid_vector_new(1);
sv->length = 0;
pv->length = 0;
fs_bind_slot(q, -1, q->bb[block], pri, pv, &junk, &var, 1);
if (sec) fs_bind_slot(q, -1, q->bb[block], sec, sv, &junk, &var, 1);
fs_quad_freq fd;
fd.pri = pv->data[0];
if (sec) {
fd.sec = sv->data[0];
} else {
fd.sec = FS_RID_NULL;
}
fs_quad_freq *res = g_hash_table_lookup(freq, &fd);
if (res) ret = res->freq;
if (ret == 0) {
if (sec) {
if (fs_opt_literal_is_rdf_type(pri)) {
/* if the primary key is rdf:type, there's likely to be lots */
ret = 100;
} else {
/* arbitrary choice - if primary and secondary keys are known,
* but theres not enough data to register in the frequency data
* then we guess there will be 1 answer */
ret = 1;
}
} else {
/* arbitrary choice - if only the primary key is known, but
* theres not enough data to register in the frequency data then we
* guess there will be 10 answers */
ret = 10;
}
}
return ret;
}
fs_rid_vector *fs_rid_vector_copy(fs_rid_vector *v)
{
if (!v) return NULL;
fs_rid_vector *v2 = fs_rid_vector_new(v->size);
v2->length = v->length;
memcpy(v2->data, v->data, v->size * sizeof(fs_rid));
return v2;
}
fs_rid_vector *fs_rid_vector_new_from_args(int length, ...)
{
va_list argp;
fs_rid_vector *t = fs_rid_vector_new(length);
va_start(argp, length);
for (int i=0; i<length; i++) {
t->data[i] = va_arg(argp, fs_rid);
}
return t;
}
fs_rid_vector *fs_rid_vector_intersect(int count, const fs_rid_vector *rv[])
{
fs_rid_vector *ret = fs_rid_vector_new(0);
for (int i=0; i<rv[0]->length; i++) {
if (inter_sub(count, 1, rv, rv[0]->data[i])) {
fs_rid_vector_append(ret, rv[0]->data[i]);
}
}
return ret;
}
fs_rid_vector *fs_metadata_get_int_vector(fs_metadata *m, const char *prop)
{
fs_rid_vector *rv = fs_rid_vector_new(0);
for (int e=0; e < m->length; e++) {
if (!strcmp(m->entries[e].key, prop)) {
fs_rid_vector_append(rv, atoll(m->entries[e].val));
}
}
return rv;
}
fs_value fn_datatype(fs_query *q, fs_value a)
{
#if 0
printf("datatype(");
fs_value_print(a);
printf(")\n");
#endif
if (a.valid & fs_valid_bit(FS_V_TYPE_ERROR)) {
return a;
}
if (a.valid & fs_valid_bit(FS_V_RID) && FS_IS_URI_BN(a.rid)) {
return fs_value_error(FS_ERROR_INVALID_TYPE, NULL);
}
if (a.attr == FS_RID_NULL) {
return fs_value_error(FS_ERROR_INVALID_TYPE, NULL);
}
if (FS_IS_LITERAL(a.attr) && a.attr != fs_c.empty) {
return fs_value_error(FS_ERROR_INVALID_TYPE, NULL);
} else {
if (a.attr == fs_c.xsd_string || a.attr == fs_c.empty) {
return fs_value_uri(XSD_STRING);
} else if (a.attr == fs_c.xsd_double) {
return fs_value_uri(XSD_DOUBLE);
} else if (a.attr == fs_c.xsd_float) {
return fs_value_uri(XSD_FLOAT);
} else if (a.attr == fs_c.xsd_decimal) {
return fs_value_uri(XSD_DECIMAL);
} else if (a.attr == fs_c.xsd_integer) {
return fs_value_uri(XSD_INTEGER);
} else if (a.attr == fs_c.xsd_boolean) {
return fs_value_uri(XSD_BOOLEAN);
} else if (a.attr == fs_c.xsd_datetime) {
return fs_value_uri(XSD_DATETIME);
}
}
fs_rid_vector *r = fs_rid_vector_new(1);
r->data[0] = a.attr;
fs_resource res;
if (fs_query_link(q)) {
fsp_resolve(fs_query_link(q), FS_RID_SEGMENT(a.attr,
fsp_link_segments(fs_query_link(q))), r, &res);
fs_rid_vector_free(r);
return fs_value_uri(res.lex);
}
return fs_value_uri("error:unresloved");
}
void fs_mhash_print(fs_mhash *mh, FILE *out, int verbosity)
{
if (!mh) {
fs_error(LOG_CRIT, "tried to print NULL mhash");
return;
}
fs_mhash_entry e;
fs_rid_vector *models = fs_rid_vector_new(0);
fs_rid last_model = FS_RID_NULL;
int entry = 0;
int count = 0;
fprintf(out, "mhash %s\n", mh->filename);
fprintf(out, " count: %d\n", mh->count);
fprintf(out, " size: %d\n", mh->size);
fprintf(out, "\n");
lseek(mh->fd, sizeof(struct mhash_header), SEEK_SET);
while (read(mh->fd, &e, sizeof(e)) == sizeof(e)) {
if (e.val) {
count++;
if (verbosity > 0) {
fprintf(out, "%8d %016llx %8d\n", entry, e.rid, e.val);
}
fs_rid_vector_append(models, e.rid);
if (e.rid == last_model) {
fprintf(out, "ERROR: %s model %016llx appears multiple times\n",
mh->filename, e.rid);
}
last_model = e.rid;
}
entry++;
}
if (mh->count != count) {
fprintf(out, "ERROR: %s header count %d != scanned count %d\n",
mh->filename, mh->count, count);
}
int oldlength = models->length;
fs_rid_vector_sort(models);
fs_rid_vector_uniq(models, 0);
if (models->length != oldlength) {
fprintf(out, "ERROR: %s some models appear > 1 time\n",
mh->filename);
}
}
static char *get_lex(fsp_link *link, fs_rid rid)
{
if (rid == FS_RID_NULL) return g_strdup("*");
const int segments = fsp_link_segments(link);
fs_resource res;
fs_rid_vector *rv = fs_rid_vector_new(1);
rv->data[0] = rid;
fsp_resolve(link, FS_RID_SEGMENT(rid, segments), rv, &res);
fs_rid_vector_free(rv);
if (!strncmp(RDF_NAMESPACE, res.lex, strlen(RDF_NAMESPACE))) {
char *new = g_strdup_printf("rdf:%s", res.lex + strlen(RDF_NAMESPACE));
g_free(res.lex);
return new;
}
END_TEST
START_TEST (check_fs_rid_vector_truncate)
{
fs_rid_vector *v = fs_rid_vector_new(0);
fail_unless(v != NULL, "fs_rid_vector is NULL");
int i=0;
for (i=100; i < 1e4; i++) {
fs_rid_vector_append(v, i);
}
fs_rid_vector_truncate(v, 100);
fail_if(fs_rid_vector_length(v) != 100);
fail_if(v->data[fs_rid_vector_length(v)-1] != 199);
fs_rid_vector_truncate(v, 1);
fail_if(v->data[fs_rid_vector_length(v)-1] != 100);
fs_rid_vector_free(v);
}
fs_binding *fs_binding_add(fs_binding *b, rasqal_variable *var, fs_rid val, int projected)
{
#ifdef DEBUG_BINDING
if (strcmp(DEBUG_BINDING, name)) printf("@@ add("DEBUG_BINDING", %016llx, %d)\n", val, projected);
#endif
fs_binding *bv = fs_binding_get(b, var);
if (bv) {
fs_rid_vector_append(bv->vals, val);
bv->bound = 1;
bv->proj |= projected;
bv->need_val |= projected;
return bv;
}
long i;
for (i=0; i < FS_BINDING_MAX_VARS && b[i].name; i++);
if (i == FS_BINDING_MAX_VARS) {
fs_error(LOG_ERR, "variable limit (%d) exceeded",
FS_BINDING_MAX_VARS);
return NULL;
}
b[i].name = g_strdup((char *)var->name);
if (val != FS_RID_NULL) {
if (b[i].vals) {
fs_error(LOG_WARNING, "loosing pointer to rid_vector");
}
b[i].vals = fs_rid_vector_new_from_args(1, val);
b[i].bound = 1;
} else {
if (b[i].vals) {
fs_error(LOG_WARNING, "loosing pointer to rid_vector");
}
b[i].vals = fs_rid_vector_new(0);
}
b[i].proj = projected;
b[i].need_val = projected;
var->user_data = (void *)i;
return b+i;
}
fs_value fn_lang(fs_query *q, fs_value a)
{
if (a.valid & fs_valid_bit(FS_V_TYPE_ERROR)) {
return a;
}
if (a.valid & fs_valid_bit(FS_V_RID) && FS_IS_URI_BN(a.rid)) {
return fs_value_error(FS_ERROR_INVALID_TYPE, NULL);
}
if (a.attr == FS_RID_NULL) {
return fs_value_error(FS_ERROR_INVALID_TYPE, NULL);
}
if (FS_IS_URI(a.attr)) {
return fs_value_plain("");
} else {
if (a.attr == fs_c.lang_en) {
return fs_value_plain("en");
} else if (a.attr == fs_c.lang_fr) {
return fs_value_plain("fr");
} else if (a.attr == fs_c.lang_de) {
return fs_value_plain("de");
} else if (a.attr == fs_c.lang_es) {
return fs_value_plain("es");
} else if (a.attr == fs_c.empty) {
return fs_value_plain("");
}
}
fs_rid_vector *r = fs_rid_vector_new(1);
r->data[0] = a.attr;
fs_resource res;
if (fs_query_link(q)) {
fsp_resolve(fs_query_link(q), FS_RID_SEGMENT(a.attr,
fsp_link_segments(fs_query_link(q))), r, &res);
fs_rid_vector_free(r);
fs_query_add_freeable(q, res.lex);
return fs_value_plain(res.lex);
}
return fs_value_plain("???");
}
END_TEST
START_TEST (check_fs_rid_vector_grow)
{
fs_rid_vector *v = fs_rid_vector_new(0);
fail_unless(v != NULL, "fs_rid_vector is NULL");
int i=0;
for (i=100; i < 1e4; i++) {
fs_rid_vector_append(v, i);
}
fail_if(fs_rid_vector_length(v) != (1e4 - 100));
fs_rid_vector_grow(v, 2e4);
fail_if(fs_rid_vector_length(v) != 2e4);
for (i=0;i<2e4;i++) {
v->data[i] = i;
}
fail_if(v->size != 2e4);
fs_rid_vector_free(v);
}
fs_rid_vector *fs_mhash_get_keys(fs_mhash *mh)
{
if (!mh) {
fs_error(LOG_CRIT, "tried to get keys from NULL mhash");
return NULL;
}
fs_rid_vector *v = fs_rid_vector_new(0);
fs_mhash_entry e;
if (!mh->locked) flock(mh->fd, LOCK_SH);
if (lseek(mh->fd, sizeof(struct mhash_header), SEEK_SET) == -1) {
fs_error(LOG_ERR, "seek error on mhash: %s", strerror(errno));
}
while (read(mh->fd, &e, sizeof(e)) == sizeof(e)) {
if (e.val) fs_rid_vector_append(v, e.rid);
}
if (!mh->locked) flock(mh->fd, LOCK_UN);
return v;
}
static fs_rid_vector *rid_file(char *filename)
{
fs_rid_vector *rids = fs_rid_vector_new(0);
FILE *fp = fopen(filename, "r");
if (!fp) {
fs_error(LOG_ERR, "could not open “%s”: %s", filename, strerror(errno));
return rids;
}
while (!feof(fp) && !ferror(fp)) {
char ridstr[21];
fs_rid rid;
if (fscanf(fp, "%20s", ridstr) < 1) break;
rid = strtoull(ridstr, NULL, 16);
fs_rid_vector_append(rids, rid);
}
fclose(fp);
return rids;
}
int fs_clear(struct update_context *uc, char *graphuri)
{
fs_rid_vector *mvec = fs_rid_vector_new(0);
fs_rid mrid;
if (graphuri) {
mrid = fs_hash_uri(graphuri);
} else {
graphuri = FS_DEFAULT_GRAPH;
mrid = fs_c.default_graph;
}
fs_rid_vector_append(mvec, mrid);
int errors = 0;
if (fsp_delete_model_all(uc->link, mvec)) {
errors++;
add_message(uc, g_strdup_printf("Error while trying to delete %s", graphuri), 1);
} else {
add_message(uc, g_strdup_printf("Deleted <%s>", graphuri), 1);
}
fs_rid_vector_free(mvec);
return errors;
}
/* return to = from [X] to, this is used to perform joins inside blocks, it
* saves allocations by doing most operations inplace, unlike fs_binding_join */
void fs_binding_merge(fs_query *q, int block, fs_binding *from, fs_binding *to)
{
fs_binding *inter_f = NULL; /* the intersecting column */
fs_binding *inter_t = NULL; /* the intersecting column */
for (int i=0; from[i].name; i++) {
from[i].sort = 0;
to[i].sort = 0;
}
int used = 0;
for (int i=1; from[i].name; i++) {
if (!from[i].bound || !to[i].bound) continue;
if (from[i].used) used++;
if (from[i].bound && to[i].bound) {
inter_f = from+i;
inter_t = to+i;
from[i].sort = 1;
to[i].sort = 1;
#ifdef DEBUG_MERGE
printf("@@ join on %s\n", to[i].name);
#endif
}
}
/* from and to bound variables do not intersect, we can just dump results,
under some circustances we need to do a combinatorial explosion */
if (!inter_f && (fs_binding_length(from) == 0)) {
const int length_f = fs_binding_length(from);
const int length_t = fs_binding_length(to);
for (int i=1; from[i].name; i++) {
if (to[i].bound && !from[i].bound) {
if (from[i].vals) {
fs_rid_vector_free(from[i].vals);
}
from[i].vals = fs_rid_vector_new(length_f);
for (int d=0; d<length_f; d++) {
from[i].vals->data[d] = FS_RID_NULL;
}
from[i].bound = 1;
}
if (!from[i].bound) continue;
if (!to[i].bound) {
if (to[i].vals) {
fs_rid_vector_free(to[i].vals);
}
to[i].vals = fs_rid_vector_new(length_t);
for (int d=0; d<length_t; d++) {
to[i].vals->data[d] = FS_RID_NULL;
}
}
fs_rid_vector_append_vector(to[i].vals, from[i].vals);
to[i].bound = 1;
}
#ifdef DEBUG_MERGE
printf("append all, result:\n");
fs_binding_print(to, stdout);
#endif
return;
}
/* If were running in restricted mode, truncate the binding tables */
if (q->flags & FS_QUERY_RESTRICTED) {
fs_binding_truncate(from, q->soft_limit);
fs_binding_truncate(to, q->soft_limit);
}
int length_t = fs_binding_length(to);
int length_f = fs_binding_length(from);
/* ms8: this list keeps track of the vars to replace */
GList *rep_list = NULL;
for (int i=1; to[i].name; i++) {
if (to+i == inter_t || to[i].used || to[i].bound) {
/* do nothing */
#if DEBUG_MERGE > 1
printf("@@ preserve %s\n", to[i].name);
#endif
} else if (from[i].bound && !to[i].bound) {
#if DEBUG_MERGE > 1
printf("@@ replace %s\n", from[i].name);
#endif
to[i].bound = 1;
if (to[i].vals) {
if (to[i].vals->length != length_t) {
fs_rid_vector_free(to[i].vals);
to[i].vals = fs_rid_vector_new(length_t);
}
} else {
to[i].vals = fs_rid_vector_new(length_t);
}
for (int d=0; d<length_t; d++) {
to[i].vals->data[d] = FS_RID_NULL;
}
rep_list = g_list_append(rep_list, GINT_TO_POINTER(i));
}
}
/* sort the two sets of bindings so they can be merged linearly */
if (inter_f) {
fs_binding_sort(from);
fs_binding_sort(to);
} else {
/* make sure the tables are not marked sorted */
from[0].vals->length = 0;
to[0].vals->length = 0;
}
#ifdef DEBUG_MERGE
printf("old: %d bindings\n", fs_binding_length(from));
fs_binding_print(from, stdout);
printf("new: %d bindings\n", fs_binding_length(to));
fs_binding_print(to, stdout);
#endif
int fpos = 0;
int tpos = 0;
while (fpos < length_f || tpos < length_t) {
if (q->flags & FS_QUERY_RESTRICTED &&
fs_binding_length(to) >= q->soft_limit) {
char *msg = g_strdup("some results have been dropped to prevent overunning time allocation");
q->warnings = g_slist_prepend(q->warnings, msg);
break;
}
int cmp;
cmp = binding_row_compare(q, from, to, fpos, tpos, length_f, length_t);
if (cmp == 0) {
/* both rows match */
int fp, tp = tpos;
for (fp = fpos; binding_row_compare(q, from, to, fp, tpos, length_f, length_t) == 0; fp++) {
#if DEBUG_MERGE > 1
if (fp == DEBUG_CUTOFF) {
printf("...\n");
}
#endif
for (tp = tpos; 1; tp++) {
if (binding_row_compare(q, from, to, fp, tp, length_f, length_t) == 0) {
#if DEBUG_MERGE > 1
if (fp < DEBUG_CUTOFF) {
printf("STEP %d, %d ", fp-fpos, tp-tpos);
}
#endif
if (fp == fpos) {
#if DEBUG_MERGE > 1
if (fp < DEBUG_CUTOFF) {
if (inter_f) {
printf("REPL %llx\n", inter_f->vals->data[fp]);
} else {
printf("REPL ???\n");
}
}
#endif
for (int c=1; to[c].name; c++) {
if (!from[c].bound && !to[c].bound) continue;
if (from[c].bound && table_value(from, c, fp) == FS_RID_NULL) {
continue;
}
if (from[c].bound && fp < from[c].vals->length) {
long wrow = to[0].vals->length ? to[0].vals->data[tp] : tp;
to[c].vals->data[wrow] = table_value(from, c, fp);
if (to[c].vals->length <= tp) {
to[c].vals->length = tp+1;
}
}
}
} else {
#if DEBUG_MERGE > 1
if (fp < DEBUG_CUTOFF) {
printf("ADD\n");
}
#endif
for (int c=1; to[c].name; c++) {
if (!from[c].bound && !to[c].bound) continue;
if (from[c].bound && fp < from[c].vals->length) {
fs_rid_vector_append(to[c].vals, table_value(from, c, fp));
} else {
fs_rid_vector_append(to[c].vals, table_value(to, c, tp));
}
}
}
} else {
break;
}
}
}
tpos = tp;
fpos = fp;
} else if (cmp <= -1) {
fpos++;
} else if (cmp >= 1) {
tpos++;
} else {
fs_error(LOG_CRIT, "unknown compare state %d in binding", cmp);
}
}
/* clear the _ord columns */
from[0].vals->length = 0;
to[0].vals->length = 0;
/* ms8: INIT code to clean up rows that where not replaced */
if (rep_list) {
unsigned char *to_del = fs_new_bit_array(length_t);
int to_del_count = 0;
while(rep_list) {
int col_r = GPOINTER_TO_INT(rep_list->data);
rep_list = g_list_next(rep_list);
for (int d=0; d<length_t; d++) {
if (to[col_r].vals->data[d] == FS_RID_NULL) {
fs_bit_array_set(to_del, d, 0);
to_del_count++;
}
}
}
g_list_free(rep_list);
if (to_del_count) {
int vars = 0;
for (int i=1; to[i].name; i++)
vars++;
fs_rid_vector **clean = calloc(vars, sizeof(fs_rid_vector *));
for (int i=0;i<vars;i++)
clean[i] = fs_rid_vector_new(0);
for (int d = 0;d<length_t;d++) {
if (fs_bit_array_get(to_del,d)) {
for (int i=0;i<vars;i++) {
fs_rid_vector_append(clean[i],to[i+1].vals->data[d]);
}
}
}
for (int i=1;i<=vars;i++) {
free(to[i].vals->data);
to[i].vals->data = clean[i-1]->data;
to[i].vals->length = clean[i-1]->length;
to[i].vals->size = clean[i-1]->size;
free(clean[i-1]);
}
free(clean);
}
fs_bit_array_destroy(to_del);
}
/* ms8: END code to clean up rows that where not replaced */
#ifdef DEBUG_MERGE
printf("result: %d bindings\n", fs_binding_length(to));
fs_binding_print(to, stdout);
#endif
}
int fs_copy(struct update_context *uc, char *from, char *to)
{
fs_rid_vector *mvec = fs_rid_vector_new(0);
fs_rid_vector *empty = fs_rid_vector_new(0);
fs_rid fromrid, torid;
if (from) {
fromrid = fs_hash_uri(from);
} else {
from = FS_DEFAULT_GRAPH;
fromrid = fs_c.default_graph;
}
if (to) {
torid = fs_hash_uri(to);
} else {
to = FS_DEFAULT_GRAPH;
torid = fs_c.default_graph;
}
if (fromrid == torid) {
/*don't need to do anything */
fs_rid_vector_free(mvec);
fs_rid_vector_free(empty);
add_message(uc, g_strdup_printf("Copied <%s> to <%s>", from, to), 1);
add_message(uc, "0 triples added, 0 removed", 0);
return 0;
}
fs_rid_vector_append(mvec, fromrid);
/* search for all the triples in from */
fs_rid_vector **results;
fs_rid_vector *slot[4] = { mvec, empty, empty, empty };
/* see if there's any data in <from> */
fs_bind_cache_wrapper(uc->qs, NULL, 1, FS_BIND_BY_SUBJECT | FS_BIND_SUBJECT,
slot, &results, -1, 1);
if (!results || results[0]->length == 0) {
if (results) {
fs_rid_vector_free(results[0]);
free(results);
}
fs_rid_vector_free(mvec);
fs_rid_vector_free(empty);
add_message(uc, g_strdup_printf("<%s> is empty, not copying", from), 1);
return 1;
}
fs_rid_vector_free(results[0]);
free(results);
/* get the contents of <from> */
fs_bind_cache_wrapper(uc->qs, NULL, 1, FS_BIND_BY_SUBJECT | FS_BIND_SUBJECT | FS_BIND_PREDICATE | FS_BIND_OBJECT,
slot, &results, -1, -1);
/* map old bnodes to new ones */
map_bnodes(uc, results[0]);
map_bnodes(uc, results[1]);
map_bnodes(uc, results[2]);
/* delete <to> */
mvec->data[0] = torid;
if (fsp_delete_model_all(uc->link, mvec)) {
fs_rid_vector_free(mvec);
fs_rid_vector_free(empty);
add_message(uc, g_strdup_printf("Error while trying to delete %s", to), 1);
return 1;
}
fs_rid_vector_free(mvec);
fs_rid_vector_free(empty);
/* insert <to> */
fs_resource tores;
tores.lex = to;
tores.attr= FS_RID_NULL;
tores.rid = torid;
fsp_res_import(uc->link, FS_RID_SEGMENT(torid, uc->segments), 1, &tores);
insert_triples(uc, torid, results[0], results[1], results[2]);
add_message(uc, g_strdup_printf("Copied <%s> to <%s>", from, to), 1);
add_message(uc, g_strdup_printf("%d triples added, ?? removed", results[0]->length), 1);
for (int i=0; i<3; i++) {
fs_rid_vector_free(results[i]);
}
free(results);
return 0;
}
int fs_add(struct update_context *uc, char *from, char *to)
{
fs_rid_vector *mvec = fs_rid_vector_new(0);
fs_rid_vector *empty = fs_rid_vector_new(0);
fs_rid fromrid, torid;
if (from) {
fromrid = fs_hash_uri(from);
} else {
from = FS_DEFAULT_GRAPH;
fromrid = fs_c.default_graph;
}
if (to) {
torid = fs_hash_uri(to);
} else {
to = FS_DEFAULT_GRAPH;
torid = fs_c.default_graph;
}
if (fromrid == torid) {
/*don't need to do anything */
add_message(uc, g_strdup_printf("Added <%s> to <%s>", from, to), 1);
add_message(uc, "0 triples added, 0 removed", 0);
return 0;
}
fs_rid_vector_append(mvec, fromrid);
int errors = 0;
/* search for all the triples in from */
fs_rid_vector **results;
fs_rid_vector *slot[4] = { mvec, empty, empty, empty };
fs_bind_cache_wrapper(uc->qs, NULL, 1, FS_BIND_BY_SUBJECT | FS_BIND_SUBJECT | FS_BIND_PREDICATE | FS_BIND_OBJECT,
slot, &results, -1, -1);
fs_rid_vector_free(mvec);
fs_rid_vector_free(empty);
if (!results || results[0]->length == 0) {
/* there's nothing to add */
if (results) {
for (int i=0; i<3; i++) {
fs_rid_vector_free(results[i]);
}
free(results);
}
add_message(uc, g_strdup_printf("Added <%s> to <%s>", from, to), 1);
add_message(uc, "0 triples added, 0 removed", 0);
return 0;
}
map_bnodes(uc, results[0]);
map_bnodes(uc, results[1]);
map_bnodes(uc, results[2]);
fs_resource tores;
tores.lex = to;
tores.attr= FS_RID_NULL;
tores.rid = torid;
fsp_res_import(uc->link, FS_RID_SEGMENT(torid, uc->segments), 1, &tores);
insert_triples(uc, torid, results[0], results[1], results[2]);
add_message(uc, g_strdup_printf("Added <%s> to <%s>", from, to), 1);
add_message(uc, g_strdup_printf("%d triples added, 0 removed", results[0]->length), 1);
for (int i=0; i<3; i++) {
fs_rid_vector_free(results[i]);
}
free(results);
return errors;
}
static int update_op(struct update_context *uc)
{
fs_rid_vector *vec[4];
switch (uc->op->type) {
case RASQAL_UPDATE_TYPE_UNKNOWN:
add_message(uc, "Unknown update operation", 0);
return 1;
case RASQAL_UPDATE_TYPE_CLEAR:
fs_clear(uc, graph_arg(uc->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_CREATE:
return 0;
case RASQAL_UPDATE_TYPE_DROP:
fs_clear(uc, graph_arg(uc->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_LOAD:
fs_load(uc, graph_arg(uc->op->document_uri),
graph_arg(uc->op->graph_uri));
return 0;
#if RASQAL_VERSION >= 924
case RASQAL_UPDATE_TYPE_ADD:
fs_add(uc, graph_arg(uc->op->graph_uri),
graph_arg(uc->op->document_uri));
return 0;
case RASQAL_UPDATE_TYPE_MOVE:
fs_move(uc, graph_arg(uc->op->graph_uri),
graph_arg(uc->op->document_uri));
return 0;
case RASQAL_UPDATE_TYPE_COPY:
fs_copy(uc, graph_arg(uc->op->graph_uri),
graph_arg(uc->op->document_uri));
return 0;
#endif
case RASQAL_UPDATE_TYPE_UPDATE:
break;
}
fs_hash_freshen();
raptor_sequence *todel = NULL;
raptor_sequence *toins = NULL;
if (uc->op->delete_templates && !uc->op->where) {
int where = 0;
/* check to see if it's a DELETE WHERE { } */
for (int t=0; t<raptor_sequence_size(uc->op->delete_templates); t++) {
rasqal_triple *tr = raptor_sequence_get_at(uc->op->delete_templates, t);
if (any_vars(tr)) {
where = 1;
break;
}
}
if (where) {
fs_error(LOG_ERR, "DELETE WHERE { x } not yet supported");
add_message(uc, "DELETE WHERE { x } not yet supported, use DELETE { x } WHERE { x }", 0);
return 1;
}
}
#if RASQAL_VERSION >= 923
if (uc->op->where) {
todel = raptor_new_sequence(NULL, NULL);
toins = raptor_new_sequence(NULL, NULL);
raptor_sequence *todel_p = raptor_new_sequence(NULL, NULL);
raptor_sequence *toins_p = raptor_new_sequence(NULL, NULL);
raptor_sequence *vars = raptor_new_sequence(NULL, NULL);
fs_query *q = calloc(1, sizeof(fs_query));
uc->q = q;
q->qs = uc->qs;
q->rq = uc->rq;
q->flags = FS_BIND_DISTINCT;
#ifdef DEBUG_MERGE
q->flags |= FS_QUERY_CONSOLE_OUTPUT;
#endif
q->boolean = 1;
q->opt_level = 3;
q->soft_limit = -1;
q->segments = fsp_link_segments(uc->link);
q->link = uc->link;
q->bb[0] = fs_binding_new();
q->bt = q->bb[0];
/* hashtable to hold runtime created resources */
q->tmp_resources = g_hash_table_new_full(fs_rid_hash, fs_rid_equal, g_free, fs_free_cached_resource);
/* add column to denote join ordering */
fs_binding_create(q->bb[0], "_ord", FS_RID_NULL, 0);
if (uc->op->delete_templates) {
for (int t=0; t<raptor_sequence_size(uc->op->delete_templates); t++) {
rasqal_triple *tr = raptor_sequence_get_at(uc->op->delete_templates, t);
if (any_vars(tr)) {
fs_check_cons_slot(q, vars, tr->subject);
fs_check_cons_slot(q, vars, tr->predicate);
fs_check_cons_slot(q, vars, tr->object);
raptor_sequence_push(todel_p, tr);
} else {
raptor_sequence_push(todel, tr);
}
}
}
if (uc->op->insert_templates) {
for (int t=0; t<raptor_sequence_size(uc->op->insert_templates); t++) {
rasqal_triple *tr = raptor_sequence_get_at(uc->op->insert_templates, t);
if (any_vars(tr)) {
fs_check_cons_slot(q, vars, tr->subject);
fs_check_cons_slot(q, vars, tr->predicate);
fs_check_cons_slot(q, vars, tr->object);
raptor_sequence_push(toins_p, tr);
} else {
raptor_sequence_push(toins, tr);
}
}
}
q->num_vars = raptor_sequence_size(vars);
for (int i=0; i < q->num_vars; i++) {
rasqal_variable *v = raptor_sequence_get_at(vars, i);
fs_binding_add(q->bb[0], v, FS_RID_NULL, 1);
}
/* perform the WHERE match */
fs_query_process_pattern(q, uc->op->where, vars);
q->length = fs_binding_length(q->bb[0]);
for (int s=0; s<4; s++) {
vec[s] = fs_rid_vector_new(0);
}
for (int t=0; t<raptor_sequence_size(todel_p); t++) {
rasqal_triple *triple = raptor_sequence_get_at(todel_p, t);
for (int row=0; row < q->length; row++) {
delete_rasqal_triple(uc, vec, triple, row);
}
if (fs_rid_vector_length(vec[0]) > 1000) {
fsp_delete_quads_all(uc->link, vec);
}
}
if (fs_rid_vector_length(vec[0]) > 0) {
fsp_delete_quads_all(uc->link, vec);
}
for (int s=0; s<4; s++) {
//fs_rid_vector_print(vec[s], 0, stdout);
fs_rid_vector_free(vec[s]);
vec[s] = NULL;
}
for (int t=0; t<raptor_sequence_size(toins_p); t++) {
rasqal_triple *triple = raptor_sequence_get_at(toins_p, t);
for (int row=0; row < q->length; row++) {
insert_rasqal_triple(uc, triple, row);
}
}
/* must not free the rasqal_query */
q->rq = NULL;
fs_query_free(q);
uc->q = NULL;
} else {
todel = uc->op->delete_templates;
toins = uc->op->insert_templates;
}
#else
if (uc->op->where) {
fs_error(LOG_ERR, "DELETE/INSERT WHERE requires Rasqal 0.9.23 or newer");
add_message(uc, "DELETE/INSERT WHERE requires Rasqal 0.9.23 or newer", 0);
}
#endif
/* delete constant triples */
if (todel) {
for (int s=0; s<4; s++) {
vec[s] = fs_rid_vector_new(0);
}
for (int t=0; t<raptor_sequence_size(todel); t++) {
rasqal_triple *triple = raptor_sequence_get_at(todel, t);
if (any_vars(triple)) {
continue;
}
delete_rasqal_triple(uc, vec, triple, 0);
}
if (fs_rid_vector_length(vec[0]) > 0) {
fsp_delete_quads_all(uc->link, vec);
}
for (int s=0; s<4; s++) {
fs_rid_vector_free(vec[s]);
vec[s] = NULL;
}
}
/* insert constant triples */
if (toins) {
for (int t=0; t<raptor_sequence_size(toins); t++) {
rasqal_triple *triple = raptor_sequence_get_at(toins, t);
if (any_vars(triple)) {
continue;
}
insert_rasqal_triple(uc, triple, 0);
}
}
fs_hash_freshen();
return 0;
}
/**
* It loads the acl system info from the system:config graph.
* Due to link->acl_system_info manipulation this function should be
* under mutex conditions.
*/
int fs_acl_load_system_info(fsp_link *link) {
if (!fsp_acl_needs_reload(link))
return 0;
int flags = FS_BIND_SUBJECT | FS_BIND_PREDICATE | FS_BIND_OBJECT | FS_BIND_BY_SUBJECT;
fs_rid_vector *mrids = fs_rid_vector_new_from_args(1, fs_c.system_config);
fs_rid_vector *srids = fs_rid_vector_new(0);
fs_rid_vector *prids = fs_rid_vector_new_from_args(2, fs_c.fs_acl_admin, fs_c.fs_acl_access_by);
fs_rid_vector *orids = fs_rid_vector_new(0);
fs_rid_vector **result = NULL;
fsp_bind_limit_all(link, flags, mrids, srids, prids, orids, &result, -1, -1);
fs_rid_vector_free(mrids);
fs_rid_vector_free(srids);
fs_rid_vector_free(prids);
fs_rid_vector_free(orids);
int admin_users_count = 0;
fs_acl_system_info *acl_system_info = link->acl_system_info;
if (result && result[0]) {
if (!acl_system_info->acl_graph_hash || acl_system_info->admin_user_set)
link->acl_system_info = acl_system_info;
if (acl_system_info->acl_graph_hash) {
g_hash_table_steal(acl_system_info->acl_graph_hash, &fs_c.system_config);
g_hash_table_destroy(acl_system_info->acl_graph_hash);
acl_system_info->acl_graph_hash = NULL;
}
acl_system_info->acl_graph_hash = g_hash_table_new_full(fs_rid_hash,fs_rid_equal,
acl_key_destroyed, acl_value_destroyed);
if (acl_system_info->admin_user_set) {
fs_rid_set_free(acl_system_info->admin_user_set);
acl_system_info->admin_user_set = NULL;
}
acl_system_info->admin_user_set = fs_rid_set_new();
for (int row = 0; row < result[0]->length; row++) {
if(result[1]->data[row] == fs_c.fs_acl_access_by) {
/* if pred is acl_access_by then subject is the graph and object is the user rid */
gpointer users_set_ref = NULL;
fs_rid_set *users_set = NULL;
if (!(users_set_ref=g_hash_table_lookup(acl_system_info->acl_graph_hash, &result[0]->data[row]))) {
users_set = fs_rid_set_new();
fs_rid *rid_graph = malloc(sizeof(fs_rid));
*rid_graph = result[0]->data[row];
g_hash_table_insert(acl_system_info->acl_graph_hash, rid_graph, users_set);
} else
users_set = (fs_rid_set *) users_set_ref;
fs_rid_set_add(users_set, result[2]->data[row]);
} else if (result[1]->data[row] == fs_c.fs_acl_admin) {
/* if admin predicate then object contains the admin user rid id */
fs_rid_set_add(acl_system_info->admin_user_set, result[2]->data[row]);
admin_users_count++;
}
}
if (admin_users_count == 0) {
fs_error(LOG_ERR,"Added default admin user %s",FS_ACL_DEFAULT_ADMIN);
fs_rid_set_add(acl_system_info->admin_user_set, fs_c.fs_acl_default_admin);
}
/* only admin users can access system:config */
g_hash_table_insert(acl_system_info->acl_graph_hash, &fs_c.system_config, acl_system_info->admin_user_set);
}
fsp_acl_reloaded(link);
if (result) {
for (int i=0;i<3;i++) {
fs_rid_vector_free(result[i]);
}
free(result);
}
return 1;
}
int fs_bind_cache_wrapper_intl(fs_query_state *qs, fs_query *q, int all,
int flags, fs_rid_vector *rids[4],
fs_rid_vector ***result, int offset, int limit)
{
g_static_mutex_lock(&qs->cache_mutex);
if (!qs->bind_cache) {
qs->bind_cache = calloc(CACHE_SIZE, sizeof(struct _fs_bind_cache));
}
if (fsp_acl_needs_reload(qs->link))
fs_acl_load_system_info(qs->link);
g_static_mutex_unlock(&qs->cache_mutex);
int slots = 0;
if (flags & FS_BIND_MODEL) slots++;
if (flags & FS_BIND_SUBJECT) slots++;
if (flags & FS_BIND_PREDICATE) slots++;
if (flags & FS_BIND_OBJECT) slots++;
/* check for no possible bindings */
for (int s=0; s<4; s++) {
if (rids[s]->length == 1 && rids[s]->data[0] == FS_RID_NULL) {
*result = calloc(slots, sizeof(fs_rid_vector));
for (int s=0; s<slots; s++) {
(*result)[s] = fs_rid_vector_new(0);
}
return 0;
}
}
int cachable = 0;
fs_rid cache_hash = 0;
fs_rid cache_key[4];
/* only consult the cache for optimasation levels 0-2 */
if (q && q->opt_level < 3) goto skip_cache;
if (q && q->qs && q->qs->cache_stats) q->qs->bind_hits++;
cachable = 1;
cache_hash += all + flags * 2 + offset * 256 + limit * 32768;
for (int s=0; s<4; s++) {
if (rids[s]->length == 1) {
cache_hash ^= (rids[s]->data[0] + s);
cache_key[s] = rids[s]->data[0];
} else if (rids[s]->length == 0) {
cache_key[s] = 0;
} else {
/* bind cache does not cache binds with any
slot containing multiple values */
cachable = 0;
break;
}
}
cache_hash %= (CACHE_SIZE - 1);
g_static_mutex_lock(&qs->cache_mutex);
if (cachable && qs->bind_cache[cache_hash].filled) {
int match = 1;
if (qs->bind_cache[cache_hash].all != all) match = 0;
if (qs->bind_cache[cache_hash].flags != flags) match = 0;
if (qs->bind_cache[cache_hash].offset != offset) match = 0;
if (qs->bind_cache[cache_hash].limit != limit) match = 0;
for (int s=0; s<4 && match; s++) {
if (cache_key[s] != qs->bind_cache[cache_hash].key[s]) {
match = 0;
}
}
if (match) {
*result = calloc(slots, sizeof(fs_rid_vector));
for (int s=0; s<slots; s++) {
(*result)[s] = fs_rid_vector_copy(qs->bind_cache[cache_hash].res[s]);
}
fsp_hit_limits_add(qs->link, qs->bind_cache[cache_hash].limited);
qs->bind_cache[cache_hash].hits++;
if (q && q->qs && q->qs->cache_stats) q->qs->bind_cache_success++;
g_static_mutex_unlock(&qs->cache_mutex);
return 0;
}
}
g_static_mutex_unlock(&qs->cache_mutex);
int ret;
skip_cache:;
int limited_before = fsp_hit_limits(qs->link);
if (all) {
ret = fsp_bind_limit_all(qs->link, flags, rids[0], rids[1], rids[2], rids[3], result, offset, limit);
} else {
ret = fsp_bind_limit_many(qs->link, flags, rids[0], rids[1], rids[2], rids[3], result, offset, limit);
}
int limited = fsp_hit_limits(qs->link) - limited_before;
if (ret) {
fs_error(LOG_ERR, "bind failed in '%s', %d segments gave errors",
fsp_kb_name(qs->link), ret);
exit(1);
}
int small = 1;
for (int s=0; s<slots; s++) {
if (fs_rid_vector_length((*result)[s]) > 10000) {
small = 0;
break;
}
}
if (cachable && small && slots > 0) {
g_static_mutex_lock(&qs->cache_mutex);
if (qs->bind_cache[cache_hash].filled == 1) {
for (int s=0; s<4; s++) {
fs_rid_vector_free(qs->bind_cache[cache_hash].res[s]);
qs->bind_cache[cache_hash].res[s] = NULL;
}
}
qs->bind_cache[cache_hash].filled = 1;
qs->bind_cache[cache_hash].all = all;
qs->bind_cache[cache_hash].flags = flags;
qs->bind_cache[cache_hash].offset = offset;
qs->bind_cache[cache_hash].limit = limit;
qs->bind_cache[cache_hash].limited = limited;
for (int s=0; s<4; s++) {
qs->bind_cache[cache_hash].key[s] = cache_key[s];
if (s < slots) {
qs->bind_cache[cache_hash].res[s] = fs_rid_vector_copy((*result)[s]);
} else {
qs->bind_cache[cache_hash].res[s] = NULL;
}
}
g_static_mutex_unlock(&qs->cache_mutex);
}
return ret;
}
xmlChar *get_uri(fsp_link *link, fs_rid rid)
{
if (cache[rid & CACHE_MASK].rid == rid) {
return (xmlChar *) cache[rid & CACHE_MASK].lex;
}
fs_rid_vector onerid = { .length = 1, .size = 1, .data = &rid };
fs_resource resource;
fsp_resolve(link, FS_RID_SEGMENT(rid, segments), &onerid, &resource);
return (xmlChar *) resource.lex;
}
xmlChar *get_attr(fsp_link *link, fs_rid rid)
{
if (attr_cache[rid & CACHE_MASK].rid == rid) {
return (xmlChar *) attr_cache[rid & CACHE_MASK].lex;
}
fs_rid_vector onerid = { .length = 1, .size = 1, .data = &rid };
fs_resource resource;
fsp_resolve(link, FS_RID_SEGMENT(rid, segments), &onerid, &resource);
memcpy(&attr_cache[rid & ATTR_CACHE_MASK], &resource, sizeof(fs_resource));
return (xmlChar *) resource.lex;
}
xmlChar *get_literal(fsp_link *link, fs_rid rid, fs_rid *attr)
{
if (cache[rid & CACHE_MASK].rid == rid) {
*attr = cache[rid & CACHE_MASK].attr;
return (xmlChar *) cache[rid & CACHE_MASK].lex;
}
fs_rid_vector onerid = { .length = 1, .size = 1, .data = &rid };
fs_resource resource;
fsp_resolve(link, FS_RID_SEGMENT(rid, segments), &onerid, &resource);
*attr = resource.attr;
return (xmlChar *) resource.lex;
}
void resolve_triples(fsp_link *link, fs_rid_vector **rids)
{
int quads = rids[0]->length;
fs_rid_vector *todo[segments];
fs_segment segment;
for (segment = 0; segment < segments; ++segment) {
todo[segment] = fs_rid_vector_new(0);
}
for (int c = 0; c < 3; ++c) {
for (int k = 0; k < quads; ++k) {
const fs_rid rid = rids[c]->data[k];
if (FS_IS_BNODE(rid) || cache[rid & CACHE_MASK].rid == rid) continue;
fs_rid_vector_append(todo[FS_RID_SEGMENT(rid, segments)], rid);
cache[rid & CACHE_MASK].rid = rid; /* well, it will be soon */
}
}
int length[segments];
fs_resource *resources[segments];
for (segment = 0; segment < segments; ++segment) {
length[segment] = todo[segment]->length;
resources[segment] = calloc(length[segment], sizeof(fs_resource));
}
fsp_resolve_all(link, todo, resources);
for (segment = 0; segment < segments; ++segment) {
fs_resource *res = resources[segment];
for (int k = 0; k < length[segment]; ++k) {
free(cache[res[k].rid & CACHE_MASK].lex);
memcpy(&cache[res[k].rid & CACHE_MASK], &res[k], sizeof(fs_resource));
}
fs_rid_vector_free(todo[segment]);
free(resources[segment]);
}
}
void dump_model(fsp_link *link, fs_rid model, xmlTextWriterPtr xml)
{
fs_rid_vector none = { .length = 0, .size = 0, .data = 0 };
fs_rid_vector one = { .length = 1, .size = 1, .data = &model };
fs_rid_vector **results;
double then; /* for time keeping */
then = fs_time();
fsp_bind_first_all(link, BIND_SPO, &one, &none, &none, &none, &results, QUAD_LIMIT);
time_bind_first += (fs_time() - then);
while (results != NULL) {
long length = results[0]->length;
if (length == 0) break;
then = fs_time();
resolve_triples(link, results);
time_resolving += (fs_time() - then);
then = fs_time();
for (int k = 0; k < length; ++k) {
xmlTextWriterStartElement(xml, (xmlChar *) "triple");
for (int r = 0; r < 3; ++r) {
fs_rid rid = results[r]->data[k];
if (FS_IS_BNODE(rid)) {
unsigned long long node = FS_BNODE_NUM(rid);
xmlTextWriterWriteFormatElement(xml, (xmlChar *) "id", "%llu", node);
} else if (FS_IS_URI(rid)) {
xmlChar *uri = get_uri(link, rid);
xmlTextWriterWriteElement(xml, (xmlChar *) "uri", uri);
} else if (FS_IS_LITERAL(rid)) {
fs_rid attr;
xmlChar *lex = get_literal(link, rid, &attr);
if (attr == fs_c.empty) {
xmlTextWriterWriteElement(xml, (xmlChar *) "plainLiteral", lex);
} else if (FS_IS_URI(attr)) {
xmlChar *type = get_uri(link, attr);
xmlTextWriterStartElement(xml, (xmlChar *) "typedLiteral");
xmlTextWriterWriteString(xml, (xmlChar *) lex);
xmlTextWriterWriteAttribute(xml, (xmlChar *) "datatype", type);
xmlTextWriterEndElement(xml);
} else if (FS_IS_LITERAL(attr)) {
xmlChar *lang = get_attr(link, attr);
xmlTextWriterStartElement(xml, (xmlChar *) "plainLiteral");
xmlTextWriterWriteAttribute(xml, (xmlChar *) "xml:lang", lang);
xmlTextWriterWriteString(xml, (xmlChar *) lex);
xmlTextWriterEndElement(xml);
}
}
}
xmlTextWriterEndElement(xml);
xmlTextWriterWriteString(xml, (xmlChar *) "\n");
}
time_write_out += (fs_time() - then);
fs_rid_vector_free(results[0]);
fs_rid_vector_free(results[1]);
fs_rid_vector_free(results[2]);
free(results);
then = fs_time();
fsp_bind_next_all(link, BIND_SPO, &results, QUAD_LIMIT);
time_bind_next += (fs_time() - then);
}
fsp_bind_done_all(link);
}
void dump_trix(fsp_link *link, xmlTextWriterPtr xml)
{
fs_rid_vector **models;
fs_rid_vector none = { .length = 0, .size = 0, .data = 0 };
fsp_bind_all(link, FS_BIND_DISTINCT | FS_BIND_MODEL | FS_BIND_BY_SUBJECT, &none, &none, &none, &none, &models);
fs_rid_vector_sort(models[0]);
fs_rid_vector_uniq(models[0], 1);
long length = models[0]->length;
for (int k = 0; k < length; ++k) {
fs_rid model = models[0]->data[k];
xmlChar *model_uri = get_uri(link, model);
xmlTextWriterStartElement(xml, (xmlChar *) "graph");
if (FS_IS_URI(model)) {
xmlTextWriterWriteElement(xml, (xmlChar *) "uri", model_uri);
} else {
fs_error(LOG_WARNING, "model %lld is not a URI", model);
}
dump_model(link, model, xml);
xmlTextWriterEndElement(xml);
xmlTextWriterWriteString(xml, (xmlChar *) "\n");
printf("%5d/%ld: %4.5f %4.5f %4.5f %4.5f\n", k + 1, length, time_resolving, time_bind_first, time_bind_next, time_write_out);
}
}
void dump_file(fsp_link *link, char *filename)
{
xmlTextWriterPtr xml = xmlNewTextWriterFilename(filename, TRUE);
if (!xml) {
fs_error(LOG_ERR, "Couldn't write output file, giving up");
exit(4);
}
xmlTextWriterStartDocument(xml, NULL, NULL, NULL);
xmlTextWriterStartElement(xml, (xmlChar *) "TriX");
dump_trix(link, xml);
xmlTextWriterEndDocument(xml); /* also closes TriX */
xmlFreeTextWriter(xml);
}
int main(int argc, char *argv[])
{
char *password = fsp_argv_password(&argc, argv);
if (argc != 3) {
fprintf(stderr, "%s revision %s\n", argv[0], FS_FRONTEND_VER);
fprintf(stderr, "Usage: %s <kbname> <uri>\n", argv[0]);
exit(1);
}
fsp_link *link = fsp_open_link(argv[1], password, FS_OPEN_HINT_RO);
if (!link) {
fs_error (LOG_ERR, "couldn't connect to “%s”", argv[1]);
exit(2);
}
fs_hash_init(fsp_hash_type(link));
segments = fsp_link_segments(link);
dump_file(link, argv[2]);
fsp_close_link(link);
}
int main(int argc, char *argv[])
{
int verbosity = 0;
int dryrun = 0;
char *password = NULL;
char *format = "auto";
FILE *msg = stderr;
char *optstring = "am:M:vnf:";
int c, opt_index = 0, help = 0;
int files = 0, adding = 0;
char *kb_name = NULL;
char *model[argc], *uri[argc];
char *model_default = NULL;
password = fsp_argv_password(&argc, argv);
static struct option long_options[] = {
{ "add", 0, 0, 'a' },
{ "model", 1, 0, 'm' },
{ "model-default", 1, 0, 'M' },
{ "verbose", 0, 0, 'v' },
{ "dryrun", 0, 0, 'n' },
{ "no-resources", 0, 0, 'R' },
{ "no-quads", 0, 0, 'Q' },
{ "format", 1, 0, 'f' },
{ "help", 0, 0, 'h' },
{ "version", 0, 0, 'V' },
{ 0, 0, 0, 0 }
};
for (int i= 0; i < argc; ++i) {
model[i] = NULL;
}
int help_return = 1;
while ((c = getopt_long (argc, argv, optstring, long_options, &opt_index)) != -1) {
if (c == 'm') {
model[files++] = optarg;
} else if (c == 'M') {
model_default = optarg;
} else if (c == 'v') {
verbosity++;
} else if (c == 'a') {
adding = 1;
} else if (c == 'n') {
dryrun |= FS_DRYRUN_DELETE | FS_DRYRUN_RESOURCES | FS_DRYRUN_QUADS;
} else if (c == 'R') {
dryrun |= FS_DRYRUN_RESOURCES;
} else if (c == 'Q') {
dryrun |= FS_DRYRUN_QUADS;
} else if (c == 'f') {
format = optarg;
} else if (c == 'h') {
help = 1;
help_return = 0;
} else if (c == 'V') {
printf("%s, built for 4store %s\n", argv[0], GIT_REV);
exit(0);
} else {
help = 1;
}
}
if (verbosity > 0) {
if (dryrun & FS_DRYRUN_DELETE) {
printf("warning: not deleting old model\n");
}
if (dryrun & FS_DRYRUN_RESOURCES) {
printf("warning: not importing resource nodes\n");
}
if (dryrun & FS_DRYRUN_QUADS) {
printf("warning: not importing quad graph\n");
}
}
files = 0;
for (int k = optind; k < argc; ++k) {
if (!kb_name) {
kb_name = argv[k];
} else {
if (strchr(argv[k], ':')) {
uri[files] = g_strdup(argv[k]);
} else {
uri[files] = (char *)raptor_uri_filename_to_uri_string(argv[k]);
}
if (!model[files]) {
if (!model_default) {
model[files] = uri[files];
} else {
model[files] = model_default;
}
}
files++;
}
}
raptor_world *rw = raptor_new_world();
if (help || !kb_name || files == 0) {
fprintf(stdout, "%s revision %s\n", argv[0], FS_FRONTEND_VER);
fprintf(stdout, "Usage: %s <kbname> <rdf file/URI> ...\n", argv[0]);
fprintf(stdout, " -v --verbose increase verbosity (can repeat)\n");
fprintf(stdout, " -a --add add data to models instead of replacing\n");
fprintf(stdout, " -m --model specify a model URI for the next RDF file\n");
fprintf(stdout, " -M --model-default specify a model URI for all RDF files\n");
fprintf(stdout, " -f --format specify an RDF syntax for the import\n");
fprintf(stdout, "\n available formats are:\n");
for (unsigned int i=0; 1; i++) {
const raptor_syntax_description *desc =
raptor_world_get_parser_description(rw, i);
if (!desc) {
break;
}
fprintf(stdout, " %12s - %s\n", desc->names[0], desc->label);
}
exit(help_return);
}
fsp_syslog_enable();
fsplink = fsp_open_link(kb_name, password, FS_OPEN_HINT_RW);
if (!fsplink) {
fs_error (LOG_ERR, "couldn't connect to “%s”", kb_name);
exit(2);
}
const char *features = fsp_link_features(fsplink);
int has_o_index = !(strstr(features, "no-o-index")); /* tweak */
fs_hash_init(fsp_hash_type(fsplink));
const int segments = fsp_link_segments(fsplink);
int total_triples = 0;
fs_import_timing timing[segments];
for (int seg = 0; seg < segments; seg++) {
fsp_get_import_times(fsplink, seg, &timing[seg]);
}
gettimeofday(&then, 0);
if (fsp_start_import_all(fsplink)) {
fs_error(LOG_ERR, "aborting import");
exit(3);
}
#if 0
printf("press enter\n");
char foo;
read(0, &foo, 1);
#endif
fs_rid_vector *mvec = fs_rid_vector_new(0);
for (int f= 0; f < files; ++f) {
fs_rid muri = fs_hash_uri(model[f]);
fs_rid_vector_append(mvec, muri);
}
if (!adding) {
if (verbosity) {
printf("removing old data\n");
fflush(stdout);
}
if (!(dryrun & FS_DRYRUN_DELETE)) {
if (fsp_delete_model_all(fsplink, mvec)) {
fs_error(LOG_ERR, "model delete failed");
return 1;
}
for (int i=0; i<mvec->length; i++) {
if (mvec->data[i] == fs_c.system_config) {
fs_import_reread_config();
}
}
}
fsp_new_model_all(fsplink, mvec);
}
fs_rid_vector_free(mvec);
gettimeofday(&then_last, 0);
for (int f = 0; f < files; ++f) {
if (verbosity) {
printf("Reading <%s>\n", uri[f]);
if (strcmp(uri[f], model[f])) {
printf(" into <%s>\n", model[f]);
}
fflush(stdout);
}
fs_import(fsplink, model[f], uri[f], format, verbosity, dryrun, has_o_index, msg, &total_triples);
if (verbosity) {
fflush(stdout);
}
}
double sthen = fs_time();
int ret = fs_import_commit(fsplink, verbosity, dryrun, has_o_index, msg, &total_triples);
if (verbosity > 0) {
printf("Updating index\n");
fflush(stdout);
}
fsp_stop_import_all(fsplink);
if (verbosity > 0) {
printf("Index update took %f seconds\n", fs_time()-sthen);
}
if (!ret) {
gettimeofday(&now, 0);
double diff = (now.tv_sec - then.tv_sec) +
(now.tv_usec - then.tv_usec) * 0.000001;
if (verbosity && total_triples > 0) {
printf("Imported %d triples, average %d triples/s\n", total_triples,
(int)((double)total_triples/diff));
fflush(stdout);
}
}
if (verbosity > 1) {
printf("seg add_q\tadd_r\t\tcommit_q\tcommit_r\tremove\t\trebuild\t\twrite\n");
long long *tics = fsp_profile_write(fsplink);
for (int seg = 0; seg < segments; seg++) {
fs_import_timing newtimes;
fsp_get_import_times(fsplink, seg, &newtimes);
printf("%2d: %f\t%f\t%f\t%f\t%f\t%f\t%f\n", seg,
newtimes.add_s - timing[seg].add_s,
newtimes.add_r - timing[seg].add_r,
newtimes.commit_q - timing[seg].commit_q,
newtimes.commit_r - timing[seg].commit_r,
newtimes.remove - timing[seg].remove,
newtimes.rebuild - timing[seg].rebuild,
tics[seg] * 0.001);
}
}
fsp_close_link(fsplink);
raptor_free_world(rw);
return 0;
}
static int update_op(struct update_context *ct)
{
fs_rid_vector *vec[4];
switch (ct->op->type) {
case RASQAL_UPDATE_TYPE_UNKNOWN:
add_message(ct, "Unknown update operation", 0);
return 1;
case RASQAL_UPDATE_TYPE_CLEAR:
fs_clear(ct, (char *)raptor_uri_as_string(ct->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_CREATE:
return 0;
case RASQAL_UPDATE_TYPE_DROP:
fs_clear(ct, (char *)raptor_uri_as_string(ct->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_LOAD:
fs_load(ct, (char *)raptor_uri_as_string(ct->op->document_uri),
(char *)raptor_uri_as_string(ct->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_UPDATE:
break;
}
fs_hash_freshen();
raptor_sequence *todel = NULL;
raptor_sequence *toins = NULL;
if (ct->op->where) {
todel = raptor_new_sequence(NULL, NULL);
toins = raptor_new_sequence(NULL, NULL);
raptor_sequence *todel_p = raptor_new_sequence(NULL, NULL);
raptor_sequence *toins_p = raptor_new_sequence(NULL, NULL);
raptor_sequence *vars = raptor_new_sequence(NULL, NULL);
fs_query *q = calloc(1, sizeof(fs_query));
ct->q = q;
q->qs = ct->qs;
q->rq = ct->rq;
q->flags = FS_BIND_DISTINCT;
q->opt_level = 3;
q->soft_limit = -1;
q->segments = fsp_link_segments(ct->link);
q->link = ct->link;
q->bb[0] = fs_binding_new();
q->bt = q->bb[0];
/* add column to denote join ordering */
fs_binding_add(q->bb[0], "_ord", FS_RID_NULL, 0);
struct pattern_data pd = { .q = q, .vars = vars, .patterns = NULL, .fixed = NULL };
if (ct->op->delete_templates) {
pd.patterns = todel_p;
pd.fixed = todel;
for (int t=0; t<raptor_sequence_size(ct->op->delete_templates); t++) {
rasqal_graph_pattern *gp = raptor_sequence_get_at(ct->op->delete_templates, t);
assign_gp(gp, NULL, &pd);
}
}
if (ct->op->insert_templates) {
pd.patterns = toins_p;
pd.fixed = toins;
for (int t=0; t<raptor_sequence_size(ct->op->insert_templates); t++) {
rasqal_graph_pattern *gp = raptor_sequence_get_at(ct->op->insert_templates, t);
assign_gp(gp, NULL, &pd);
}
}
q->num_vars = raptor_sequence_size(vars);
for (int i=0; i < q->num_vars; i++) {
rasqal_variable *v = raptor_sequence_get_at(vars, i);
fs_binding *b = fs_binding_get(q->bb[0], (char *)v->name);
if (b) {
b->need_val = 1;
} else {
fs_binding_add(q->bb[0], (char *)v->name, FS_RID_NULL, 1);
}
}
fs_query_process_pattern(q, ct->op->where, vars);
q->length = fs_binding_length(q->bb[0]);
for (int s=0; s<4; s++) {
vec[s] = fs_rid_vector_new(0);
}
for (int t=0; t<raptor_sequence_size(todel_p); t++) {
rasqal_triple *triple = raptor_sequence_get_at(todel_p, t);
for (int row=0; row < q->length; row++) {
delete_rasqal_triple(ct, vec, triple, row);
if (fs_rid_vector_length(vec[0]) > 0) {
fsp_delete_quads_all(ct->link, vec);
}
}
}
for (int s=0; s<4; s++) {
//fs_rid_vector_print(vec[s], 0, stdout);
fs_rid_vector_free(vec[s]);
}
for (int t=0; t<raptor_sequence_size(toins_p); t++) {
rasqal_triple *triple = raptor_sequence_get_at(toins_p, t);
for (int row=0; row < q->length; row++) {
insert_rasqal_triple(ct, triple, row);
}
}
/* must not free the rasqal_query */
q->rq = NULL;
fs_query_free(q);
ct->q = NULL;
} else {