int main(int argc, char *argv[])
{
char *password = fsp_argv_password(&argc, argv);
if (argc != 2) {
fprintf(stderr, "%s revision %s\n", argv[0], FS_FRONTEND_VER);
fprintf(stderr, "Usage: %s <kbname>\n", argv[0]);
return 1;
}
fsp_syslog_enable();
fsp_link *link = fsp_open_link(argv[1], password, FS_OPEN_HINT_RO);
if (!link) {
fs_error(LOG_ERR, "couldn't connect to “%s”", argv[1]);
return 2;
}
if (fsp_no_op(link, 0)) {
fs_error(LOG_ERR, "NO-OP failed\n");
return 3;
}
fs_data_size total = {0, 0, 0, 0, 0, 0};
fs_data_size sz;
if (fsp_get_data_size(link, 0, &sz)) {
fs_error(LOG_ERR, "cannot get size information");
exit(2);
}
printf("%5s%12s%12s%12s%12s\n", "seg","quads (s)","quads (sr)",
"models", "resources");
const int segments = fsp_link_segments(link);
for (fs_segment seg = 0; seg < segments; ++seg) {
fs_data_size sz;
int ret = fsp_get_data_size(link, seg, &sz);
if (ret) {
printf("%5d -- problem obtaining size information --\n", seg);
} else {
printf("%5d%12lld%+12lld%12lld%12lld\n", seg,
sz.quads_s, sz.quads_sr - sz.quads_s,
sz.models_s, sz.resources);
total.quads_s += sz.quads_s;
total.quads_sr += sz.quads_sr;
if (sz.models_s > total.models_s) total.models_s = sz.models_s;
total.resources += sz.resources;
}
}
printf("\n");
printf("%5s%12lld%+12lld%12lld%12lld\n",
"TOTAL", total.quads_s, (total.quads_sr - total.quads_s),
total.models_s, total.resources);
printf("\ncollision probability ≅ %.2Lf%%\n", 100.0 * (1.0 - expl(-(total.resources * (total.resources-1.0) / (2.0 * CONST_2to63)))));
fsp_close_link(link);
}
/* ..._start and ..._finish share an int * count parameter
* the same variable should be passed by reference both times */
int fs_import_stream_start(fsp_link *link, const char *model_uri, const char *mimetype, int has_o_index, int *count)
{
if (inited == 0) {
memset(&parse_data, 0, sizeof(parse_data));
inited = 1;
}
parse_data.link = link;
parse_data.segments = fsp_link_segments(link);
parse_data.ext_count = count;
for (int i=0; i<parse_data.segments; i++) {
for (int j=0; j<RES_BUF_SIZE; j++) {
lex_tmp[i][j] = malloc(RES_BUF_SIZE);
}
}
memset(nodecache, 0, sizeof(nodecache));
parse_data.quad_fn = g_strdup(FS_TMP_PATH "/importXXXXXX");
parse_data.quad_fd = mkstemp(parse_data.quad_fn);
if (parse_data.quad_fd < 0) {
fs_error(LOG_ERR, "Cannot create tmp file “%s”", parse_data.quad_fn);
return 1;
}
parse_data.muri = raptor_new_uri((unsigned char *) model_uri);
parse_data.model = g_strdup(model_uri);
parse_data.model_hash = fs_hash_uri(model_uri);
parse_data.count_trip = 0;
parse_data.count_err = 0;
parse_data.last_count = 0;
parse_data.has_o_index = has_o_index;
/* store the model uri */
buffer_res(link, parse_data.segments, parse_data.model_hash, parse_data.model, FS_RID_NULL, parse_data.dryrun);
parse_data.parser = raptor_new_parser_for_content(NULL, mimetype, NULL, 0, (unsigned char *) parse_data.model);
if (!parse_data.parser) {
return 1;
}
/* use us as a vector for an indirect attack? no thanks */
raptor_set_feature(parse_data.parser, RAPTOR_FEATURE_NO_NET, 0);
raptor_set_fatal_error_handler(parse_data.parser, link, fatal_rdf_parser_error);
raptor_set_error_handler(parse_data.parser, link, rdf_parser_error);
raptor_set_statement_handler(parse_data.parser, &parse_data, store_stmt);
raptor_set_graph_handler(parse_data.parser, &parse_data, graph_handler);
raptor_start_parse(parse_data.parser, parse_data.muri);
fs_hash_freshen(); /* blank nodes are unique per file */
return 0;
}
fs_value fn_datatype(fs_query *q, fs_value a)
{
#if 0
printf("datatype(");
fs_value_print(a);
printf(")\n");
#endif
if (a.valid & fs_valid_bit(FS_V_TYPE_ERROR)) {
return a;
}
if (a.valid & fs_valid_bit(FS_V_RID) && FS_IS_URI_BN(a.rid)) {
return fs_value_error(FS_ERROR_INVALID_TYPE, NULL);
}
if (a.attr == FS_RID_NULL) {
return fs_value_error(FS_ERROR_INVALID_TYPE, NULL);
}
if (FS_IS_LITERAL(a.attr) && a.attr != fs_c.empty) {
return fs_value_error(FS_ERROR_INVALID_TYPE, NULL);
} else {
if (a.attr == fs_c.xsd_string || a.attr == fs_c.empty) {
return fs_value_uri(XSD_STRING);
} else if (a.attr == fs_c.xsd_double) {
return fs_value_uri(XSD_DOUBLE);
} else if (a.attr == fs_c.xsd_float) {
return fs_value_uri(XSD_FLOAT);
} else if (a.attr == fs_c.xsd_decimal) {
return fs_value_uri(XSD_DECIMAL);
} else if (a.attr == fs_c.xsd_integer) {
return fs_value_uri(XSD_INTEGER);
} else if (a.attr == fs_c.xsd_boolean) {
return fs_value_uri(XSD_BOOLEAN);
} else if (a.attr == fs_c.xsd_datetime) {
return fs_value_uri(XSD_DATETIME);
}
}
fs_rid_vector *r = fs_rid_vector_new(1);
r->data[0] = a.attr;
fs_resource res;
if (fs_query_link(q)) {
fsp_resolve(fs_query_link(q), FS_RID_SEGMENT(a.attr,
fsp_link_segments(fs_query_link(q))), r, &res);
fs_rid_vector_free(r);
return fs_value_uri(res.lex);
}
return fs_value_uri("error:unresloved");
}
int fs_import_stream_finish(fsp_link *link, int *count, int *errors)
{
raptor_parse_chunk(parse_data.parser, NULL, 0, 1); /* finish */
raptor_free_parser(parse_data.parser);
raptor_free_uri(parse_data.muri);
g_free(parse_data.model);
const int segments = fsp_link_segments(link);
*count += process_quads(&parse_data);
close(parse_data.quad_fd);
unlink(parse_data.quad_fn);
free(parse_data.quad_fn);
parse_data.quad_fd = 0xdeadbeef;
parse_data.quad_fn = NULL;
/* make sure buffers are flushed */
for (int seg = 0; seg < segments; seg++) {
if (res_pos[seg] > 0 && fsp_res_import(link, seg, res_pos[seg], res_buffer[seg])) {
fs_error(LOG_ERR, "resource import failed");
return 1;
}
}
if (fsp_res_import_commit_all(link)) {
fs_error(LOG_ERR, "resource commit failed");
return 2;
}
for (int seg = 0; seg < segments; seg++) {
for (int i=0; i<res_pos[seg]; i++) {
if (res_buffer[seg][i].lex != lex_tmp[seg][i]) {
free(res_buffer[seg][i].lex);
}
}
res_pos[seg] = 0;
}
if (fsp_quad_import_commit_all(link, FS_BIND_BY_SUBJECT)) {
fs_error(LOG_ERR, "quad commit failed");
return 3;
}
for (int i=0; i<segments; i++) {
for (int j=0; j<RES_BUF_SIZE; j++) {
free(lex_tmp[i][j]);
lex_tmp[i][j] = NULL;
}
}
*errors = parse_data.count_err;
return 0;
}
int fs_import_commit(fsp_link *link, int verbosity, int dryrun, int has_o_index, FILE *msg, int *count)
{
const int segments = fsp_link_segments(link);
*count += process_quads(&parse_data);
close(parse_data.quad_fd);
unlink(parse_data.quad_fn);
free(parse_data.quad_fn);
parse_data.quad_fd = 0xdeadbeef;
parse_data.quad_fn = NULL;
/* make sure buffers are flushed */
for (int seg = 0; seg < segments; seg++) {
if (!(dryrun & FS_DRYRUN_RESOURCES) && res_pos[seg] > 0 && fsp_res_import(link, seg, res_pos[seg], res_buffer[seg])) {
fs_error(LOG_ERR, "resource import failed");
return 1;
}
}
if (!(dryrun & FS_DRYRUN_RESOURCES) && fsp_res_import_commit_all(link)) {
fs_error(LOG_ERR, "resource commit failed");
return 2;
}
for (int seg = 0; seg < segments; seg++) {
for (int i=0; i<res_pos[seg]; i++) {
if (res_buffer[seg][i].lex != lex_tmp[seg][i]) {
free(res_buffer[seg][i].lex);
}
}
res_pos[seg] = 0;
}
if (!(dryrun & FS_DRYRUN_QUADS) && fsp_quad_import_commit_all(link, FS_BIND_BY_SUBJECT)) {
fs_error(LOG_ERR, "quad commit failed");
return 3;
}
for (int i=0; i<segments; i++) {
for (int j=0; j<RES_BUF_SIZE; j++) {
free(lex_tmp[i][j]);
lex_tmp[i][j] = NULL;
}
}
inited = 0;
return 0;
}
void restore_file(fsp_link *link, char *filename)
{
xmlctxt *ctxt = calloc(sizeof(xmlctxt), 1);
ctxt->link = link;
ctxt->segments = fsp_link_segments(link);
for (int s = 0; s < ctxt->segments; ++s) {
res_count[s] = 0;
}
xmlSAXUserParseFile(&sax, (void *) ctxt, filename);
flush_resources(ctxt);
flush_quads(ctxt);
free(ctxt);
}
static char *get_lex(fsp_link *link, fs_rid rid)
{
if (rid == FS_RID_NULL) return g_strdup("*");
const int segments = fsp_link_segments(link);
fs_resource res;
fs_rid_vector *rv = fs_rid_vector_new(1);
rv->data[0] = rid;
fsp_resolve(link, FS_RID_SEGMENT(rid, segments), rv, &res);
fs_rid_vector_free(rv);
if (!strncmp(RDF_NAMESPACE, res.lex, strlen(RDF_NAMESPACE))) {
char *new = g_strdup_printf("rdf:%s", res.lex + strlen(RDF_NAMESPACE));
g_free(res.lex);
return new;
}
fs_value fn_lang(fs_query *q, fs_value a)
{
if (a.valid & fs_valid_bit(FS_V_TYPE_ERROR)) {
return a;
}
if (a.valid & fs_valid_bit(FS_V_RID) && FS_IS_URI_BN(a.rid)) {
return fs_value_error(FS_ERROR_INVALID_TYPE, NULL);
}
if (a.attr == FS_RID_NULL) {
return fs_value_error(FS_ERROR_INVALID_TYPE, NULL);
}
if (FS_IS_URI(a.attr)) {
return fs_value_plain("");
} else {
if (a.attr == fs_c.lang_en) {
return fs_value_plain("en");
} else if (a.attr == fs_c.lang_fr) {
return fs_value_plain("fr");
} else if (a.attr == fs_c.lang_de) {
return fs_value_plain("de");
} else if (a.attr == fs_c.lang_es) {
return fs_value_plain("es");
} else if (a.attr == fs_c.empty) {
return fs_value_plain("");
}
}
fs_rid_vector *r = fs_rid_vector_new(1);
r->data[0] = a.attr;
fs_resource res;
if (fs_query_link(q)) {
fsp_resolve(fs_query_link(q), FS_RID_SEGMENT(a.attr,
fsp_link_segments(fs_query_link(q))), r, &res);
fs_rid_vector_free(r);
fs_query_add_freeable(q, res.lex);
return fs_value_plain(res.lex);
}
return fs_value_plain("???");
}
int main(int argc, char *argv[])
{
char *password = fsp_argv_password(&argc, argv);
int flags = 0, many = 0, all = 0;
int seg = 0; /* deliberately using signed type */
fs_rid_vector *mrids= NULL, *srids= NULL, *prids= NULL, *orids= NULL;
fs_rid_vector **result = NULL;
if (argc < 7) {
fprintf(stderr, "%s revision %s\n", argv[0], FS_FRONTEND_VER);
fprintf(stderr, "Usage: %s <kbname> { many | all | seg# } <flags>\n", argv[0]);
fprintf(stderr, " mrid-file srid-file prid-file orid-file [offset limit]\n");
fprintf(stderr, "For flags use FS_BIND_... symbols or a numeric value\n");
fprintf(stderr, "RID files are one RID per line\n");
exit(1);
}
char *kbname = argv[1];
if (!strcasecmp(argv[2], "many")) {
many = 1;
} else if (!strcasecmp(argv[2], "all")) {
all = 1;
} else {
seg = atoi(argv[2]);
}
int param = 3;
flags = strtol(argv[param], NULL, 0);
if (flags == 0) { /* symbolic flags, hopefully */
while (param < argc) {
const int len = sizeof(flag_name) / sizeof(char *);
int k;
for (k = 0; k < len; ++k) {
if (!strcmp(flag_name[k], argv[param])) {
flags |= flag_value[k];
break;
}
}
if (k == len) break;
param ++;
}
} else {
param ++; /* done with the numeric flags then */
}
if (argc < param + 4) {
fprintf(stderr, "Wrong number of arguments\n");
exit(1);
}
mrids = rid_file(argv[param++]);
srids = rid_file(argv[param++]);
prids = rid_file(argv[param++]);
orids = rid_file(argv[param++]);
int limit, offset;
if (argc == param ) {
/* defaults */
limit = -1;
offset = -1;
} else if (argc > param + 2) {
fprintf(stderr, "Wrong number of arguments\n");
exit(1);
} else if (argc < param + 2) {
fprintf(stderr, "Wrong number of arguments\n");
exit(1);
} else {
offset = atoi(argv[param]);
limit = atoi(argv[param + 1]);
}
fsp_link *link = fsp_open_link(kbname, password, FS_OPEN_HINT_RO);
if (!link) {
fs_error (LOG_ERR, "couldn't connect to “%s”", argv[1]);
exit(2);
}
segments = fsp_link_segments(link);
if (seg < 0 || seg > segments) {
fs_error (LOG_ERR, "Segment %d out of range (0-%u)", seg, segments);
exit(1);
}
double then = fs_time();
int ans = 0;
if (all) {
ans = fsp_bind_limit_all(link, flags, mrids, srids, prids, orids, &result, offset, limit);
} else if (many) {
ans = fsp_bind_limit_many(link, flags, mrids, srids, prids, orids, &result, offset, limit);
} else {
ans = fsp_bind_limit(link, seg, flags, mrids, srids, prids, orids, &result, offset, limit);
}
double time_binding = fs_time() - then;
if (ans != 0) exit(1);
/* print results */
int cols = 0;
for (int k = 0; k < 4; ++k) {
if (flags & (1 << k)) cols++;
}
if (!result) {
printf("NO MATCH found.\n");
} else if (cols == 0) {
printf("MATCH found.\n");
} else if (result[0]) {
int length = result[0]->length;
if (flags & FS_BIND_MODEL) printf("-----Model------ ");
if (flags & FS_BIND_SUBJECT) printf("----Subject----- ");
if (flags & FS_BIND_PREDICATE) printf("----Predicate--- ");
if (flags & FS_BIND_OBJECT) printf("-----Object-----");
putchar('\n');
for (int k = 0; k < length; ++k) {
for (int c = 0; c < cols; ++c) {
printf("%016llX ", result[c]->data[k]);
}
putchar('\n');
}
}
fprintf(stderr, "bind took %f seconds on client\n", time_binding);
fs_query_timing times;
if (all || many) {
fprintf(stderr, "binding on all or many segments, times in seconds...\n");
for (int s = 0; s < segments; ++s) {
fsp_get_query_times(link, s, ×);
if (times.bind > 0.0f) {
fprintf(stderr, "%d: %f\n", s, times.bind);
}
}
fputc('\n', stderr);
} else {
fsp_get_query_times(link, seg, ×);
fprintf(stderr, "binding segment %d took %f seconds\n", seg, times.bind);
}
fsp_close_link(link);
}
int main(int argc, char *argv[])
{
char *password = fsp_argv_password(&argc, argv);
static char *optstring = "hevf:PO:Ib:rs:d";
char *format = getenv("FORMAT");
char *kb_name = NULL, *query = NULL;
int programatic = 0, help = 0;
int c, opt_index = 0;
int verbosity = 0;
int opt_level = 3;
int insert_mode = 0;
int restricted = 0;
int soft_limit = 0;
int explain = 0;
int default_graph = 0;
char *base_uri = "local:";
raptor_world *rw = NULL;
static struct option long_options[] = {
{ "help", 0, 0, 'h' },
{ "version", 0, 0, 'V' },
{ "verbose", 0, 0, 'v' },
{ "explain", 0, 0, 'e' },
{ "format", 1, 0, 'f' },
{ "programatic", 0, 0, 'P' },
{ "opt-level", 1, 0, 'O' },
{ "insert", 0, 0, 'I' },
{ "restricted", 0, 0, 'r' },
{ "soft-limit", 1, 0, 's' },
{ "default-graph", 0, 0, 'd' },
{ "base", 1, 0, 'b' },
{ 0, 0, 0, 0 }
};
int help_return = 1;
while ((c = getopt_long (argc, argv, optstring, long_options, &opt_index)) != -1) {
if (c == 'f') {
format = optarg;
} else if (c == 'P') {
programatic = TRUE;
} else if (c == 'v') {
verbosity++;
} else if (c == 'O') {
opt_level = atoi(optarg);
} else if (c == 'I') {
insert_mode = 1;
} else if (c == 'r') {
restricted = 1;
} else if (c == 's') {
soft_limit = atoi(optarg);
} else if (c == 'd') {
default_graph = 1;
} else if (c == 'b') {
base_uri = optarg;
} else if (c == 'h') {
help = 1;
help_return = 0;
} else if (c == 'e') {
explain = 1;
} else if (c == 'V') {
printf("%s, built for 4store %s\n", argv[0], GIT_REV);
exit(0);
} else {
help = 1;
}
}
for (int k = optind; k < argc; ++k) {
if (!kb_name) {
kb_name = argv[k];
} else if (!query && !programatic) {
query = argv[k];
} else {
help = 1;
}
}
if (help || !kb_name) {
char *langs = "";
if (fs_query_have_laqrs()) {
langs = "/LAQRS";
}
fprintf(stdout, "%s revision %s\n", basename(argv[0]), GIT_REV);
fprintf(stdout, "Usage: %s <kbname> [-f format] [-O opt-level] [-I] [-b uri] [query]\n", argv[0]);
fprintf(stdout, " or: %s <kbname> -P\n", basename(argv[0]));
fprintf(stdout, " query is a SPARQL%s query, remember to use"
" shell quoting if necessary\n", langs);
fprintf(stdout, " -f Output format one of, sparql, text, json, or testcase\n");
fprintf(stdout, " -O, --opt-level Set optimisation level, range 0-3\n");
fprintf(stdout, " -I, --insert Interpret CONSTRUCT statements as inserts\n");
fprintf(stdout, " -r, --restricted Enable query complexity restriction\n");
fprintf(stdout, " -s, --soft-limit Override default soft limit on search breadth\n");
fprintf(stdout, " -d, --default-graph Enable SPARQL default graph support\n");
fprintf(stdout, " -b, --base Set base URI for query\n");
fprintf(stdout, " -e, --explain Show explain results for execution plan\n");
exit(help_return);
}
if (programatic || query) {
/* don't syslog interactive errors */
fsp_syslog_enable();
}
double then = ftime();
/* if query does UPDATE or DELETE operations this needs a re-think */
fsp_link *link = fsp_open_link(kb_name, password, FS_OPEN_HINT_RO);
double now = ftime();
show_timing = (getenv("SHOW_TIMING") != NULL) | (verbosity > 1);
if (show_timing) {
printf("Link open time: %f\n", now-then);
}
if (!link) {
fs_error(LOG_ERR, "couldn't connect to “%s”", kb_name);
return 2;
}
const int segments = fsp_link_segments(link);
fs_query_timing timing[segments];
memset(timing, 0, sizeof(fs_query_timing) * segments);
if (show_timing) {
for (int seg = 0; seg < segments; seg++) {
fsp_get_query_times(link, seg, timing+seg);
}
}
if (fsp_no_op(link, 0)) {
fs_error(LOG_ERR, "NO-OP failed for “%s”", kb_name);
return 2;
}
rw = raptor_new_world();
fs_hash_init(fsp_hash_type(link));
raptor_uri *bu = raptor_new_uri(rw, (unsigned char *)base_uri);
unsigned int flags = FS_QUERY_CONSOLE_OUTPUT; /* signal that we're using the */
/* console, allows better explain functionality */
flags |= insert_mode ? FS_RESULT_FLAG_CONSTRUCT_AS_INSERT : 0;
flags |= restricted ? FS_QUERY_RESTRICTED : 0;
flags |= default_graph ? FS_QUERY_DEFAULT_GRAPH : 0;
if (programatic) {
programatic_io(link, bu, "sparql", format, timing, verbosity, opt_level,
FS_RESULT_FLAG_HEADERS | flags, soft_limit, rw);
} else if (!query) {
if (!format) format = "text";
interactive(link, bu, format, verbosity, opt_level,
insert_mode ? FS_RESULT_FLAG_CONSTRUCT_AS_INSERT : flags,
soft_limit, rw);
}
int ret = 0;
fs_query_state *qs = fs_query_init(link, NULL, NULL);
qs->verbosity = verbosity;
fs_query *qr = fs_query_execute(qs, link, bu, query, flags, opt_level, soft_limit, explain);
if (fs_query_errors(qr)) {
ret = 1;
}
fs_query_results_output(qr, format, 0, stdout);
fs_query_free(qr);
if (show_timing) {
printf("seg bind\t(secs)\t\tprice\t(secs)\t\tresolve\t(secs)\t\twait (secs)\n");
long long *tics = fsp_profile_write(link);
for (int seg = 0; seg < segments; seg++) {
fs_query_timing newtimes;
fsp_get_query_times(link, seg, &newtimes);
printf("%2d: %4d\t%f\t%4d\t%f\t%4d\t%f\t%f\n", seg,
newtimes.bind_count - timing[seg].bind_count,
newtimes.bind - timing[seg].bind,
newtimes.price_count - timing[seg].price_count,
newtimes.price - timing[seg].price,
newtimes.resolve_count - timing[seg].resolve_count,
newtimes.resolve - timing[seg].resolve,
tics[seg] * 0.001);
}
}
raptor_free_uri(bu);
raptor_free_world(rw);
fs_query_cache_flush(qs, verbosity);
fs_query_fini(qs);
fsp_close_link(link);
return ret;
}
static void programatic_io(fsp_link *link, raptor_uri *bu, const char *query_lang, const char *result_format, fs_query_timing *timing, int verbosity, int opt_level, unsigned int result_flags, int soft_limit, raptor_world *rw)
{
char query[MAX_Q_SIZE];
char *pos;
char *newl;
const int segments = fsp_link_segments(link);
fs_query_state *qs = fs_query_init(link, NULL, NULL);
qs->verbosity = verbosity;
do {
pos = query;
*query = '\0';
/* assemble query string */
do {
newl = fgets(pos, query + MAX_Q_SIZE - pos - 1, stdin);
if (newl) {
pos += strlen(newl);
}
} while (newl && strcmp(newl, "#EOQ\n") && strcmp(newl, "#END\n"));
/* process query string */
if (*query && strcmp(query, "#EOQ\n") && strcmp(query, "#END\n")) {
if (show_timing) {
printf("Q: %s\n", query);
}
fs_query *tq = fs_query_execute(qs, link, bu, query,
result_flags, opt_level, soft_limit, 0);
fs_query_results_output(tq, result_format, 0, stdout);
if (show_timing) {
printf("# time: %f s\n", fs_time() - fs_query_start_time(tq));
printf("seg bind\t(secs)\t\tprice\t(secs)\t\tresolve\t(secs)\t\twait (secs)\n");
long long *tics = fsp_profile_write(link);
fs_query_timing total_time = {0, 0, 0, 0, 0, 0};
for (int seg = 0; seg < segments; seg++) {
fs_query_timing newtimes;
fsp_get_query_times(link, seg, &newtimes);
printf("%2d: %4d\t%f\t%4d\t%f\t%4d\t%f\t%f\n", seg,
newtimes.bind_count - timing[seg].bind_count,
newtimes.bind - timing[seg].bind,
newtimes.price_count - timing[seg].price_count,
newtimes.price - timing[seg].price,
newtimes.resolve_count - timing[seg].resolve_count,
newtimes.resolve - timing[seg].resolve,
tics[seg] * 0.001);
total_time.bind_count += newtimes.bind_count - timing[seg].bind_count;
total_time.bind += newtimes.bind- timing[seg].bind;
total_time.price_count += newtimes.price_count - timing[seg].price_count;
total_time.price += newtimes.price - timing[seg].price;
total_time.resolve_count += newtimes.resolve_count - timing[seg].resolve_count;
total_time.resolve += newtimes.resolve - timing[seg].resolve;
}
printf("TT: %4d\t%f\t%4d\t%f\t%4d\t%f\n",
total_time.bind_count, total_time.bind,
total_time.price_count, total_time.price,
total_time.resolve_count, total_time.resolve);
}
fs_query_free(tq);
if (result_format && !strcmp(result_format, "sparql")) {
printf("<!-- EOR -->\n");
} else {
printf("#EOR\n");
}
fflush(stdout);
}
} while (newl && strcmp(newl, "#END\n"));
raptor_free_uri(bu);
fsp_close_link(link);
raptor_free_world(rw);
fs_query_cache_flush(qs, verbosity);
fs_query_fini(qs);
exit(0);
}
int fs_update(fs_query_state *qs, char *update, char **message, int unsafe)
{
rasqal_query *rq = rasqal_new_query(qs->rasqal_world, "sparql11-update", NULL);
if (!rq) {
*message = g_strdup_printf("Unable to initialise update parser");
return 1;
}
struct update_context uctxt;
rasqal_world_set_log_handler(qs->rasqal_world, &uctxt, error_handler);
memset(&uctxt, 0, sizeof(uctxt));
uctxt.link = qs->link;
uctxt.segments = fsp_link_segments(qs->link);
uctxt.qs = qs;
uctxt.rq = rq;
raptor_uri *bu = raptor_new_uri(qs->raptor_world, (unsigned char *)"local:local");
rasqal_query_prepare(rq, (unsigned char *)update, bu);
if (uctxt.error) {
if (uctxt.messages) {
*message = build_update_error_message(uctxt.messages);
g_slist_free(uctxt.messages);
}
return 1;
}
if (!quad_buffer) {
quad_buffer = calloc(uctxt.segments, sizeof(struct quad_buf));
}
int ok = 1;
for (int i=0; 1; i++) {
rasqal_update_operation *op = rasqal_query_get_update_operation(rq, i);
if (!op) {
break;
}
uctxt.op = op;
uctxt.opid = i;
if (update_op(&uctxt)) {
ok = 0;
break;
}
}
fsp_res_import_commit_all(qs->link);
fsp_quad_import_commit_all(qs->link, FS_BIND_BY_SUBJECT);
rasqal_free_query(rq);
if (uctxt.messages) {
*message = build_update_error_message(uctxt.messages);
g_slist_free(uctxt.messages);
}
for (GSList *it=uctxt.freeable; it; it=it->next) {
g_free(it->data);
}
g_slist_free(uctxt.freeable);
raptor_free_uri(bu);
if (ok) {
return 0;
} else {
return 1;
}
}
xmlChar *get_uri(fsp_link *link, fs_rid rid)
{
if (cache[rid & CACHE_MASK].rid == rid) {
return (xmlChar *) cache[rid & CACHE_MASK].lex;
}
fs_rid_vector onerid = { .length = 1, .size = 1, .data = &rid };
fs_resource resource;
fsp_resolve(link, FS_RID_SEGMENT(rid, segments), &onerid, &resource);
return (xmlChar *) resource.lex;
}
xmlChar *get_attr(fsp_link *link, fs_rid rid)
{
if (attr_cache[rid & CACHE_MASK].rid == rid) {
return (xmlChar *) attr_cache[rid & CACHE_MASK].lex;
}
fs_rid_vector onerid = { .length = 1, .size = 1, .data = &rid };
fs_resource resource;
fsp_resolve(link, FS_RID_SEGMENT(rid, segments), &onerid, &resource);
memcpy(&attr_cache[rid & ATTR_CACHE_MASK], &resource, sizeof(fs_resource));
return (xmlChar *) resource.lex;
}
xmlChar *get_literal(fsp_link *link, fs_rid rid, fs_rid *attr)
{
if (cache[rid & CACHE_MASK].rid == rid) {
*attr = cache[rid & CACHE_MASK].attr;
return (xmlChar *) cache[rid & CACHE_MASK].lex;
}
fs_rid_vector onerid = { .length = 1, .size = 1, .data = &rid };
fs_resource resource;
fsp_resolve(link, FS_RID_SEGMENT(rid, segments), &onerid, &resource);
*attr = resource.attr;
return (xmlChar *) resource.lex;
}
void resolve_triples(fsp_link *link, fs_rid_vector **rids)
{
int quads = rids[0]->length;
fs_rid_vector *todo[segments];
fs_segment segment;
for (segment = 0; segment < segments; ++segment) {
todo[segment] = fs_rid_vector_new(0);
}
for (int c = 0; c < 3; ++c) {
for (int k = 0; k < quads; ++k) {
const fs_rid rid = rids[c]->data[k];
if (FS_IS_BNODE(rid) || cache[rid & CACHE_MASK].rid == rid) continue;
fs_rid_vector_append(todo[FS_RID_SEGMENT(rid, segments)], rid);
cache[rid & CACHE_MASK].rid = rid; /* well, it will be soon */
}
}
int length[segments];
fs_resource *resources[segments];
for (segment = 0; segment < segments; ++segment) {
length[segment] = todo[segment]->length;
resources[segment] = calloc(length[segment], sizeof(fs_resource));
}
fsp_resolve_all(link, todo, resources);
for (segment = 0; segment < segments; ++segment) {
fs_resource *res = resources[segment];
for (int k = 0; k < length[segment]; ++k) {
free(cache[res[k].rid & CACHE_MASK].lex);
memcpy(&cache[res[k].rid & CACHE_MASK], &res[k], sizeof(fs_resource));
}
fs_rid_vector_free(todo[segment]);
free(resources[segment]);
}
}
void dump_model(fsp_link *link, fs_rid model, xmlTextWriterPtr xml)
{
fs_rid_vector none = { .length = 0, .size = 0, .data = 0 };
fs_rid_vector one = { .length = 1, .size = 1, .data = &model };
fs_rid_vector **results;
double then; /* for time keeping */
then = fs_time();
fsp_bind_first_all(link, BIND_SPO, &one, &none, &none, &none, &results, QUAD_LIMIT);
time_bind_first += (fs_time() - then);
while (results != NULL) {
long length = results[0]->length;
if (length == 0) break;
then = fs_time();
resolve_triples(link, results);
time_resolving += (fs_time() - then);
then = fs_time();
for (int k = 0; k < length; ++k) {
xmlTextWriterStartElement(xml, (xmlChar *) "triple");
for (int r = 0; r < 3; ++r) {
fs_rid rid = results[r]->data[k];
if (FS_IS_BNODE(rid)) {
unsigned long long node = FS_BNODE_NUM(rid);
xmlTextWriterWriteFormatElement(xml, (xmlChar *) "id", "%llu", node);
} else if (FS_IS_URI(rid)) {
xmlChar *uri = get_uri(link, rid);
xmlTextWriterWriteElement(xml, (xmlChar *) "uri", uri);
} else if (FS_IS_LITERAL(rid)) {
fs_rid attr;
xmlChar *lex = get_literal(link, rid, &attr);
if (attr == fs_c.empty) {
xmlTextWriterWriteElement(xml, (xmlChar *) "plainLiteral", lex);
} else if (FS_IS_URI(attr)) {
xmlChar *type = get_uri(link, attr);
xmlTextWriterStartElement(xml, (xmlChar *) "typedLiteral");
xmlTextWriterWriteString(xml, (xmlChar *) lex);
xmlTextWriterWriteAttribute(xml, (xmlChar *) "datatype", type);
xmlTextWriterEndElement(xml);
} else if (FS_IS_LITERAL(attr)) {
xmlChar *lang = get_attr(link, attr);
xmlTextWriterStartElement(xml, (xmlChar *) "plainLiteral");
xmlTextWriterWriteAttribute(xml, (xmlChar *) "xml:lang", lang);
xmlTextWriterWriteString(xml, (xmlChar *) lex);
xmlTextWriterEndElement(xml);
}
}
}
xmlTextWriterEndElement(xml);
xmlTextWriterWriteString(xml, (xmlChar *) "\n");
}
time_write_out += (fs_time() - then);
fs_rid_vector_free(results[0]);
fs_rid_vector_free(results[1]);
fs_rid_vector_free(results[2]);
free(results);
then = fs_time();
fsp_bind_next_all(link, BIND_SPO, &results, QUAD_LIMIT);
time_bind_next += (fs_time() - then);
}
fsp_bind_done_all(link);
}
void dump_trix(fsp_link *link, xmlTextWriterPtr xml)
{
fs_rid_vector **models;
fs_rid_vector none = { .length = 0, .size = 0, .data = 0 };
fsp_bind_all(link, FS_BIND_DISTINCT | FS_BIND_MODEL | FS_BIND_BY_SUBJECT, &none, &none, &none, &none, &models);
fs_rid_vector_sort(models[0]);
fs_rid_vector_uniq(models[0], 1);
long length = models[0]->length;
for (int k = 0; k < length; ++k) {
fs_rid model = models[0]->data[k];
xmlChar *model_uri = get_uri(link, model);
xmlTextWriterStartElement(xml, (xmlChar *) "graph");
if (FS_IS_URI(model)) {
xmlTextWriterWriteElement(xml, (xmlChar *) "uri", model_uri);
} else {
fs_error(LOG_WARNING, "model %lld is not a URI", model);
}
dump_model(link, model, xml);
xmlTextWriterEndElement(xml);
xmlTextWriterWriteString(xml, (xmlChar *) "\n");
printf("%5d/%ld: %4.5f %4.5f %4.5f %4.5f\n", k + 1, length, time_resolving, time_bind_first, time_bind_next, time_write_out);
}
}
void dump_file(fsp_link *link, char *filename)
{
xmlTextWriterPtr xml = xmlNewTextWriterFilename(filename, TRUE);
if (!xml) {
fs_error(LOG_ERR, "Couldn't write output file, giving up");
exit(4);
}
xmlTextWriterStartDocument(xml, NULL, NULL, NULL);
xmlTextWriterStartElement(xml, (xmlChar *) "TriX");
dump_trix(link, xml);
xmlTextWriterEndDocument(xml); /* also closes TriX */
xmlFreeTextWriter(xml);
}
int main(int argc, char *argv[])
{
char *password = fsp_argv_password(&argc, argv);
if (argc != 3) {
fprintf(stderr, "%s revision %s\n", argv[0], FS_FRONTEND_VER);
fprintf(stderr, "Usage: %s <kbname> <uri>\n", argv[0]);
exit(1);
}
fsp_link *link = fsp_open_link(argv[1], password, FS_OPEN_HINT_RO);
if (!link) {
fs_error (LOG_ERR, "couldn't connect to “%s”", argv[1]);
exit(2);
}
fs_hash_init(fsp_hash_type(link));
segments = fsp_link_segments(link);
dump_file(link, argv[2]);
fsp_close_link(link);
}
int fs_import(fsp_link *link, const char *model_uri, char *resource_uri,
const char *format, int verbosity, int dryrun, int has_o_index,
FILE *msg, int *count)
{
raptor_parser *rdf_parser = NULL;
raptor_uri ruri = NULL;
int ret = 0;
const int segments = fsp_link_segments(link);
parse_data.ext_count = count;
if (!inited) {
inited = 1;
parse_data.link = link;
parse_data.segments = fsp_link_segments(link);
for (int i=0; i<parse_data.segments; i++) {
for (int j=0; j<RES_BUF_SIZE; j++) {
lex_tmp[i][j] = malloc(RES_BUF_SIZE);
}
}
memset(nodecache, 0, sizeof(nodecache));
parse_data.quad_fn = g_strdup(FS_TMP_PATH "/importXXXXXX");
parse_data.quad_fd = mkstemp(parse_data.quad_fn);
if (parse_data.quad_fd < 0) {
fs_error(LOG_ERR, "Cannot create tmp file “%s”", parse_data.quad_fn);
return 1;
}
gettimeofday(&then_last, 0);
}
parse_data.verbosity = verbosity;
parse_data.model = g_strdup(model_uri);
parse_data.model_hash = fs_hash_uri(model_uri);
parse_data.count_trip = 0;
parse_data.last_count = 0;
parse_data.dryrun = dryrun;
parse_data.has_o_index = has_o_index;
/* store the model uri */
buffer_res(link, segments, parse_data.model_hash, parse_data.model, FS_RID_NULL, dryrun);
if (strcmp(format, "auto")) {
rdf_parser = raptor_new_parser(format);
} else if (strstr(resource_uri, ".n3") || strstr(resource_uri, ".ttl")) {
rdf_parser = raptor_new_parser("turtle");
} else if (strstr(resource_uri, ".nt")) {
rdf_parser = raptor_new_parser("ntriples");
} else {
rdf_parser = raptor_new_parser("rdfxml");
}
if (!rdf_parser) {
fs_error(LOG_ERR, "failed to create RDF parser");
return 1;
}
raptor_set_statement_handler(rdf_parser, &parse_data, store_stmt);
raptor_set_graph_handler(rdf_parser, &parse_data, graph_handler);
ruri = raptor_new_uri((unsigned char *) resource_uri);
parse_data.muri = raptor_new_uri((unsigned char *) model_uri);
if (raptor_parse_uri(rdf_parser, ruri, parse_data.muri)) {
fs_error(LOG_ERR, "failed to parse file “%s”", resource_uri);
ret++;
}
if (verbosity) {
printf("Pass 1, processed %d triples (%d)\n", total_triples_parsed, parse_data.count_trip);
}
raptor_free_parser(rdf_parser);
raptor_free_uri(ruri);
raptor_free_uri(parse_data.muri);
g_free(parse_data.model);
fs_hash_freshen(); /* blank nodes are unique per file */
return ret;
}
static int update_op(struct update_context *uc)
{
fs_rid_vector *vec[4];
switch (uc->op->type) {
case RASQAL_UPDATE_TYPE_UNKNOWN:
add_message(uc, "Unknown update operation", 0);
return 1;
case RASQAL_UPDATE_TYPE_CLEAR:
fs_clear(uc, graph_arg(uc->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_CREATE:
return 0;
case RASQAL_UPDATE_TYPE_DROP:
fs_clear(uc, graph_arg(uc->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_LOAD:
fs_load(uc, graph_arg(uc->op->document_uri),
graph_arg(uc->op->graph_uri));
return 0;
#if RASQAL_VERSION >= 924
case RASQAL_UPDATE_TYPE_ADD:
fs_add(uc, graph_arg(uc->op->graph_uri),
graph_arg(uc->op->document_uri));
return 0;
case RASQAL_UPDATE_TYPE_MOVE:
fs_move(uc, graph_arg(uc->op->graph_uri),
graph_arg(uc->op->document_uri));
return 0;
case RASQAL_UPDATE_TYPE_COPY:
fs_copy(uc, graph_arg(uc->op->graph_uri),
graph_arg(uc->op->document_uri));
return 0;
#endif
case RASQAL_UPDATE_TYPE_UPDATE:
break;
}
fs_hash_freshen();
raptor_sequence *todel = NULL;
raptor_sequence *toins = NULL;
if (uc->op->delete_templates && !uc->op->where) {
int where = 0;
/* check to see if it's a DELETE WHERE { } */
for (int t=0; t<raptor_sequence_size(uc->op->delete_templates); t++) {
rasqal_triple *tr = raptor_sequence_get_at(uc->op->delete_templates, t);
if (any_vars(tr)) {
where = 1;
break;
}
}
if (where) {
fs_error(LOG_ERR, "DELETE WHERE { x } not yet supported");
add_message(uc, "DELETE WHERE { x } not yet supported, use DELETE { x } WHERE { x }", 0);
return 1;
}
}
#if RASQAL_VERSION >= 923
if (uc->op->where) {
todel = raptor_new_sequence(NULL, NULL);
toins = raptor_new_sequence(NULL, NULL);
raptor_sequence *todel_p = raptor_new_sequence(NULL, NULL);
raptor_sequence *toins_p = raptor_new_sequence(NULL, NULL);
raptor_sequence *vars = raptor_new_sequence(NULL, NULL);
fs_query *q = calloc(1, sizeof(fs_query));
uc->q = q;
q->qs = uc->qs;
q->rq = uc->rq;
q->flags = FS_BIND_DISTINCT;
#ifdef DEBUG_MERGE
q->flags |= FS_QUERY_CONSOLE_OUTPUT;
#endif
q->boolean = 1;
q->opt_level = 3;
q->soft_limit = -1;
q->segments = fsp_link_segments(uc->link);
q->link = uc->link;
q->bb[0] = fs_binding_new();
q->bt = q->bb[0];
/* hashtable to hold runtime created resources */
q->tmp_resources = g_hash_table_new_full(fs_rid_hash, fs_rid_equal, g_free, fs_free_cached_resource);
/* add column to denote join ordering */
fs_binding_create(q->bb[0], "_ord", FS_RID_NULL, 0);
if (uc->op->delete_templates) {
for (int t=0; t<raptor_sequence_size(uc->op->delete_templates); t++) {
rasqal_triple *tr = raptor_sequence_get_at(uc->op->delete_templates, t);
if (any_vars(tr)) {
fs_check_cons_slot(q, vars, tr->subject);
fs_check_cons_slot(q, vars, tr->predicate);
fs_check_cons_slot(q, vars, tr->object);
raptor_sequence_push(todel_p, tr);
} else {
raptor_sequence_push(todel, tr);
}
}
}
if (uc->op->insert_templates) {
for (int t=0; t<raptor_sequence_size(uc->op->insert_templates); t++) {
rasqal_triple *tr = raptor_sequence_get_at(uc->op->insert_templates, t);
if (any_vars(tr)) {
fs_check_cons_slot(q, vars, tr->subject);
fs_check_cons_slot(q, vars, tr->predicate);
fs_check_cons_slot(q, vars, tr->object);
raptor_sequence_push(toins_p, tr);
} else {
raptor_sequence_push(toins, tr);
}
}
}
q->num_vars = raptor_sequence_size(vars);
for (int i=0; i < q->num_vars; i++) {
rasqal_variable *v = raptor_sequence_get_at(vars, i);
fs_binding_add(q->bb[0], v, FS_RID_NULL, 1);
}
/* perform the WHERE match */
fs_query_process_pattern(q, uc->op->where, vars);
q->length = fs_binding_length(q->bb[0]);
for (int s=0; s<4; s++) {
vec[s] = fs_rid_vector_new(0);
}
for (int t=0; t<raptor_sequence_size(todel_p); t++) {
rasqal_triple *triple = raptor_sequence_get_at(todel_p, t);
for (int row=0; row < q->length; row++) {
delete_rasqal_triple(uc, vec, triple, row);
}
if (fs_rid_vector_length(vec[0]) > 1000) {
fsp_delete_quads_all(uc->link, vec);
}
}
if (fs_rid_vector_length(vec[0]) > 0) {
fsp_delete_quads_all(uc->link, vec);
}
for (int s=0; s<4; s++) {
//fs_rid_vector_print(vec[s], 0, stdout);
fs_rid_vector_free(vec[s]);
vec[s] = NULL;
}
for (int t=0; t<raptor_sequence_size(toins_p); t++) {
rasqal_triple *triple = raptor_sequence_get_at(toins_p, t);
for (int row=0; row < q->length; row++) {
insert_rasqal_triple(uc, triple, row);
}
}
/* must not free the rasqal_query */
q->rq = NULL;
fs_query_free(q);
uc->q = NULL;
} else {
todel = uc->op->delete_templates;
toins = uc->op->insert_templates;
}
#else
if (uc->op->where) {
fs_error(LOG_ERR, "DELETE/INSERT WHERE requires Rasqal 0.9.23 or newer");
add_message(uc, "DELETE/INSERT WHERE requires Rasqal 0.9.23 or newer", 0);
}
#endif
/* delete constant triples */
if (todel) {
for (int s=0; s<4; s++) {
vec[s] = fs_rid_vector_new(0);
}
for (int t=0; t<raptor_sequence_size(todel); t++) {
rasqal_triple *triple = raptor_sequence_get_at(todel, t);
if (any_vars(triple)) {
continue;
}
delete_rasqal_triple(uc, vec, triple, 0);
}
if (fs_rid_vector_length(vec[0]) > 0) {
fsp_delete_quads_all(uc->link, vec);
}
for (int s=0; s<4; s++) {
fs_rid_vector_free(vec[s]);
vec[s] = NULL;
}
}
/* insert constant triples */
if (toins) {
for (int t=0; t<raptor_sequence_size(toins); t++) {
rasqal_triple *triple = raptor_sequence_get_at(toins, t);
if (any_vars(triple)) {
continue;
}
insert_rasqal_triple(uc, triple, 0);
}
}
fs_hash_freshen();
return 0;
}
int main(int argc, char *argv[])
{
int verbosity = 0;
int dryrun = 0;
char *password = NULL;
char *format = "auto";
FILE *msg = stderr;
char *optstring = "am:M:vnf:";
int c, opt_index = 0, help = 0;
int files = 0, adding = 0;
char *kb_name = NULL;
char *model[argc], *uri[argc];
char *model_default = NULL;
password = fsp_argv_password(&argc, argv);
static struct option long_options[] = {
{ "add", 0, 0, 'a' },
{ "model", 1, 0, 'm' },
{ "model-default", 1, 0, 'M' },
{ "verbose", 0, 0, 'v' },
{ "dryrun", 0, 0, 'n' },
{ "no-resources", 0, 0, 'R' },
{ "no-quads", 0, 0, 'Q' },
{ "format", 1, 0, 'f' },
{ "help", 0, 0, 'h' },
{ "version", 0, 0, 'V' },
{ 0, 0, 0, 0 }
};
for (int i= 0; i < argc; ++i) {
model[i] = NULL;
}
int help_return = 1;
while ((c = getopt_long (argc, argv, optstring, long_options, &opt_index)) != -1) {
if (c == 'm') {
model[files++] = optarg;
} else if (c == 'M') {
model_default = optarg;
} else if (c == 'v') {
verbosity++;
} else if (c == 'a') {
adding = 1;
} else if (c == 'n') {
dryrun |= FS_DRYRUN_DELETE | FS_DRYRUN_RESOURCES | FS_DRYRUN_QUADS;
} else if (c == 'R') {
dryrun |= FS_DRYRUN_RESOURCES;
} else if (c == 'Q') {
dryrun |= FS_DRYRUN_QUADS;
} else if (c == 'f') {
format = optarg;
} else if (c == 'h') {
help = 1;
help_return = 0;
} else if (c == 'V') {
printf("%s, built for 4store %s\n", argv[0], GIT_REV);
exit(0);
} else {
help = 1;
}
}
if (verbosity > 0) {
if (dryrun & FS_DRYRUN_DELETE) {
printf("warning: not deleting old model\n");
}
if (dryrun & FS_DRYRUN_RESOURCES) {
printf("warning: not importing resource nodes\n");
}
if (dryrun & FS_DRYRUN_QUADS) {
printf("warning: not importing quad graph\n");
}
}
files = 0;
for (int k = optind; k < argc; ++k) {
if (!kb_name) {
kb_name = argv[k];
} else {
if (strchr(argv[k], ':')) {
uri[files] = g_strdup(argv[k]);
} else {
uri[files] = (char *)raptor_uri_filename_to_uri_string(argv[k]);
}
if (!model[files]) {
if (!model_default) {
model[files] = uri[files];
} else {
model[files] = model_default;
}
}
files++;
}
}
raptor_world *rw = raptor_new_world();
if (help || !kb_name || files == 0) {
fprintf(stdout, "%s revision %s\n", argv[0], FS_FRONTEND_VER);
fprintf(stdout, "Usage: %s <kbname> <rdf file/URI> ...\n", argv[0]);
fprintf(stdout, " -v --verbose increase verbosity (can repeat)\n");
fprintf(stdout, " -a --add add data to models instead of replacing\n");
fprintf(stdout, " -m --model specify a model URI for the next RDF file\n");
fprintf(stdout, " -M --model-default specify a model URI for all RDF files\n");
fprintf(stdout, " -f --format specify an RDF syntax for the import\n");
fprintf(stdout, "\n available formats are:\n");
for (unsigned int i=0; 1; i++) {
const raptor_syntax_description *desc =
raptor_world_get_parser_description(rw, i);
if (!desc) {
break;
}
fprintf(stdout, " %12s - %s\n", desc->names[0], desc->label);
}
exit(help_return);
}
fsp_syslog_enable();
fsplink = fsp_open_link(kb_name, password, FS_OPEN_HINT_RW);
if (!fsplink) {
fs_error (LOG_ERR, "couldn't connect to “%s”", kb_name);
exit(2);
}
const char *features = fsp_link_features(fsplink);
int has_o_index = !(strstr(features, "no-o-index")); /* tweak */
fs_hash_init(fsp_hash_type(fsplink));
const int segments = fsp_link_segments(fsplink);
int total_triples = 0;
fs_import_timing timing[segments];
for (int seg = 0; seg < segments; seg++) {
fsp_get_import_times(fsplink, seg, &timing[seg]);
}
gettimeofday(&then, 0);
if (fsp_start_import_all(fsplink)) {
fs_error(LOG_ERR, "aborting import");
exit(3);
}
#if 0
printf("press enter\n");
char foo;
read(0, &foo, 1);
#endif
fs_rid_vector *mvec = fs_rid_vector_new(0);
for (int f= 0; f < files; ++f) {
fs_rid muri = fs_hash_uri(model[f]);
fs_rid_vector_append(mvec, muri);
}
if (!adding) {
if (verbosity) {
printf("removing old data\n");
fflush(stdout);
}
if (!(dryrun & FS_DRYRUN_DELETE)) {
if (fsp_delete_model_all(fsplink, mvec)) {
fs_error(LOG_ERR, "model delete failed");
return 1;
}
for (int i=0; i<mvec->length; i++) {
if (mvec->data[i] == fs_c.system_config) {
fs_import_reread_config();
}
}
}
fsp_new_model_all(fsplink, mvec);
}
fs_rid_vector_free(mvec);
gettimeofday(&then_last, 0);
for (int f = 0; f < files; ++f) {
if (verbosity) {
printf("Reading <%s>\n", uri[f]);
if (strcmp(uri[f], model[f])) {
printf(" into <%s>\n", model[f]);
}
fflush(stdout);
}
fs_import(fsplink, model[f], uri[f], format, verbosity, dryrun, has_o_index, msg, &total_triples);
if (verbosity) {
fflush(stdout);
}
}
double sthen = fs_time();
int ret = fs_import_commit(fsplink, verbosity, dryrun, has_o_index, msg, &total_triples);
if (verbosity > 0) {
printf("Updating index\n");
fflush(stdout);
}
fsp_stop_import_all(fsplink);
if (verbosity > 0) {
printf("Index update took %f seconds\n", fs_time()-sthen);
}
if (!ret) {
gettimeofday(&now, 0);
double diff = (now.tv_sec - then.tv_sec) +
(now.tv_usec - then.tv_usec) * 0.000001;
if (verbosity && total_triples > 0) {
printf("Imported %d triples, average %d triples/s\n", total_triples,
(int)((double)total_triples/diff));
fflush(stdout);
}
}
if (verbosity > 1) {
printf("seg add_q\tadd_r\t\tcommit_q\tcommit_r\tremove\t\trebuild\t\twrite\n");
long long *tics = fsp_profile_write(fsplink);
for (int seg = 0; seg < segments; seg++) {
fs_import_timing newtimes;
fsp_get_import_times(fsplink, seg, &newtimes);
printf("%2d: %f\t%f\t%f\t%f\t%f\t%f\t%f\n", seg,
newtimes.add_s - timing[seg].add_s,
newtimes.add_r - timing[seg].add_r,
newtimes.commit_q - timing[seg].commit_q,
newtimes.commit_r - timing[seg].commit_r,
newtimes.remove - timing[seg].remove,
newtimes.rebuild - timing[seg].rebuild,
tics[seg] * 0.001);
}
}
fsp_close_link(fsplink);
raptor_free_world(rw);
return 0;
}
static int update_op(struct update_context *ct)
{
fs_rid_vector *vec[4];
switch (ct->op->type) {
case RASQAL_UPDATE_TYPE_UNKNOWN:
add_message(ct, "Unknown update operation", 0);
return 1;
case RASQAL_UPDATE_TYPE_CLEAR:
fs_clear(ct, (char *)raptor_uri_as_string(ct->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_CREATE:
return 0;
case RASQAL_UPDATE_TYPE_DROP:
fs_clear(ct, (char *)raptor_uri_as_string(ct->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_LOAD:
fs_load(ct, (char *)raptor_uri_as_string(ct->op->document_uri),
(char *)raptor_uri_as_string(ct->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_UPDATE:
break;
}
fs_hash_freshen();
raptor_sequence *todel = NULL;
raptor_sequence *toins = NULL;
if (ct->op->where) {
todel = raptor_new_sequence(NULL, NULL);
toins = raptor_new_sequence(NULL, NULL);
raptor_sequence *todel_p = raptor_new_sequence(NULL, NULL);
raptor_sequence *toins_p = raptor_new_sequence(NULL, NULL);
raptor_sequence *vars = raptor_new_sequence(NULL, NULL);
fs_query *q = calloc(1, sizeof(fs_query));
ct->q = q;
q->qs = ct->qs;
q->rq = ct->rq;
q->flags = FS_BIND_DISTINCT;
q->opt_level = 3;
q->soft_limit = -1;
q->segments = fsp_link_segments(ct->link);
q->link = ct->link;
q->bb[0] = fs_binding_new();
q->bt = q->bb[0];
/* add column to denote join ordering */
fs_binding_add(q->bb[0], "_ord", FS_RID_NULL, 0);
struct pattern_data pd = { .q = q, .vars = vars, .patterns = NULL, .fixed = NULL };
if (ct->op->delete_templates) {
pd.patterns = todel_p;
pd.fixed = todel;
for (int t=0; t<raptor_sequence_size(ct->op->delete_templates); t++) {
rasqal_graph_pattern *gp = raptor_sequence_get_at(ct->op->delete_templates, t);
assign_gp(gp, NULL, &pd);
}
}
if (ct->op->insert_templates) {
pd.patterns = toins_p;
pd.fixed = toins;
for (int t=0; t<raptor_sequence_size(ct->op->insert_templates); t++) {
rasqal_graph_pattern *gp = raptor_sequence_get_at(ct->op->insert_templates, t);
assign_gp(gp, NULL, &pd);
}
}
q->num_vars = raptor_sequence_size(vars);
for (int i=0; i < q->num_vars; i++) {
rasqal_variable *v = raptor_sequence_get_at(vars, i);
fs_binding *b = fs_binding_get(q->bb[0], (char *)v->name);
if (b) {
b->need_val = 1;
} else {
fs_binding_add(q->bb[0], (char *)v->name, FS_RID_NULL, 1);
}
}
fs_query_process_pattern(q, ct->op->where, vars);
q->length = fs_binding_length(q->bb[0]);
for (int s=0; s<4; s++) {
vec[s] = fs_rid_vector_new(0);
}
for (int t=0; t<raptor_sequence_size(todel_p); t++) {
rasqal_triple *triple = raptor_sequence_get_at(todel_p, t);
for (int row=0; row < q->length; row++) {
delete_rasqal_triple(ct, vec, triple, row);
if (fs_rid_vector_length(vec[0]) > 0) {
fsp_delete_quads_all(ct->link, vec);
}
}
}
for (int s=0; s<4; s++) {
//fs_rid_vector_print(vec[s], 0, stdout);
fs_rid_vector_free(vec[s]);
}
for (int t=0; t<raptor_sequence_size(toins_p); t++) {
rasqal_triple *triple = raptor_sequence_get_at(toins_p, t);
for (int row=0; row < q->length; row++) {
insert_rasqal_triple(ct, triple, row);
}
}
/* must not free the rasqal_query */
q->rq = NULL;
fs_query_free(q);
ct->q = NULL;
} else {