/* ..._start and ..._finish share an int * count parameter
* the same variable should be passed by reference both times */
int fs_import_stream_start(fsp_link *link, const char *model_uri, const char *mimetype, int has_o_index, int *count)
{
if (inited == 0) {
memset(&parse_data, 0, sizeof(parse_data));
inited = 1;
}
parse_data.link = link;
parse_data.segments = fsp_link_segments(link);
parse_data.ext_count = count;
for (int i=0; i<parse_data.segments; i++) {
for (int j=0; j<RES_BUF_SIZE; j++) {
lex_tmp[i][j] = malloc(RES_BUF_SIZE);
}
}
memset(nodecache, 0, sizeof(nodecache));
parse_data.quad_fn = g_strdup(FS_TMP_PATH "/importXXXXXX");
parse_data.quad_fd = mkstemp(parse_data.quad_fn);
if (parse_data.quad_fd < 0) {
fs_error(LOG_ERR, "Cannot create tmp file “%s”", parse_data.quad_fn);
return 1;
}
parse_data.muri = raptor_new_uri((unsigned char *) model_uri);
parse_data.model = g_strdup(model_uri);
parse_data.model_hash = fs_hash_uri(model_uri);
parse_data.count_trip = 0;
parse_data.count_err = 0;
parse_data.last_count = 0;
parse_data.has_o_index = has_o_index;
/* store the model uri */
buffer_res(link, parse_data.segments, parse_data.model_hash, parse_data.model, FS_RID_NULL, parse_data.dryrun);
parse_data.parser = raptor_new_parser_for_content(NULL, mimetype, NULL, 0, (unsigned char *) parse_data.model);
if (!parse_data.parser) {
return 1;
}
/* use us as a vector for an indirect attack? no thanks */
raptor_set_feature(parse_data.parser, RAPTOR_FEATURE_NO_NET, 0);
raptor_set_fatal_error_handler(parse_data.parser, link, fatal_rdf_parser_error);
raptor_set_error_handler(parse_data.parser, link, rdf_parser_error);
raptor_set_statement_handler(parse_data.parser, &parse_data, store_stmt);
raptor_set_graph_handler(parse_data.parser, &parse_data, graph_handler);
raptor_start_parse(parse_data.parser, parse_data.muri);
fs_hash_freshen(); /* blank nodes are unique per file */
return 0;
}
static int update_op(struct update_context *uc)
{
fs_rid_vector *vec[4];
switch (uc->op->type) {
case RASQAL_UPDATE_TYPE_UNKNOWN:
add_message(uc, "Unknown update operation", 0);
return 1;
case RASQAL_UPDATE_TYPE_CLEAR:
fs_clear(uc, graph_arg(uc->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_CREATE:
return 0;
case RASQAL_UPDATE_TYPE_DROP:
fs_clear(uc, graph_arg(uc->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_LOAD:
fs_load(uc, graph_arg(uc->op->document_uri),
graph_arg(uc->op->graph_uri));
return 0;
#if RASQAL_VERSION >= 924
case RASQAL_UPDATE_TYPE_ADD:
fs_add(uc, graph_arg(uc->op->graph_uri),
graph_arg(uc->op->document_uri));
return 0;
case RASQAL_UPDATE_TYPE_MOVE:
fs_move(uc, graph_arg(uc->op->graph_uri),
graph_arg(uc->op->document_uri));
return 0;
case RASQAL_UPDATE_TYPE_COPY:
fs_copy(uc, graph_arg(uc->op->graph_uri),
graph_arg(uc->op->document_uri));
return 0;
#endif
case RASQAL_UPDATE_TYPE_UPDATE:
break;
}
fs_hash_freshen();
raptor_sequence *todel = NULL;
raptor_sequence *toins = NULL;
if (uc->op->delete_templates && !uc->op->where) {
int where = 0;
/* check to see if it's a DELETE WHERE { } */
for (int t=0; t<raptor_sequence_size(uc->op->delete_templates); t++) {
rasqal_triple *tr = raptor_sequence_get_at(uc->op->delete_templates, t);
if (any_vars(tr)) {
where = 1;
break;
}
}
if (where) {
fs_error(LOG_ERR, "DELETE WHERE { x } not yet supported");
add_message(uc, "DELETE WHERE { x } not yet supported, use DELETE { x } WHERE { x }", 0);
return 1;
}
}
#if RASQAL_VERSION >= 923
if (uc->op->where) {
todel = raptor_new_sequence(NULL, NULL);
toins = raptor_new_sequence(NULL, NULL);
raptor_sequence *todel_p = raptor_new_sequence(NULL, NULL);
raptor_sequence *toins_p = raptor_new_sequence(NULL, NULL);
raptor_sequence *vars = raptor_new_sequence(NULL, NULL);
fs_query *q = calloc(1, sizeof(fs_query));
uc->q = q;
q->qs = uc->qs;
q->rq = uc->rq;
q->flags = FS_BIND_DISTINCT;
#ifdef DEBUG_MERGE
q->flags |= FS_QUERY_CONSOLE_OUTPUT;
#endif
q->boolean = 1;
q->opt_level = 3;
q->soft_limit = -1;
q->segments = fsp_link_segments(uc->link);
q->link = uc->link;
q->bb[0] = fs_binding_new();
q->bt = q->bb[0];
/* hashtable to hold runtime created resources */
q->tmp_resources = g_hash_table_new_full(fs_rid_hash, fs_rid_equal, g_free, fs_free_cached_resource);
/* add column to denote join ordering */
fs_binding_create(q->bb[0], "_ord", FS_RID_NULL, 0);
if (uc->op->delete_templates) {
for (int t=0; t<raptor_sequence_size(uc->op->delete_templates); t++) {
rasqal_triple *tr = raptor_sequence_get_at(uc->op->delete_templates, t);
if (any_vars(tr)) {
fs_check_cons_slot(q, vars, tr->subject);
fs_check_cons_slot(q, vars, tr->predicate);
fs_check_cons_slot(q, vars, tr->object);
raptor_sequence_push(todel_p, tr);
} else {
raptor_sequence_push(todel, tr);
}
}
}
if (uc->op->insert_templates) {
for (int t=0; t<raptor_sequence_size(uc->op->insert_templates); t++) {
rasqal_triple *tr = raptor_sequence_get_at(uc->op->insert_templates, t);
if (any_vars(tr)) {
fs_check_cons_slot(q, vars, tr->subject);
fs_check_cons_slot(q, vars, tr->predicate);
fs_check_cons_slot(q, vars, tr->object);
raptor_sequence_push(toins_p, tr);
} else {
raptor_sequence_push(toins, tr);
}
}
}
q->num_vars = raptor_sequence_size(vars);
for (int i=0; i < q->num_vars; i++) {
rasqal_variable *v = raptor_sequence_get_at(vars, i);
fs_binding_add(q->bb[0], v, FS_RID_NULL, 1);
}
/* perform the WHERE match */
fs_query_process_pattern(q, uc->op->where, vars);
q->length = fs_binding_length(q->bb[0]);
for (int s=0; s<4; s++) {
vec[s] = fs_rid_vector_new(0);
}
for (int t=0; t<raptor_sequence_size(todel_p); t++) {
rasqal_triple *triple = raptor_sequence_get_at(todel_p, t);
for (int row=0; row < q->length; row++) {
delete_rasqal_triple(uc, vec, triple, row);
}
if (fs_rid_vector_length(vec[0]) > 1000) {
fsp_delete_quads_all(uc->link, vec);
}
}
if (fs_rid_vector_length(vec[0]) > 0) {
fsp_delete_quads_all(uc->link, vec);
}
for (int s=0; s<4; s++) {
//fs_rid_vector_print(vec[s], 0, stdout);
fs_rid_vector_free(vec[s]);
vec[s] = NULL;
}
for (int t=0; t<raptor_sequence_size(toins_p); t++) {
rasqal_triple *triple = raptor_sequence_get_at(toins_p, t);
for (int row=0; row < q->length; row++) {
insert_rasqal_triple(uc, triple, row);
}
}
/* must not free the rasqal_query */
q->rq = NULL;
fs_query_free(q);
uc->q = NULL;
} else {
todel = uc->op->delete_templates;
toins = uc->op->insert_templates;
}
#else
if (uc->op->where) {
fs_error(LOG_ERR, "DELETE/INSERT WHERE requires Rasqal 0.9.23 or newer");
add_message(uc, "DELETE/INSERT WHERE requires Rasqal 0.9.23 or newer", 0);
}
#endif
/* delete constant triples */
if (todel) {
for (int s=0; s<4; s++) {
vec[s] = fs_rid_vector_new(0);
}
for (int t=0; t<raptor_sequence_size(todel); t++) {
rasqal_triple *triple = raptor_sequence_get_at(todel, t);
if (any_vars(triple)) {
continue;
}
delete_rasqal_triple(uc, vec, triple, 0);
}
if (fs_rid_vector_length(vec[0]) > 0) {
fsp_delete_quads_all(uc->link, vec);
}
for (int s=0; s<4; s++) {
fs_rid_vector_free(vec[s]);
vec[s] = NULL;
}
}
/* insert constant triples */
if (toins) {
for (int t=0; t<raptor_sequence_size(toins); t++) {
rasqal_triple *triple = raptor_sequence_get_at(toins, t);
if (any_vars(triple)) {
continue;
}
insert_rasqal_triple(uc, triple, 0);
}
}
fs_hash_freshen();
return 0;
}
int fs_import(fsp_link *link, const char *model_uri, char *resource_uri,
const char *format, int verbosity, int dryrun, int has_o_index,
FILE *msg, int *count)
{
raptor_parser *rdf_parser = NULL;
raptor_uri ruri = NULL;
int ret = 0;
const int segments = fsp_link_segments(link);
parse_data.ext_count = count;
if (!inited) {
inited = 1;
parse_data.link = link;
parse_data.segments = fsp_link_segments(link);
for (int i=0; i<parse_data.segments; i++) {
for (int j=0; j<RES_BUF_SIZE; j++) {
lex_tmp[i][j] = malloc(RES_BUF_SIZE);
}
}
memset(nodecache, 0, sizeof(nodecache));
parse_data.quad_fn = g_strdup(FS_TMP_PATH "/importXXXXXX");
parse_data.quad_fd = mkstemp(parse_data.quad_fn);
if (parse_data.quad_fd < 0) {
fs_error(LOG_ERR, "Cannot create tmp file “%s”", parse_data.quad_fn);
return 1;
}
gettimeofday(&then_last, 0);
}
parse_data.verbosity = verbosity;
parse_data.model = g_strdup(model_uri);
parse_data.model_hash = fs_hash_uri(model_uri);
parse_data.count_trip = 0;
parse_data.last_count = 0;
parse_data.dryrun = dryrun;
parse_data.has_o_index = has_o_index;
/* store the model uri */
buffer_res(link, segments, parse_data.model_hash, parse_data.model, FS_RID_NULL, dryrun);
if (strcmp(format, "auto")) {
rdf_parser = raptor_new_parser(format);
} else if (strstr(resource_uri, ".n3") || strstr(resource_uri, ".ttl")) {
rdf_parser = raptor_new_parser("turtle");
} else if (strstr(resource_uri, ".nt")) {
rdf_parser = raptor_new_parser("ntriples");
} else {
rdf_parser = raptor_new_parser("rdfxml");
}
if (!rdf_parser) {
fs_error(LOG_ERR, "failed to create RDF parser");
return 1;
}
raptor_set_statement_handler(rdf_parser, &parse_data, store_stmt);
raptor_set_graph_handler(rdf_parser, &parse_data, graph_handler);
ruri = raptor_new_uri((unsigned char *) resource_uri);
parse_data.muri = raptor_new_uri((unsigned char *) model_uri);
if (raptor_parse_uri(rdf_parser, ruri, parse_data.muri)) {
fs_error(LOG_ERR, "failed to parse file “%s”", resource_uri);
ret++;
}
if (verbosity) {
printf("Pass 1, processed %d triples (%d)\n", total_triples_parsed, parse_data.count_trip);
}
raptor_free_parser(rdf_parser);
raptor_free_uri(ruri);
raptor_free_uri(parse_data.muri);
g_free(parse_data.model);
fs_hash_freshen(); /* blank nodes are unique per file */
return ret;
}
static int update_op(struct update_context *ct)
{
fs_rid_vector *vec[4];
switch (ct->op->type) {
case RASQAL_UPDATE_TYPE_UNKNOWN:
add_message(ct, "Unknown update operation", 0);
return 1;
case RASQAL_UPDATE_TYPE_CLEAR:
fs_clear(ct, (char *)raptor_uri_as_string(ct->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_CREATE:
return 0;
case RASQAL_UPDATE_TYPE_DROP:
fs_clear(ct, (char *)raptor_uri_as_string(ct->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_LOAD:
fs_load(ct, (char *)raptor_uri_as_string(ct->op->document_uri),
(char *)raptor_uri_as_string(ct->op->graph_uri));
return 0;
case RASQAL_UPDATE_TYPE_UPDATE:
break;
}
fs_hash_freshen();
raptor_sequence *todel = NULL;
raptor_sequence *toins = NULL;
if (ct->op->where) {
todel = raptor_new_sequence(NULL, NULL);
toins = raptor_new_sequence(NULL, NULL);
raptor_sequence *todel_p = raptor_new_sequence(NULL, NULL);
raptor_sequence *toins_p = raptor_new_sequence(NULL, NULL);
raptor_sequence *vars = raptor_new_sequence(NULL, NULL);
fs_query *q = calloc(1, sizeof(fs_query));
ct->q = q;
q->qs = ct->qs;
q->rq = ct->rq;
q->flags = FS_BIND_DISTINCT;
q->opt_level = 3;
q->soft_limit = -1;
q->segments = fsp_link_segments(ct->link);
q->link = ct->link;
q->bb[0] = fs_binding_new();
q->bt = q->bb[0];
/* add column to denote join ordering */
fs_binding_add(q->bb[0], "_ord", FS_RID_NULL, 0);
struct pattern_data pd = { .q = q, .vars = vars, .patterns = NULL, .fixed = NULL };
if (ct->op->delete_templates) {
pd.patterns = todel_p;
pd.fixed = todel;
for (int t=0; t<raptor_sequence_size(ct->op->delete_templates); t++) {
rasqal_graph_pattern *gp = raptor_sequence_get_at(ct->op->delete_templates, t);
assign_gp(gp, NULL, &pd);
}
}
if (ct->op->insert_templates) {
pd.patterns = toins_p;
pd.fixed = toins;
for (int t=0; t<raptor_sequence_size(ct->op->insert_templates); t++) {
rasqal_graph_pattern *gp = raptor_sequence_get_at(ct->op->insert_templates, t);
assign_gp(gp, NULL, &pd);
}
}
q->num_vars = raptor_sequence_size(vars);
for (int i=0; i < q->num_vars; i++) {
rasqal_variable *v = raptor_sequence_get_at(vars, i);
fs_binding *b = fs_binding_get(q->bb[0], (char *)v->name);
if (b) {
b->need_val = 1;
} else {
fs_binding_add(q->bb[0], (char *)v->name, FS_RID_NULL, 1);
}
}
fs_query_process_pattern(q, ct->op->where, vars);
q->length = fs_binding_length(q->bb[0]);
for (int s=0; s<4; s++) {
vec[s] = fs_rid_vector_new(0);
}
for (int t=0; t<raptor_sequence_size(todel_p); t++) {
rasqal_triple *triple = raptor_sequence_get_at(todel_p, t);
for (int row=0; row < q->length; row++) {
delete_rasqal_triple(ct, vec, triple, row);
if (fs_rid_vector_length(vec[0]) > 0) {
fsp_delete_quads_all(ct->link, vec);
}
}
}
for (int s=0; s<4; s++) {
//fs_rid_vector_print(vec[s], 0, stdout);
fs_rid_vector_free(vec[s]);
}
for (int t=0; t<raptor_sequence_size(toins_p); t++) {
rasqal_triple *triple = raptor_sequence_get_at(toins_p, t);
for (int row=0; row < q->length; row++) {
insert_rasqal_triple(ct, triple, row);
}
}
/* must not free the rasqal_query */
q->rq = NULL;
fs_query_free(q);
ct->q = NULL;
} else {
