/* ..._start and ..._finish share an int * count parameter * the same variable should be passed by reference both times */ int fs_import_stream_start(fsp_link *link, const char *model_uri, const char *mimetype, int has_o_index, int *count) { if (inited == 0) { memset(&parse_data, 0, sizeof(parse_data)); inited = 1; } parse_data.link = link; parse_data.segments = fsp_link_segments(link); parse_data.ext_count = count; for (int i=0; i<parse_data.segments; i++) { for (int j=0; j<RES_BUF_SIZE; j++) { lex_tmp[i][j] = malloc(RES_BUF_SIZE); } } memset(nodecache, 0, sizeof(nodecache)); parse_data.quad_fn = g_strdup(FS_TMP_PATH "/importXXXXXX"); parse_data.quad_fd = mkstemp(parse_data.quad_fn); if (parse_data.quad_fd < 0) { fs_error(LOG_ERR, "Cannot create tmp file “%s”", parse_data.quad_fn); return 1; } parse_data.muri = raptor_new_uri((unsigned char *) model_uri); parse_data.model = g_strdup(model_uri); parse_data.model_hash = fs_hash_uri(model_uri); parse_data.count_trip = 0; parse_data.count_err = 0; parse_data.last_count = 0; parse_data.has_o_index = has_o_index; /* store the model uri */ buffer_res(link, parse_data.segments, parse_data.model_hash, parse_data.model, FS_RID_NULL, parse_data.dryrun); parse_data.parser = raptor_new_parser_for_content(NULL, mimetype, NULL, 0, (unsigned char *) parse_data.model); if (!parse_data.parser) { return 1; } /* use us as a vector for an indirect attack? no thanks */ raptor_set_feature(parse_data.parser, RAPTOR_FEATURE_NO_NET, 0); raptor_set_fatal_error_handler(parse_data.parser, link, fatal_rdf_parser_error); raptor_set_error_handler(parse_data.parser, link, rdf_parser_error); raptor_set_statement_handler(parse_data.parser, &parse_data, store_stmt); raptor_set_graph_handler(parse_data.parser, &parse_data, graph_handler); raptor_start_parse(parse_data.parser, parse_data.muri); fs_hash_freshen(); /* blank nodes are unique per file */ return 0; }
static int update_op(struct update_context *uc) { fs_rid_vector *vec[4]; switch (uc->op->type) { case RASQAL_UPDATE_TYPE_UNKNOWN: add_message(uc, "Unknown update operation", 0); return 1; case RASQAL_UPDATE_TYPE_CLEAR: fs_clear(uc, graph_arg(uc->op->graph_uri)); return 0; case RASQAL_UPDATE_TYPE_CREATE: return 0; case RASQAL_UPDATE_TYPE_DROP: fs_clear(uc, graph_arg(uc->op->graph_uri)); return 0; case RASQAL_UPDATE_TYPE_LOAD: fs_load(uc, graph_arg(uc->op->document_uri), graph_arg(uc->op->graph_uri)); return 0; #if RASQAL_VERSION >= 924 case RASQAL_UPDATE_TYPE_ADD: fs_add(uc, graph_arg(uc->op->graph_uri), graph_arg(uc->op->document_uri)); return 0; case RASQAL_UPDATE_TYPE_MOVE: fs_move(uc, graph_arg(uc->op->graph_uri), graph_arg(uc->op->document_uri)); return 0; case RASQAL_UPDATE_TYPE_COPY: fs_copy(uc, graph_arg(uc->op->graph_uri), graph_arg(uc->op->document_uri)); return 0; #endif case RASQAL_UPDATE_TYPE_UPDATE: break; } fs_hash_freshen(); raptor_sequence *todel = NULL; raptor_sequence *toins = NULL; if (uc->op->delete_templates && !uc->op->where) { int where = 0; /* check to see if it's a DELETE WHERE { } */ for (int t=0; t<raptor_sequence_size(uc->op->delete_templates); t++) { rasqal_triple *tr = raptor_sequence_get_at(uc->op->delete_templates, t); if (any_vars(tr)) { where = 1; break; } } if (where) { fs_error(LOG_ERR, "DELETE WHERE { x } not yet supported"); add_message(uc, "DELETE WHERE { x } not yet supported, use DELETE { x } WHERE { x }", 0); return 1; } } #if RASQAL_VERSION >= 923 if (uc->op->where) { todel = raptor_new_sequence(NULL, NULL); toins = raptor_new_sequence(NULL, NULL); raptor_sequence *todel_p = raptor_new_sequence(NULL, NULL); raptor_sequence *toins_p = raptor_new_sequence(NULL, NULL); raptor_sequence *vars = raptor_new_sequence(NULL, NULL); fs_query *q = calloc(1, sizeof(fs_query)); uc->q = q; q->qs = uc->qs; q->rq = uc->rq; q->flags = FS_BIND_DISTINCT; #ifdef DEBUG_MERGE q->flags |= FS_QUERY_CONSOLE_OUTPUT; #endif q->boolean = 1; q->opt_level = 3; q->soft_limit = -1; q->segments = fsp_link_segments(uc->link); q->link = uc->link; q->bb[0] = fs_binding_new(); q->bt = q->bb[0]; /* hashtable to hold runtime created resources */ q->tmp_resources = g_hash_table_new_full(fs_rid_hash, fs_rid_equal, g_free, fs_free_cached_resource); /* add column to denote join ordering */ fs_binding_create(q->bb[0], "_ord", FS_RID_NULL, 0); if (uc->op->delete_templates) { for (int t=0; t<raptor_sequence_size(uc->op->delete_templates); t++) { rasqal_triple *tr = raptor_sequence_get_at(uc->op->delete_templates, t); if (any_vars(tr)) { fs_check_cons_slot(q, vars, tr->subject); fs_check_cons_slot(q, vars, tr->predicate); fs_check_cons_slot(q, vars, tr->object); raptor_sequence_push(todel_p, tr); } else { raptor_sequence_push(todel, tr); } } } if (uc->op->insert_templates) { for (int t=0; t<raptor_sequence_size(uc->op->insert_templates); t++) { rasqal_triple *tr = raptor_sequence_get_at(uc->op->insert_templates, t); if (any_vars(tr)) { fs_check_cons_slot(q, vars, tr->subject); fs_check_cons_slot(q, vars, tr->predicate); fs_check_cons_slot(q, vars, tr->object); raptor_sequence_push(toins_p, tr); } else { raptor_sequence_push(toins, tr); } } } q->num_vars = raptor_sequence_size(vars); for (int i=0; i < q->num_vars; i++) { rasqal_variable *v = raptor_sequence_get_at(vars, i); fs_binding_add(q->bb[0], v, FS_RID_NULL, 1); } /* perform the WHERE match */ fs_query_process_pattern(q, uc->op->where, vars); q->length = fs_binding_length(q->bb[0]); for (int s=0; s<4; s++) { vec[s] = fs_rid_vector_new(0); } for (int t=0; t<raptor_sequence_size(todel_p); t++) { rasqal_triple *triple = raptor_sequence_get_at(todel_p, t); for (int row=0; row < q->length; row++) { delete_rasqal_triple(uc, vec, triple, row); } if (fs_rid_vector_length(vec[0]) > 1000) { fsp_delete_quads_all(uc->link, vec); } } if (fs_rid_vector_length(vec[0]) > 0) { fsp_delete_quads_all(uc->link, vec); } for (int s=0; s<4; s++) { //fs_rid_vector_print(vec[s], 0, stdout); fs_rid_vector_free(vec[s]); vec[s] = NULL; } for (int t=0; t<raptor_sequence_size(toins_p); t++) { rasqal_triple *triple = raptor_sequence_get_at(toins_p, t); for (int row=0; row < q->length; row++) { insert_rasqal_triple(uc, triple, row); } } /* must not free the rasqal_query */ q->rq = NULL; fs_query_free(q); uc->q = NULL; } else { todel = uc->op->delete_templates; toins = uc->op->insert_templates; } #else if (uc->op->where) { fs_error(LOG_ERR, "DELETE/INSERT WHERE requires Rasqal 0.9.23 or newer"); add_message(uc, "DELETE/INSERT WHERE requires Rasqal 0.9.23 or newer", 0); } #endif /* delete constant triples */ if (todel) { for (int s=0; s<4; s++) { vec[s] = fs_rid_vector_new(0); } for (int t=0; t<raptor_sequence_size(todel); t++) { rasqal_triple *triple = raptor_sequence_get_at(todel, t); if (any_vars(triple)) { continue; } delete_rasqal_triple(uc, vec, triple, 0); } if (fs_rid_vector_length(vec[0]) > 0) { fsp_delete_quads_all(uc->link, vec); } for (int s=0; s<4; s++) { fs_rid_vector_free(vec[s]); vec[s] = NULL; } } /* insert constant triples */ if (toins) { for (int t=0; t<raptor_sequence_size(toins); t++) { rasqal_triple *triple = raptor_sequence_get_at(toins, t); if (any_vars(triple)) { continue; } insert_rasqal_triple(uc, triple, 0); } } fs_hash_freshen(); return 0; }
int fs_import(fsp_link *link, const char *model_uri, char *resource_uri, const char *format, int verbosity, int dryrun, int has_o_index, FILE *msg, int *count) { raptor_parser *rdf_parser = NULL; raptor_uri ruri = NULL; int ret = 0; const int segments = fsp_link_segments(link); parse_data.ext_count = count; if (!inited) { inited = 1; parse_data.link = link; parse_data.segments = fsp_link_segments(link); for (int i=0; i<parse_data.segments; i++) { for (int j=0; j<RES_BUF_SIZE; j++) { lex_tmp[i][j] = malloc(RES_BUF_SIZE); } } memset(nodecache, 0, sizeof(nodecache)); parse_data.quad_fn = g_strdup(FS_TMP_PATH "/importXXXXXX"); parse_data.quad_fd = mkstemp(parse_data.quad_fn); if (parse_data.quad_fd < 0) { fs_error(LOG_ERR, "Cannot create tmp file “%s”", parse_data.quad_fn); return 1; } gettimeofday(&then_last, 0); } parse_data.verbosity = verbosity; parse_data.model = g_strdup(model_uri); parse_data.model_hash = fs_hash_uri(model_uri); parse_data.count_trip = 0; parse_data.last_count = 0; parse_data.dryrun = dryrun; parse_data.has_o_index = has_o_index; /* store the model uri */ buffer_res(link, segments, parse_data.model_hash, parse_data.model, FS_RID_NULL, dryrun); if (strcmp(format, "auto")) { rdf_parser = raptor_new_parser(format); } else if (strstr(resource_uri, ".n3") || strstr(resource_uri, ".ttl")) { rdf_parser = raptor_new_parser("turtle"); } else if (strstr(resource_uri, ".nt")) { rdf_parser = raptor_new_parser("ntriples"); } else { rdf_parser = raptor_new_parser("rdfxml"); } if (!rdf_parser) { fs_error(LOG_ERR, "failed to create RDF parser"); return 1; } raptor_set_statement_handler(rdf_parser, &parse_data, store_stmt); raptor_set_graph_handler(rdf_parser, &parse_data, graph_handler); ruri = raptor_new_uri((unsigned char *) resource_uri); parse_data.muri = raptor_new_uri((unsigned char *) model_uri); if (raptor_parse_uri(rdf_parser, ruri, parse_data.muri)) { fs_error(LOG_ERR, "failed to parse file “%s”", resource_uri); ret++; } if (verbosity) { printf("Pass 1, processed %d triples (%d)\n", total_triples_parsed, parse_data.count_trip); } raptor_free_parser(rdf_parser); raptor_free_uri(ruri); raptor_free_uri(parse_data.muri); g_free(parse_data.model); fs_hash_freshen(); /* blank nodes are unique per file */ return ret; }
static int update_op(struct update_context *ct) { fs_rid_vector *vec[4]; switch (ct->op->type) { case RASQAL_UPDATE_TYPE_UNKNOWN: add_message(ct, "Unknown update operation", 0); return 1; case RASQAL_UPDATE_TYPE_CLEAR: fs_clear(ct, (char *)raptor_uri_as_string(ct->op->graph_uri)); return 0; case RASQAL_UPDATE_TYPE_CREATE: return 0; case RASQAL_UPDATE_TYPE_DROP: fs_clear(ct, (char *)raptor_uri_as_string(ct->op->graph_uri)); return 0; case RASQAL_UPDATE_TYPE_LOAD: fs_load(ct, (char *)raptor_uri_as_string(ct->op->document_uri), (char *)raptor_uri_as_string(ct->op->graph_uri)); return 0; case RASQAL_UPDATE_TYPE_UPDATE: break; } fs_hash_freshen(); raptor_sequence *todel = NULL; raptor_sequence *toins = NULL; if (ct->op->where) { todel = raptor_new_sequence(NULL, NULL); toins = raptor_new_sequence(NULL, NULL); raptor_sequence *todel_p = raptor_new_sequence(NULL, NULL); raptor_sequence *toins_p = raptor_new_sequence(NULL, NULL); raptor_sequence *vars = raptor_new_sequence(NULL, NULL); fs_query *q = calloc(1, sizeof(fs_query)); ct->q = q; q->qs = ct->qs; q->rq = ct->rq; q->flags = FS_BIND_DISTINCT; q->opt_level = 3; q->soft_limit = -1; q->segments = fsp_link_segments(ct->link); q->link = ct->link; q->bb[0] = fs_binding_new(); q->bt = q->bb[0]; /* add column to denote join ordering */ fs_binding_add(q->bb[0], "_ord", FS_RID_NULL, 0); struct pattern_data pd = { .q = q, .vars = vars, .patterns = NULL, .fixed = NULL }; if (ct->op->delete_templates) { pd.patterns = todel_p; pd.fixed = todel; for (int t=0; t<raptor_sequence_size(ct->op->delete_templates); t++) { rasqal_graph_pattern *gp = raptor_sequence_get_at(ct->op->delete_templates, t); assign_gp(gp, NULL, &pd); } } if (ct->op->insert_templates) { pd.patterns = toins_p; pd.fixed = toins; for (int t=0; t<raptor_sequence_size(ct->op->insert_templates); t++) { rasqal_graph_pattern *gp = raptor_sequence_get_at(ct->op->insert_templates, t); assign_gp(gp, NULL, &pd); } } q->num_vars = raptor_sequence_size(vars); for (int i=0; i < q->num_vars; i++) { rasqal_variable *v = raptor_sequence_get_at(vars, i); fs_binding *b = fs_binding_get(q->bb[0], (char *)v->name); if (b) { b->need_val = 1; } else { fs_binding_add(q->bb[0], (char *)v->name, FS_RID_NULL, 1); } } fs_query_process_pattern(q, ct->op->where, vars); q->length = fs_binding_length(q->bb[0]); for (int s=0; s<4; s++) { vec[s] = fs_rid_vector_new(0); } for (int t=0; t<raptor_sequence_size(todel_p); t++) { rasqal_triple *triple = raptor_sequence_get_at(todel_p, t); for (int row=0; row < q->length; row++) { delete_rasqal_triple(ct, vec, triple, row); if (fs_rid_vector_length(vec[0]) > 0) { fsp_delete_quads_all(ct->link, vec); } } } for (int s=0; s<4; s++) { //fs_rid_vector_print(vec[s], 0, stdout); fs_rid_vector_free(vec[s]); } for (int t=0; t<raptor_sequence_size(toins_p); t++) { rasqal_triple *triple = raptor_sequence_get_at(toins_p, t); for (int row=0; row < q->length; row++) { insert_rasqal_triple(ct, triple, row); } } /* must not free the rasqal_query */ q->rq = NULL; fs_query_free(q); ct->q = NULL; } else {