static unsigned char * handle_delete_quads (fs_backend *be, fs_segment segment,
unsigned int length,
unsigned char *content)
{
if (segment > be->segments) {
fs_error(LOG_ERR, "invalid segment number: %d", segment);
return fsp_error_new(segment, "invalid segment number");
}
if (length < 32) {
fs_error(LOG_ERR, "delete_quads(%d) much too short", segment);
return fsp_error_new(segment, "much too short");
}
fs_rid_vector models, subjects, predicates, objects;
models.size = models.length = length / 32;
subjects.size = subjects.length = length / 32;
predicates.size = predicates.length = length / 32;
objects.size = objects.length = length / 32;
if (length < (models.size + subjects.size + predicates.size + objects.size) * 8) {
fs_error(LOG_ERR, "delete_quads(%d) too short (%d < %d)", segment, length,
(models.size + subjects.size + predicates.size + objects.size) * 8);
return fsp_error_new(segment, "too short");
}
models.data = (fs_rid *) content;
content += models.length * 8;
subjects.data = (fs_rid *) content;
content += subjects.length * 8;
predicates.data = (fs_rid *) content;
content += predicates.length * 8;
objects.data = (fs_rid *) content;
fs_rid_vector *args[4] = { &models, &subjects, &predicates, &objects };
fs_delete_quads(be, args);
/* FIXME, should check return value */
return message_new(FS_DONE_OK, 0, 0);
}
unsigned char *message_recv(int sock,
unsigned int *segment,
unsigned int *length)
{
int err;
unsigned char header[FS_HEADER];
unsigned char *buffer, *p;
unsigned int * const l = (unsigned int *) (header + 4);
unsigned int * const s = (unsigned int *) (header + 8);
unsigned int len;
err= recv(sock, header, FS_HEADER, MSG_WAITALL);
if (err < 0) {
fs_error(LOG_ERR, "recv header from socket failed, %s", strerror(errno));
return NULL;
} else if (err == 0) {
return NULL;
}
if (memcmp(header, fsp_vermagic, 3)) {
fs_error(LOG_ERR, "incorrect version magic %02x%02x%02x", header[0], header[1], header[2]);
return NULL;
}
*segment = *s;
len = *length = *l; /* FIXME check length overflow */
buffer = calloc(1, FS_HEADER + len);
memcpy(buffer, header, FS_HEADER);
p = buffer + FS_HEADER;
while (len > 0) {
int count = recv(sock, p, len, 0);
if (count <= 0) {
fs_error(LOG_ERR, "recv body from socket failed, %s", strerror(errno));
break;
}
p+= count;
len-= count;
}
return buffer;
}
fs_metadata *fs_metadata_open(const char *kb)
{
fs_metadata *m = calloc(1, sizeof(fs_metadata));
gchar *fs_md_file_uri_format;
fs_md_file_uri_format = g_strconcat("file://",
fs_get_md_file_format(),
NULL);
m->size = 16;
m->length = 0;
m->entries = calloc(m->size, sizeof(struct m_entry));
m->uri = g_strdup_printf(fs_md_file_uri_format, kb);
g_free(fs_md_file_uri_format);
int fd;
if ((fd = open(m->uri + 7, FS_O_NOATIME | O_CREAT, FS_FILE_MODE)) == -1) {
fs_error(LOG_CRIT, "failed to touch metadata file %s: %s",
m->uri, strerror(errno));
free(m);
return NULL;
}
close(fd);
m->rw = raptor_new_world();
if (!m->rw) {
fs_error(LOG_CRIT, "failed to initialise raptor");
free(m);
return NULL;
}
raptor_parser *rdf_parser = raptor_new_parser(m->rw, "turtle");
raptor_parser_set_statement_handler(rdf_parser, m, parse_stmt);
char *uri = strdup(m->uri);
raptor_uri *ruri = raptor_new_uri(m->rw, (unsigned char *) uri);
raptor_uri *muri = raptor_new_uri(m->rw, (unsigned char *) uri);
free(uri);
if (raptor_parser_parse_uri(rdf_parser, ruri, muri)) {
fs_error(LOG_ERR, "failed to parse metadata file “%s”", m->uri);
return NULL;
}
raptor_free_parser(rdf_parser);
raptor_free_uri(ruri);
raptor_free_uri(muri);
return m;
}
fs_binding *fs_binding_add(fs_binding *b, rasqal_variable *var, fs_rid val, int projected)
{
#ifdef DEBUG_BINDING
if (strcmp(DEBUG_BINDING, name)) printf("@@ add("DEBUG_BINDING", %016llx, %d)\n", val, projected);
#endif
fs_binding *bv = fs_binding_get(b, var);
if (bv) {
fs_rid_vector_append(bv->vals, val);
bv->bound = 1;
bv->proj |= projected;
bv->need_val |= projected;
return bv;
}
long i;
for (i=0; i < FS_BINDING_MAX_VARS && b[i].name; i++);
if (i == FS_BINDING_MAX_VARS) {
fs_error(LOG_ERR, "variable limit (%d) exceeded",
FS_BINDING_MAX_VARS);
return NULL;
}
b[i].name = g_strdup((char *)var->name);
if (val != FS_RID_NULL) {
if (b[i].vals) {
fs_error(LOG_WARNING, "loosing pointer to rid_vector");
}
b[i].vals = fs_rid_vector_new_from_args(1, val);
b[i].bound = 1;
} else {
if (b[i].vals) {
fs_error(LOG_WARNING, "loosing pointer to rid_vector");
}
b[i].vals = fs_rid_vector_new(0);
}
b[i].proj = projected;
b[i].need_val = projected;
var->user_data = (void *)i;
return b+i;
}
fs_row_id fs_ptable_get_next(fs_ptable *pt, fs_row_id r)
{
if (r > pt->header->length) {
fs_error(LOG_CRIT, "tried to read off end of ptable %s", pt->filename);
return 0;
}
return pt->data[r].cont;
}
static void map_file(fs_ptree *pt)
{
pt->ptr = mmap(NULL, pt->file_length, PROT_READ | PROT_WRITE, MAP_SHARED, pt->fd, 0);
if (pt->ptr == (void *)-1) {
fs_error(LOG_ERR, "failed to mmap '%s'", pt->filename);
}
pt->header = pt->ptr;
pt->nodes = (node *)((char *)(pt->ptr) + sizeof(struct ptree_header));
pt->leaves = (leaf *)(pt->nodes);
}
static void error_handler(void *user_data, raptor_log_message *message)
{
struct update_context *uc = user_data;
char *msg = g_strdup_printf("%s: %s at line %d of operation %d",
raptor_log_level_get_label(message->level), message->text, raptor_locator_line(message->locator), uc->opid);
uc->error = (message->level == RAPTOR_LOG_LEVEL_ERROR);
add_message(uc, msg, 1);
fs_error(LOG_ERR, "%s", msg);
}
int fs_ptable_sync(fs_ptable *pt)
{
if (msync(pt->ptr, pt->len, MS_SYNC) == -1) {
fs_error(LOG_CRIT, "failed to msync ptable: %s", strerror(errno));
return 1;
}
return 0;
}
int fs_ptable_get_row(fs_ptable *pt, fs_row_id b, fs_rid pair[2])
{
if (b == 0) {
fs_error(LOG_CRIT, "tried to read row 0\n");
return 1;
}
if (b > pt->header->length) {
fs_error(LOG_CRIT, "tried to read off end of ptable\n");
return 1;
}
row *r = &(pt->data[b]);
pair[0] = r->data[0];
pair[1] = r->data[1];
return 0;
}
int fs_tbchain_sync(fs_tbchain *bc)
{
if (msync(bc->ptr, bc->len, MS_SYNC) == -1) {
fs_error(LOG_CRIT, "failed to msync chain: %s", strerror(errno));
return 1;
}
return 0;
}
static int bind_same(const fs_rid ref[4], int flags)
{
int match = 1;
switch (flags & FS_BIND_SAME_MASK) {
case FS_BIND_SAME_XXXX:
break;
case FS_BIND_SAME_XXAA:
match = (ref[2] == ref[3]);
break;
case FS_BIND_SAME_XAXA:
match = (ref[1] == ref[3]);
break;
case FS_BIND_SAME_XAAX:
match = (ref[1] == ref[2]);
break;
case FS_BIND_SAME_XAAA:
match = (ref[1] == ref[2] && ref[2] == ref[3]);
break;
case FS_BIND_SAME_AXXA:
match = (ref[0] == ref[3]);
break;
case FS_BIND_SAME_AXAX:
match = (ref[0] == ref[2]);
break;
case FS_BIND_SAME_AXAA:
match = (ref[0] == ref[2] && ref[2] == ref[3]);
break;
case FS_BIND_SAME_AAXX:
match = (ref[0] == ref[1]);
break;
case FS_BIND_SAME_AAXA:
match = (ref[0] == ref[1] && ref[1] == ref[3]);
break;
case FS_BIND_SAME_AAAX:
match = (ref[0] == ref[1] && ref[1] == ref[2]);
break;
case FS_BIND_SAME_AAAA:
match = (ref[0] == ref[1] && ref[1] == ref[2] &&
ref[2] == ref[3]);
break;
case FS_BIND_SAME_AABB:
match = (ref[0] == ref[1] && ref[2] == ref[3]);
break;
case FS_BIND_SAME_ABAB:
match = (ref[0] == ref[2] && ref[1] == ref[3]);
break;
case FS_BIND_SAME_ABBA:
match = (ref[0] == ref[3] && ref[1] == ref[2]);
break;
default:
fs_error(LOG_ERR, "unhandled BIND_SAME value");
}
return match;
}
static int io_readline (lua_State *L) {
int *pf = (int *)lua_touserdata(L, lua_upvalueindex(1));
int sucess;
if (pf == NULL || *pf == FS_OPEN_OK - 1){ /* file is already closed? */
luaL_error(L, "file is already closed");
return 0;
}
sucess = read_line(L, *pf);
if (fs_error(*pf))
return luaL_error(L, "err(%d)", fs_error(*pf));
if (sucess) return 1;
else { /* EOF */
if (lua_toboolean(L, lua_upvalueindex(2))) { /* generator created file? */
lua_settop(L, 0);
lua_pushvalue(L, lua_upvalueindex(1));
aux_close(L); /* close it */
}
return 0;
}
}
static unsigned char * handle_get_size_reverse (fs_backend *be, fs_segment segment,
unsigned int length,
unsigned char *content)
{
if (segment > be->segments) {
fs_error(LOG_ERR, "invalid segment number: %d", segment);
return fsp_error_new(segment, "invalid segment number");
}
if (length > 0) {
fs_error(LOG_ERR, "get_data_size(%d) extraneous content", segment);
return fsp_error_new(segment, "extraneous content");
}
fs_data_size size = fs_get_data_size(be, segment);
unsigned char *reply = message_new(FS_SIZE_REVERSE, segment, sizeof(size.quads_sr));
memcpy(reply + FS_HEADER, &size.quads_sr, sizeof(size.quads_sr));
return reply;
}
void fs_ptree_free_leaf(fs_ptree *pt, nodeid n)
{
if (IS_NODE(n)) {
fs_error(LOG_ERR, "tried to free node as leaf");
return;
}
leaf *lr = LEAF_REF(pt, n);
lr->block = pt->header->leaf_free;
pt->header->leaf_free = n;
}
int fs_bind_done(fs_backend *be, fs_segment segment)
{
if (be->stream) {
/* TODO cleanup */
return 0;
} else {
fs_error(LOG_ERR, "bind_done(%d) while not streaming", segment);
return 1;
}
}
static void xml_end_document(void *user_data)
{
xmlctxt *ctxt = (xmlctxt *) user_data;
if (ctxt->state != DONE_TRIX) {
fs_error(LOG_ERR, "TriX document ends abruptly");
}
g_free(ctxt->resource);
ctxt->resource = NULL;
g_free(ctxt->attr);
ctxt->attr = NULL;
}
static unsigned char * handle_auth (fs_backend *be,
fs_segment segment,
unsigned int length,
unsigned char *content)
{
md5_state_t md5;
if (length <= 16) {
fs_error(LOG_ERR, "auth(%d) missing kbname", segment);
return fsp_error_new(segment, "missing kbname");
}
if (strncmp(be->db_name, (char *) content + 16, length - 16)) {
fs_error(LOG_ERR, "auth(%d) wrong kbname", segment);
return fsp_error_new(segment, "wrong kbname");
}
if (be->salt) {
unsigned char data[20], hash[16];
char string[33];
memcpy(data, (&be->salt), sizeof(be->salt));
memcpy(data + 4, content, 16);
md5_init(&md5);
md5_append(&md5, data, sizeof(data));
md5_finish(&md5, hash);
sprintf(string, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x",
hash[0], hash[1], hash[2], hash[3], hash[4], hash[5], hash[6], hash[7],
hash[8], hash[9], hash[10], hash[11], hash[12], hash[13], hash[14], hash[15]);
if (strcmp(string, be->hash)) {
fs_error(LOG_ERR, "auth(%d) invalid password", segment);
return fsp_error_new(segment, "invalid password");
}
}
/* in-band signal, string of features */
unsigned char *reply = message_new(FS_DONE_OK, 0, sizeof(feature_string));
memcpy(reply + FS_HEADER, feature_string, sizeof(feature_string));
return reply;
}
fs_ptable *fs_ptable_open_filename(const char *fname, int flags)
{
fs_ptable *pt = calloc(1, sizeof(fs_ptable));
if (sizeof(struct ptable_header) != 512) {
fs_error(LOG_CRIT, "ptable header size not 512 bytes");
return NULL;
}
pt->fd = open(fname, FS_O_NOATIME | flags, FS_FILE_MODE);
if (pt->fd == -1) {
fs_error(LOG_CRIT, "failed to open ptable %s: %s", fname, strerror(errno));
return NULL;
}
pt->filename = g_strdup(fname);
pt->flags = flags;
struct ptable_header header;
lseek(pt->fd, 0, SEEK_SET);
if (flags & O_TRUNC || read(pt->fd, &header, sizeof(header)) <= 0) {
header.id = PTABLE_ID;
header.size = 1024;
header.length = 0;
header.revision = PTABLE_REVISION;
}
if (header.id != PTABLE_ID) {
fs_error(LOG_CRIT, "%s does not look like a ptable file", pt->filename);
return NULL;
}
if (header.revision != PTABLE_REVISION) {
fs_error(LOG_CRIT, "%s is wrong revision of ptable file", pt->filename);
return NULL;
}
if (map_pt(pt, header.length, header.size)) {
return NULL;
}
return pt;
}
fs_value fs_value_rid(fs_rid r)
{
fs_value v = fs_value_blank();
if (r == FS_RID_GONE) {
fs_error(LOG_ERR, "found RID_GONE value, optimiser elimination bug");
r = FS_RID_NULL;
}
v.rid = r;
v.valid = fs_valid_bit(FS_V_RID);
return v;
}
static unsigned char * handle_get_query_times (fs_backend *be,
fs_segment segment,
unsigned int length,
unsigned char *content)
{
if (segment > be->segments) {
fs_error(LOG_ERR, "invalid segment number: %d", segment);
return fsp_error_new(segment, "invalid segment number");
}
if (length > 0) {
fs_error(LOG_ERR, "get_query_times(%d) extraneous content", segment);
return fsp_error_new(segment, "extraneous content");
}
fs_query_timing timing = fs_get_query_times(be, segment);
unsigned char *reply = message_new(FS_QUERY_TIMES, segment, sizeof(timing));
memcpy(reply + FS_HEADER, &timing, sizeof(timing));
return reply;
}
static unsigned char * handle_choose_segment (fs_backend *be, fs_segment segment,
unsigned int length,
unsigned char *content)
{
if (fs_backend_open_files(be, segment, O_RDWR | O_CREAT, FS_OPEN_ALL)) {
fs_error(LOG_ERR, "failed to open files for segment %d", segment);
return fsp_error_new(segment, "cannot open indexes");
}
return message_new(FS_DONE_OK, 0, 0);
}
int fs_tlist_truncate(fs_tlist *l)
{
if (ftruncate(l->fd, sizeof(struct tlist_header)) == -1) {
fs_error(LOG_CRIT, "failed to truncate '%s': %s", l->filename, strerror(errno));
return 1;
}
l->offset = 0;
l->buffer_pos = 0;
return 0;
}
static fs_rid insert_uri(xmlctxt *ctxt)
{
char *uri = ctxt->resource;
if (!uri || uri[0] == '\0') {
fs_error(LOG_ERR, "NULL URI inserted");
return 0;
}
fs_rid r = fs_hash_uri(uri);
insert_resource(ctxt, r, fs_c.empty, uri);
return r;
}
void fs_binding_set_used(fs_binding *b, rasqal_variable *var)
{
#ifdef DEBUG_BINDING
if (!strcmp(DEBUG_BINDING, name)) printf("@@ set_used("DEBUG_BINDING")\n");
#endif
fs_binding *vb = fs_binding_get(b, var);
if (vb) {
vb->used = 1;
} else {
fs_error(LOG_ERR, "tried to set 'used' on unknown varaible %s", var->name);
}
}
static fs_index_node fs_tbchain_new_block(fs_tbchain *bc)
{
TIME(NULL);
if (!bc->ptr) {
fs_error(LOG_CRIT, "attempted to get block from unmapped chain");
return 0;
}
if (bc->header->length == 0) {
fs_error(LOG_CRIT, "length reset, this is a bug, attempting recovery");
bc->header->length = bc->header->size;
}
/* if free list is empty, we need to allocate space */
if (!bc->header->free_list) {
int length = bc->header->length;
int size = bc->header->size;
unmap_bc(bc);
map_bc(bc, length, size * 2);
TIME("grow chain");
bc->header->length = size * 2;
for (fs_index_node i = size; i < size * 2; i++) {
fs_tbchain_free_block(bc, i);
}
}
if (bc->header->free_list) {
fs_index_node newb = bc->header->free_list;
bc->header->free_list = bc->data[newb].cont;
memset(&bc->data[newb], 0, sizeof(fs_tblock));
TIME("reuse + clear block");
return newb;
}
fs_error(LOG_CRIT, "failed to get free block");
return 0;
}
int fs_ptree_write_header(fs_ptree *pt)
{
struct ptree_header header;
memset(&header, 0, sizeof(header));
header.id = FS_PTREE_ID;
header.revision = FS_PTREE_REVISION;
header.node_size = FS_PTREE_SIZE_INC;
header.node_alloc = FS_PTREE_SIZE_INC;
header.node_count = 2;
header.leaf_size = FS_PTREE_SIZE_INC * sizeof(node) / sizeof(leaf) + FS_PTREE_SIZE_INC;
header.leaf_alloc = FS_PTREE_SIZE_INC;
header.leaf_count = 2;
/* reserve node 0 as the null node, and 1 as the root */
header.node_base = 2;
/* reserve the null leaf */
header.leaf_base = header.node_size * sizeof(node) / sizeof(leaf) + 1;
header.alloc = header.leaf_size * sizeof(leaf);
if (pwrite(pt->fd, &header, sizeof(header), 0) == -1) {
fs_error(LOG_CRIT, "failed to write header on %s: %s",
pt->filename, strerror(errno));
return 1;
}
pt->file_length = sizeof(struct ptree_header) + header.alloc;
char junk = '\0';
if (pwrite(pt->fd, &junk, 1, pt->file_length) == -1) {
fs_error(LOG_ERR, "failed to extend ptree file");
}
map_file(pt);
memset(pt->nodes, 0, sizeof(node));
memset(pt->leaves, 0, sizeof(leaf));
node *root = pt->nodes+1;
for (int i=0; i<FS_PTREE_BRANCHES; i++) {
root->branch[i] = FS_PTREE_NULL_NODE;
}
return 0;
}
fs_rid_vector *fs_mhash_get_keys(fs_mhash *mh)
{
if (!mh) {
fs_error(LOG_CRIT, "tried to get keys from NULL mhash");
return NULL;
}
fs_rid_vector *v = fs_rid_vector_new(0);
fs_mhash_entry e;
if (!mh->locked) flock(mh->fd, LOCK_SH);
if (lseek(mh->fd, sizeof(struct mhash_header), SEEK_SET) == -1) {
fs_error(LOG_ERR, "seek error on mhash: %s", strerror(errno));
}
while (read(mh->fd, &e, sizeof(e)) == sizeof(e)) {
if (e.val) fs_rid_vector_append(v, e.rid);
}
if (!mh->locked) flock(mh->fd, LOCK_UN);
return v;
}
int fs_tlist_next_value(fs_tlist *l, void *out)
{
if (!l) {
fs_error(LOG_ERR, "cannot read from null list");
return 0;
}
int ret = read(l->fd, out, WIDTH);
if (ret == -1) {
fs_error(LOG_ERR, "error reading entry from list: %s\n", strerror(errno));
return 0;
} else if (ret == 0) {
return 0;
} else if (ret != WIDTH) {
fs_error(LOG_ERR, "error reading entry from list, got %d bytes instead of %zd\n", ret, WIDTH);
return 0;
}
return 1;
}
int fs_tree_sync(fs_tree *t)
{
//TIME(NULL);
fs_chain_sync(t->bc);
if (msync(t->ptr, t->len, MS_SYNC) == -1) {
fs_error(LOG_CRIT, "failed to msync tree: %s", strerror(errno));
return 1;
}
//TIME("tree sync");
return 0;
}
int fs_binding_set_expression(fs_binding *b, rasqal_variable *var, rasqal_expression *ex)
{
fs_binding *vb = fs_binding_get(b, var);
if (vb) {
vb->expression = ex;
return 0;
}
fs_error(LOG_ERR, "cannot find varaible %s", var->name);
return 1;
}