static void push_str(CParseInfo *pCPI, enum CTErrorSeverity severity, void *str)
{
if (pCPI == NULL || pCPI->errorStack == NULL)
F.fatalerr(str);
LL_push(pCPI->errorStack, error_new(severity, str));
}
Error *context_interpret(Context *context, const char *str)
{
Value *value;
if (context->defining_variable) { // 変数定義
context->defining_variable = FALSE;
value = value_new_symbol(str);
if (NULL == value) { // エラー (変数名不正など)
return error_new(IllegalVariableError, NULL);
}
map_put(context->map, value_symbol_name(value), value);
} else if (str_is_integer(str)) { // 整数
value = value_new_integer_str(str);
stack_push(context->stack, value);
} else if (0 == strcmp(str, "VARIABLE")) { // 変数定義の開始
context->defining_variable = TRUE;
} else if (NULL != (value = context_resolve(context, str))) { // シンボル
ForshFunc *func;
switch (value->type) {
case TYPE_FUNCTION:
func = value_function(value);
return func(context->stack);
default:
stack_push(context->stack, value);
break;
}
} else {
fprintf(stderr, "Failed to interpret: %s\n", str);
}
return NULL;
}
// [new] ReferenceError(message)
js_val *
error_ref_new(js_val *instance, js_args *args, eval_state *state)
{
js_val *err = error_new(instance, args, state);
fh_set_class(err, "ReferenceError");
return err;
}
// [new] URIError(message)
js_val *
error_uri_new(js_val *instance, js_args *args, eval_state *state)
{
js_val *err = error_new(instance, args, state);
fh_set_class(err, "URIError");
return err;
}
// [new] URIError(message)
js_val *
error_uri_new(js_val *instance, js_args *args, eval_state *state)
{
js_val *err = error_new(instance, args, state);
fh_set(err, "name", JSSTR(E_URI));
err->proto = fh_try_get_proto(E_URI);
return err;
}
void error_set_current(enum error_types etype, int line_number) {
current_error = error_new(etype, line_number);
//error_print();
//YYABORT;
//printf("Fin du programme imprévue.\n");
//error_on_exit();
//exit(EXIT_FAILURE);
};
PageRankScorerError
page_rank_scorer_new(PageRankScorer **prs, PageDB *db) {
PageRankScorer *p = *prs = malloc(sizeof(*p));
if (!p)
return page_rank_scorer_error_memory;
p->error = error_new();
if (p->error == 0) {
free(p);
return page_rank_scorer_error_memory;
}
p->page_db = db;
if (page_rank_new(&p->page_rank, db->path, 1000) != 0) {
page_rank_scorer_set_error(p, page_rank_scorer_error_internal, __func__);
page_rank_scorer_add_error(p, "initializing PageRank");
page_rank_scorer_add_error(p, p? p->error->message: "NULL");
return p->error->code;
}
page_rank_scorer_set_persist(p, PAGE_RANK_SCORER_PERSIST);
page_rank_scorer_set_use_content_scores(p, PAGE_RANK_SCORER_USE_CONTENT_SCORES);
return 0;
}
PageDBError
page_db_new(PageDB **db, const char *path) {
PageDB *p = *db = malloc(sizeof(*p));
if (p == 0)
return page_db_error_memory;
p->error = error_new();
if (p->error == 0) {
free(p);
return page_db_error_memory;
}
p->persist = PAGE_DB_DEFAULT_PERSIST;
p->domain_temp = 0;
// create directory if not present yet
const char *error = make_dir(path);
if ((p->path = strdup(path)) == 0) {
error = "could not duplicate path string";
}
if (error != 0) {
page_db_set_error(p, page_db_error_invalid_path, __func__);
page_db_add_error(p, error);
return p->error->code;
}
if (txn_manager_new(&p->txn_manager, 0) != 0) {
page_db_set_error(p, page_db_error_internal, __func__);
page_db_add_error(p, p->txn_manager?
p->txn_manager->error->message:
"NULL");
return p->error->code;
}
// initialize LMDB on the directory
MDB_txn *txn;
MDB_dbi dbi;
int mdb_rc = 0;
if ((mdb_rc = mdb_env_create(&p->txn_manager->env) != 0))
error = "creating environment";
else if ((mdb_rc = mdb_env_set_mapsize(
p->txn_manager->env, PAGE_DB_DEFAULT_SIZE)) != 0)
error = "setting map size";
else if ((mdb_rc = mdb_env_set_maxdbs(p->txn_manager->env, 5)) != 0)
error = "setting number of databases";
else if ((mdb_rc = mdb_env_open(
p->txn_manager->env,
path,
MDB_NOTLS | MDB_NOSYNC, 0664) != 0))
error = "opening environment";
else if ((mdb_rc = txn_manager_begin(p->txn_manager, 0, &txn)) != 0)
error = "starting transaction";
// create all database
else if ((mdb_rc = mdb_dbi_open(txn,
"hash2info",
MDB_CREATE | MDB_INTEGERKEY,
&dbi)) != 0)
error = "creating hash2info database";
else if ((mdb_rc = mdb_dbi_open(txn,
"hash2idx",
MDB_CREATE | MDB_INTEGERKEY,
&dbi)) != 0)
error = "creating hash2idx database";
else if ((mdb_rc = mdb_dbi_open(txn,
"links",
MDB_CREATE | MDB_INTEGERKEY,
&dbi)) != 0)
error = "creating links database";
else if ((mdb_rc = mdb_dbi_open(txn, "info", MDB_CREATE, &dbi)) != 0)
error = "creating info database";
else {
// initialize n_pages inside info database
size_t n_pages = 0;
MDB_val key = {
.mv_size = sizeof(info_n_pages),
.mv_data = info_n_pages
};
MDB_val val = {
.mv_size = sizeof(size_t),
.mv_data = &n_pages
};
switch (mdb_rc = mdb_put(txn, dbi, &key, &val, MDB_NOOVERWRITE)) {
case MDB_KEYEXIST:
case 0:
if (txn_manager_commit(p->txn_manager, txn) != 0)
error = p->txn_manager->error->message;
break; // we good
default:
error = "could not initialize info.n_pages";
}
}
if (error != 0) {
page_db_set_error(p, page_db_error_internal, __func__);
page_db_add_error(p, error);
page_db_add_error(p, mdb_strerror(mdb_rc));
mdb_env_close(p->txn_manager->env);
}
return p->error->code;
}
/** Store a new or updated @ref PageInfo for a crawled page.
*
* @param cur An open cursor to the hash2info database
* @param key The key (hash) to the page
* @param page
* @param mdb_error In case of failure, if the error occurs inside LMDB this output parameter
* will be set with the error (otherwise is set to zero).
* @return 0 if success, -1 if failure.
*/
static int
page_db_add_crawled_page_info(MDB_cursor *cur,
MDB_val *key,
const CrawledPage *page,
PageInfo **page_info,
int *mdb_error) {
MDB_val val;
*page_info = 0;
int mdb_rc = mdb_cursor_get(cur, key, &val, MDB_SET);
int put_flags = 0;
switch (mdb_rc) {
case 0:
if (!(*page_info = page_info_load(&val)))
goto on_error;
if ((page_info_update(*page_info, page) != 0))
goto on_error;
put_flags = MDB_CURRENT;
break;
case MDB_NOTFOUND:
if (!(*page_info = page_info_new_crawled(page)))
goto on_error;
break;
default:
goto on_error;
}
if ((page_info_dump(*page_info, &val) != 0))
goto on_error;
if ((mdb_rc = mdb_cursor_put(cur, key, &val, put_flags)) != 0)
goto on_error;
free(val.mv_data);
val.mv_data = 0;
*mdb_error = 0;
return 0;
on_error:
if (val.mv_data)
free(val.mv_data);
*mdb_error = mdb_rc;
page_info_delete(*page_info);
return -1;
}
/** Store a new or updated @ref PageInfo for an uncrawled link.
*
* @param cur An open cursor to the hash2info database
* @param key The key (hash) to the page
* @param url
* @param mdb_error In case of failure, if the error occurs inside LMDB this output parameter
* will be set with the error (otherwise is set to zero).
* @return 0 if success, -1 if failure.
*/
static int
page_db_add_link_page_info(MDB_cursor *cur,
MDB_val *key,
uint64_t linked_from,
uint64_t depth,
const LinkInfo *link,
PageInfo **page_info,
int *mdb_error) {
MDB_val val;
int mdb_rc = 0;
PageInfo *pi = *page_info =
page_info_new_link(link->url, linked_from, depth, link->score);
if (!pi)
goto on_error;
if ((page_info_dump(pi, &val) != 0))
goto on_error;
if ((mdb_rc = mdb_cursor_put(cur, key, &val, MDB_NOOVERWRITE)) != 0)
goto on_error;
free(val.mv_data);
val.mv_data = 0;
*mdb_error = 0;
return 0;
on_error:
if (val.mv_data)
free(val.mv_data);
*mdb_error = mdb_rc;
page_info_delete(pi);
return -1;
}
FreqSchedulerError
freq_scheduler_new(FreqScheduler **sch, PageDB *db, const char *path) {
FreqScheduler *p = *sch = malloc(sizeof(*p));
if (p == 0)
return freq_scheduler_error_memory;
p->error = error_new();
if (p->error == 0) {
free(p);
return freq_scheduler_error_memory;
}
p->page_db = db;
p->persist = FREQ_SCHEDULER_DEFAULT_PERSIST;
p->margin = -1.0; // disabled
p->max_n_crawls = 0;
// create directory if not present yet
char *error = 0;
p->path = path? strdup(path): concat(db->path, "freqs", '_');
if (!p->path)
error = "building scheduler path";
else
error = make_dir(p->path);
if (error != 0) {
freq_scheduler_set_error(p, freq_scheduler_error_invalid_path, __func__);
freq_scheduler_add_error(p, error);
return p->error->code;
}
if (txn_manager_new(&p->txn_manager, 0) != 0) {
freq_scheduler_set_error(p, freq_scheduler_error_internal, __func__);
freq_scheduler_add_error(p, p->txn_manager?
p->txn_manager->error->message:
"NULL");
return p->error->code;
}
int rc;
// initialize LMDB on the directory
if ((rc = mdb_env_create(&p->txn_manager->env) != 0))
error = "creating environment";
else if ((rc = mdb_env_set_mapsize(p->txn_manager->env,
FREQ_SCHEDULER_DEFAULT_SIZE)) != 0)
error = "setting map size";
else if ((rc = mdb_env_set_maxdbs(p->txn_manager->env, 1)) != 0)
error = "setting number of databases";
else if ((rc = mdb_env_open(
p->txn_manager->env,
p->path,
MDB_NOTLS | MDB_NOSYNC, 0664) != 0))
error = "opening environment";
if (error != 0) {
freq_scheduler_set_error(p, freq_scheduler_error_internal, __func__);
freq_scheduler_add_error(p, error);
freq_scheduler_add_error(p, mdb_strerror(rc));
return p->error->code;
}
return p->error->code;
}
static void processor(const char *tx_addr, int port)
{
sys_init(1);
sys_log('E', "started port=%d, tx=%s\n", port, tx_addr);
/* connect to the control thread */
char addr[MAX_ADDR];
sys_address(addr, port);
IO *io = sys_connect(addr, IO_CHUNK);
tx_attach(tx_addr);
/* get env code from the tx */
char *code = tx_program();
char *res = mem_alloc(MAX_BLOCK);
while (!io->stop) {
sys_iready(io, -1);
int status = -1;
long long sid = 0LL, time = sys_millis();
Env *env = NULL;
Arg *arg = NULL;
Vars *v = vars_new(0), *r = NULL, *w = NULL;
Http_Req *req = http_parse_req(io);
if (io->stop)
goto exit;
if (req == NULL) {
status = http_400(io);
goto exit;
}
if (req->method == OPTIONS) {
status = http_opts(io);
goto exit;
}
env = env_new("net", code);
if (str_idx(req->path, "/fn") == 0) {
int idx = (req->path[3] == '/') ? 4 : 3;
int i = 0, len = 1, cnt = 0;
Func **fns = env_funcs(env, req->path + idx, &cnt);
status = http_200(io);
while (status == 200 && len) {
len = pack_fn2csv(fns, cnt, res, MAX_BLOCK, &i);
status = http_chunk(io, res, len);
}
mem_free(fns);
goto exit;
}
/* compare the request with the function defintion */
Func *fn = env_func(env, req->path + 1);
if (fn == NULL) {
Error *err = error_new("unknown function '%s'", req->path + 1);
status = http_404(io, err->msg);
mem_free(err);
goto exit;
}
if (fn->rp.name != NULL && req->method != POST) {
status = http_405(io, POST);
goto exit;
}
if (fn->rp.name == NULL && req->method == POST) {
status = http_405(io, GET);
goto exit;
}
/* TODO: think what to do with duplicate parameter values */
for (int i = 0; i < req->args->len; ++i) {
char *name = req->args->names[i];
if (array_freq(req->args->names, req->args->len, name) > 1) {
Error *err = error_new("duplicate parameter '%s' "
"(not supported)",
name);
status = http_404(io, err->msg);
mem_free(err);
goto exit;
}
}
if (fn->pp.len != req->args->len) {
Error *err = error_new("expected %d primitive parameters, got %d",
fn->pp.len, req->args->len);
status = http_404(io, err->msg);
mem_free(err);
goto exit;
}
arg = mem_alloc(sizeof(Arg));
for (int i = 0; i < fn->pp.len; ++i) {
char *name = fn->pp.names[i];
Type t = fn->pp.types[i];
int idx = array_scan(req->args->names, req->args->len, name);
if (idx < 0) {
Error *err = error_new("unknown parameter '%s'", name);
status = http_404(io, err->msg);
mem_free(err);
goto exit;
}
char *val = req->args->vals[idx];
int error = 0;
if (t == Int) {
arg->vals[i].v_int = str_int(val, &error);
} else if (t == Real)
arg->vals[i].v_real = str_real(val, &error);
else if (t == Long)
arg->vals[i].v_long = str_long(val, &error);
else if (t == String) {
error = str_len(val) > MAX_STRING;
if (!error)
str_cpy(arg->vals[i].v_str, val);
}
if (error) {
Error *err = error_new("value '%s' (parameter '%s') "
"is not of type '%s'",
val, name, type_to_str(t));
status = http_404(io, err->msg);
mem_free(err);
goto exit;
}
}
if (fn->rp.name != NULL) {
TBuf *body = NULL;
if (req->len > 0) {
Error *err = pack_csv2rel(req->body, fn->rp.head, &body);
if (err != NULL) {
status = http_404(io, err->msg);
mem_free(err);
goto exit;
}
} else {
body = tbuf_new();
}
vars_add(v, fn->rp.name, 0, body);
/* project the parameter */
Rel *param = rel_project(rel_load(fn->rp.head, fn->rp.name),
fn->rp.head->names,
fn->rp.head->len);
rel_eval(param, v, arg);
/* clean the previous version */
tbuf_clean(body);
tbuf_free(body);
/* replace with the new body */
int vpos = array_scan(v->names, v->len, fn->rp.name);
v->vals[vpos] = param->body;
param->body = NULL;
rel_free(param);
}
/* start a transaction */
r = vars_new(fn->r.len);
w = vars_new(fn->w.len);
for (int i = 0; i < fn->r.len; ++i)
vars_add(r, fn->r.names[i], 0, NULL);
for (int i = 0; i < fn->w.len; ++i)
vars_add(w, fn->w.names[i], 0, NULL);
sid = tx_enter(addr, r, w);
/* prepare variables */
for (int i = 0; i < r->len; ++i) {
TBuf *body = vol_read(r->vols[i], r->names[i], r->vers[i]);
vars_add(v, r->names[i], 0, body);
}
for (int i = 0; i < w->len; ++i) {
int pos = array_scan(v->names, v->len, w->names[i]);
if (pos < 0)
vars_add(v, w->names[i], 0, NULL);
}
for (int i = 0; i < fn->t.len; ++i)
vars_add(v, fn->t.names[i], 0, NULL);
/* evaluate the function body */
for (int i = 0; i < fn->slen; ++i)
rel_eval(fn->stmts[i], v, arg);
/* prepare the return value. note, the resulting relation
is just a container for the body, so it is not freed */
Rel *ret = NULL;
if (fn->ret != NULL)
ret = fn->stmts[fn->slen - 1];
/* persist the global variables */
for (int i = 0; i < w->len; ++i) {
int idx = array_scan(v->names, v->len, w->names[i]);
if (idx < 0) {
status = http_500(io);
goto exit;
}
vol_write(w->vols[i], v->vals[idx], w->names[i], w->vers[i]);
tbuf_free(v->vals[idx]);
v->vals[idx] = NULL;
}
/* confirm a success and send the result back */
status = http_200(io);
if (status != 200)
goto exit;
tx_commit(sid);
/* N.B. there is no explicit revert as the transaction manager handles
nested tx_enter and a connectivity failure as a rollback */
int len = 1, i = 0;
while (status == 200 && len) {
len = pack_rel2csv(ret, res, MAX_BLOCK, i++);
status = http_chunk(io, res, len);
}
exit:
if (status != -1)
sys_log('E', "%016llX method %c, path %s, time %lldms - %3d\n",
sid,
(req == NULL) ? '?' : req->method,
(req == NULL) ? "malformed" : req->path,
sys_millis() - time,
status);
if (r != NULL)
vars_free(r);
if (w != NULL)
vars_free(w);
if (arg != NULL)
mem_free(arg);
if (req != NULL)
http_free_req(req);
if (env != NULL)
env_free(env);
for (int i = 0; i < v->len; ++i)
if (v->vals[i] != NULL) {
tbuf_clean(v->vals[i]);
tbuf_free(v->vals[i]);
}
vars_free(v);
sys_term(io);
}
mem_free(code);
mem_free(res);
tx_detach();
sys_close(io);
}
token_t *error_new_syntactic(error_id_t id, gsize position) {
return error_new(error_info_new_syntactic(id, position));
}
token_t *error_new_lexical(error_id_t id, gchar *position) {
return error_new(error_info_new_lexical(id, position));
}