__hlt_memory_nullbuffer* __hlt_memory_nullbuffer_new()
{
__hlt_memory_nullbuffer* nbuf =
(__hlt_memory_nullbuffer*)hlt_malloc(sizeof(__hlt_memory_nullbuffer));
nbuf->used = 0;
nbuf->allocated = __INITIAL_NULLBUFFER_SIZE;
nbuf->flush_pos = -1;
nbuf->objs =
(struct __obj_with_rtti*)hlt_malloc(sizeof(struct __obj_with_rtti) * nbuf->allocated);
return nbuf;
}
__hlt_fiber_pool* __hlt_fiber_pool_new()
{
__hlt_fiber_pool* pool = hlt_malloc(sizeof(__hlt_fiber_pool));
pool->head = 0;
pool->size = 0;
return pool;
}
hlt_free_list* hlt_free_list_new(size_t size)
{
hlt_free_list* list = hlt_malloc(sizeof(hlt_free_list));
list->pool = 0;
list->size = sizeof(__hlt_free_list_block) + size;
return list;
}
hlt_execution_context* __hlt_execution_context_new_ref(hlt_vthread_id vid, int8_t run_module_init)
{
hlt_execution_context* ctx = (hlt_execution_context*)hlt_malloc(sizeof(hlt_execution_context) +
__hlt_globals()->globals_size);
ctx->vid = vid;
ctx->nullbuffer = __hlt_memory_nullbuffer_new(); // init first
ctx->excpt = 0;
ctx->fiber = 0;
ctx->fiber_pool = __hlt_fiber_pool_new();
ctx->worker = 0;
ctx->tcontext = 0;
ctx->tcontext_type = 0;
ctx->pstate = 0;
ctx->blockable = 0;
ctx->tmgr = hlt_timer_mgr_new(&ctx->excpt, ctx);
GC_CCTOR(ctx->tmgr, hlt_timer_mgr, ctx);
__hlt_globals_init(ctx);
if ( run_module_init )
__hlt_modules_init(ctx);
if ( ctx->excpt )
hlt_exception_print_uncaught_abort(ctx->excpt, ctx);
return ctx;
}
void hlt_regexp_compile_set(hlt_regexp* re, hlt_list* patterns, hlt_exception** excpt, hlt_execution_context* ctx)
{
if ( re->num != 0 ) {
hlt_set_exception(excpt, &hlt_exception_value_error, 0, ctx);
return;
}
re->num = hlt_list_size(patterns, excpt, ctx);
re->patterns = hlt_malloc(re->num * sizeof(hlt_string));
jrx_regset_init(&re->regexp, -1, _cflags(re->flags));
hlt_iterator_list i = hlt_list_begin(patterns, excpt, ctx);
hlt_iterator_list end = hlt_list_end(patterns, excpt, ctx);
int idx = 0;
while ( ! hlt_iterator_list_eq(i, end, excpt, ctx) ) {
hlt_string* pattern = hlt_iterator_list_deref(i, excpt, ctx);
_compile_one(re, *pattern, idx, 0, excpt, ctx);
if ( hlt_check_exception(excpt) )
return;
i = hlt_iterator_list_incr(i, excpt, ctx);
idx++;
}
jrx_regset_finalize(&re->regexp);
}
void hlt_classifier_add(hlt_classifier* c, hlt_classifier_field** fields, int64_t priority, const hlt_type_info* vtype, void* value, hlt_exception** excpt, hlt_execution_context* ctx)
{
if ( c->compiled ) {
hlt_set_exception(excpt, &hlt_exception_value_error, 0, ctx);
return;
}
hlt_classifier_rule* r = hlt_malloc(sizeof(hlt_classifier_rule));
r->priority = priority;
r->fields = fields;
r->value = _to_voidp(vtype, value);
GC_CCTOR_GENERIC(r->value, vtype, ctx);
if ( c->num_rules >= c->max_rules ) {
// Grow rule array.
int64_t old_max_rules = c->max_rules;
c->max_rules = (old_max_rules ? (int)(old_max_rules * 1.5) : 5);
c->rules = (hlt_classifier_rule**) hlt_realloc(c->rules, c->max_rules * sizeof(hlt_classifier_rule), old_max_rules * sizeof(hlt_classifier_rule));
}
c->rules[c->num_rules++] = r;
if ( priority > c->max_prio )
c->max_prio = priority;
#ifdef DEBUG
DBG_LOG("hilti-classifier", "%s: new rule %p with priority %d for classifier %p", "classifier_add", r, priority, c);
dbg_print_fields(c, "classifier_add", fields);
#endif
}
// Internal version that really creates a fiber (vs. the external version
// that might recycle a previously created on from a fiber pool). Note that
// this function does not intialize the "run" and "cookie" fields.
static hlt_fiber* __hlt_fiber_create(hlt_execution_context* ctx)
{
hlt_fiber* fiber = (hlt_fiber*) hlt_malloc(sizeof(hlt_fiber));
if ( getcontext(&fiber->uctx) < 0 ) {
fprintf(stderr, "getcontext failed in __hlt_fiber_create\n");
abort();
}
fiber->state = INIT;
fiber->run = 0;
fiber->cookie = 0;
fiber->context = ctx;
fiber->uctx.uc_link = 0;
fiber->uctx.uc_stack.ss_size = hlt_config_get()->fiber_stack_size;
fiber->uctx.uc_stack.ss_sp = hlt_stack_alloc(fiber->uctx.uc_stack.ss_size);
fiber->uctx.uc_stack.ss_flags = 0;
fiber->next = 0;
// Magic from from libtask/task.c to turn the pointer into two words.
unsigned long z = (unsigned long)fiber;
unsigned int y = z;
z >>= 16;
unsigned int x = (z >> 16);
makecontext(&fiber->uctx, (void (*)())_fiber_trampoline, 2, y, x);
return fiber;
}
void _binpachilti_sink_connect_intern(binpac_sink* sink, const hlt_type_info* type, void** pobj, binpac_parser* parser, hlt_bytes* mtype, hlt_exception** excpt, hlt_execution_context* ctx)
{
__parser_state* state = hlt_malloc(sizeof(__parser_state));
state->parser = parser;
GC_CCTOR(state->parser, hlt_BinPACHilti_Parser, ctx);
state->pobj = *pobj;
GC_CCTOR_GENERIC(&state->pobj, type, ctx);
state->data = 0;
state->resume = 0;
state->disconnected = 0;
state->next = sink->head;
sink->head = state;
#ifdef DEBUG
if ( mtype ) {
hlt_string s = hlt_string_decode(mtype, Hilti_Charset_ASCII, excpt, ctx);
char* r1 = hlt_string_to_native(s, excpt, ctx);
char* r2 = hlt_string_to_native(parser->name, excpt, ctx);
DBG_LOG("binpac-sinks", "connected parser %s [%p] to sink %p for MIME type %s", r2, *pobj, sink, r1);
hlt_free(r1);
hlt_free(r2);
}
else {
char* p = hlt_string_to_native(parser->name, excpt, ctx);
DBG_LOG("binpac-sinks", "connected parser %s [%p] to sink %p", p, *pobj, sink);
hlt_free(p);
}
#endif
}
static __chunk* __new_chunk(hlt_bytes* data, uint64_t rseq, uint64_t len, hlt_exception** excpt, hlt_execution_context* ctx)
{
__chunk* c = hlt_malloc(sizeof(__chunk));
c->next = 0;
c->prev = 0;
c->rseq = rseq;
c->rupper = rseq + len;
GC_ASSIGN(c->data, data, hlt_bytes, ctx);
return c;
}
static void __add_parser(hlt_bytes* mt, binpac_parser* parser, hlt_exception** excpt, hlt_execution_context* ctx)
{
__mime_parser* mp = GC_NEW_REF(__mime_parser, ctx);
GC_INIT(mp->parser, parser, hlt_BinPACHilti_Parser, ctx);
mp->next = 0;
// Deep-copy the pointers, the map needs that.
__mime_parser** cmp = hlt_malloc(sizeof(__mime_parser*));
hlt_bytes** cmt = hlt_malloc(sizeof(hlt_bytes*));
*cmt = mt;
*cmp = mp;
__mime_parser** current = hlt_map_get_default(__binpac_globals_get()->mime_types, &hlt_type_info_hlt_bytes, cmt, &hlt_type_info___mime_parser, 0, excpt, ctx);
if ( current ) {
mp->next = *current;
GC_CCTOR(mp->next, __mime_parser, ctx);
}
hlt_map_insert(__binpac_globals_get()->mime_types, &hlt_type_info_hlt_bytes, cmt, &hlt_type_info___mime_parser, cmp, excpt, ctx);
GC_DTOR(mp, __mime_parser, ctx);
hlt_free(cmp);
hlt_free(cmt);
#ifdef DEBUG
hlt_string s = hlt_string_decode(mt, Hilti_Charset_ASCII, excpt, ctx);
char* r1 = hlt_string_to_native(s, excpt, ctx);
char* r2 = hlt_string_to_native(parser->name, excpt, ctx);
DBG_LOG("binpac-sinks", "MIME type %s registered for parser %s", r1, r2);
hlt_free(r1);
hlt_free(r2);
#endif
}
void* hlt_free_list_alloc(hlt_free_list* list)
{
__hlt_free_list_block* b = 0;
if ( list->pool ) {
b = list->pool;
list->pool = list->pool->next;
bzero(b, list->size); // Make contents consistent.
}
else
b = (__hlt_free_list_block*)hlt_malloc(list->size);
return ((char*)b) + __data_offset;
}
void __hlt_memory_nullbuffer_flush(__hlt_memory_nullbuffer* nbuf, hlt_execution_context* ctx)
{
if ( nbuf->flush_pos >= 0 )
return;
#ifdef DEBUG
_dbg_mem_raw("nullbuffer_flush", nbuf, nbuf->used, 0, "start", 0, ctx);
#endif
// Note, flush_pos is examined during flushing by nullbuffer_add().
for ( nbuf->flush_pos = 0; nbuf->flush_pos < nbuf->used; ++nbuf->flush_pos ) {
struct __obj_with_rtti x = nbuf->objs[nbuf->flush_pos];
if ( ! x.obj )
// May have been removed.
continue;
#ifdef DEBUG
--__hlt_globals()->num_nullbuffer;
#endif
__hlt_gchdr* hdr = (__hlt_gchdr*)x.obj;
if ( hdr->ref_cnt > 0 )
// Still alive actually.
continue;
if ( x.ti->obj_dtor )
(*(x.ti->obj_dtor))(x.ti, x.obj, ctx);
__hlt_free(x.obj, x.ti->tag, "nullbuffer_flush");
}
nbuf->used = 0;
if ( nbuf->allocated > __INITIAL_NULLBUFFER_SIZE ) {
hlt_free(nbuf->objs);
nbuf->allocated = __INITIAL_NULLBUFFER_SIZE;
nbuf->objs =
(struct __obj_with_rtti*)hlt_malloc(sizeof(struct __obj_with_rtti) * nbuf->allocated);
}
#ifdef DEBUG
_dbg_mem_raw("nullbuffer_flush", nbuf, nbuf->used, 0, "end", 0, ctx);
#endif
nbuf->flush_pos = -1;
}
void hlt_regexp_compile(hlt_regexp* re, const hlt_string pattern, hlt_exception** excpt, hlt_execution_context* ctx)
{
if ( re->num != 0 ) {
hlt_set_exception(excpt, &hlt_exception_value_error, 0, ctx);
return;
}
re->num = 1;
re->patterns = hlt_malloc(sizeof(hlt_string));
jrx_regset_init(&re->regexp, -1, _cflags(re->flags));
_compile_one(re, pattern, 0, 0, excpt, ctx);
if ( hlt_check_exception(excpt) )
return;
jrx_regset_finalize(&re->regexp);
}
static void _hlt_regexp_new_from_regexp_init(hlt_regexp* dst, hlt_regexp* other, hlt_exception** excpt, hlt_execution_context* ctx)
{
dst->flags = other->flags;
dst->num = other->num;
dst->patterns = hlt_malloc(dst->num * sizeof(hlt_string));
jrx_regset_init(&dst->regexp, -1, _cflags(dst->flags));
for ( int idx = 0; idx < other->num; idx++ ) {
hlt_string pattern = other->patterns[idx];
_compile_one(dst, pattern, idx, 0, excpt, ctx);
if ( hlt_check_exception(excpt) )
return;
}
jrx_regset_finalize(&dst->regexp);
}
static inline void _hlt_exception_init(hlt_exception* excpt, hlt_exception_type* type, void* arg,
const char* location, hlt_execution_context* ctx)
{
excpt->type = type;
if ( arg ) {
excpt->arg = hlt_malloc(type->argtype->size);
memcpy(excpt->arg, &arg, type->argtype->size);
GC_CCTOR_GENERIC(excpt->arg, type->argtype, ctx);
}
else
arg = 0;
excpt->fiber = 0;
excpt->vid = HLT_VID_MAIN;
excpt->location = location;
}
void hlt_thread_queue_write(hlt_thread_queue* queue, int writer, void *elem)
{
if ( queue->lock_writers_terminated[writer] )
// Ignore when we have already terminated. We can read this without
// locking as we're the only thread ever going to write to it.
return;
do {
batch* b = queue->writer_batches[writer];
// If we don't have a batch, get us one.
if ( ! b ) {
b = (batch*) hlt_malloc(sizeof(batch) * queue->batch_size * sizeof(void*));
if ( ! b )
_fatal_error("out of memory");
b->write_pos = 0;
b->next = 0;
queue->writer_batches[writer] = b;
}
// If there isn't enough space in our batch, flush.
if ( b->write_pos >= queue->batch_size )
hlt_thread_queue_flush(queue, writer);
} while ( ! queue->writer_batches[writer] );
// Write the element.
batch* b = queue->writer_batches[writer];
assert(b && b->write_pos < queue->batch_size);
b->elems[b->write_pos++] = elem;
++queue->writer_num_written[writer];
++queue->writer_stats[writer].elems;
hlt_thread_queue_writer_update(queue, writer);
}
void hlt_regexp_clone_init(void* dstp, const hlt_type_info* ti, void* srcp, __hlt_clone_state* cstate, hlt_exception** excpt, hlt_execution_context* ctx)
{
hlt_regexp* src = *(hlt_regexp**)srcp;
hlt_regexp* dst = *(hlt_regexp**)dstp;
dst->num = src->num;
dst->flags = src->flags;
dst->patterns = hlt_malloc(src->num * sizeof(hlt_string));
for ( int i = 0; i < src->num; i++ )
__hlt_clone(&dst->patterns[i], &hlt_type_info_hlt_string, &src->patterns[i], cstate, excpt, ctx);
// TODO: Figure out a way to reuse the compiled regexp. Need to make that
// thread-safe though.
jrx_regset_init(&dst->regexp, -1, _cflags(dst->flags));
for ( int idx = 0; idx < dst->num; idx++ ) {
hlt_string pattern = dst->patterns[idx];
_compile_one(dst, pattern, idx, 1, excpt, ctx);
}
jrx_regset_finalize(&dst->regexp);
}
hlt_thread_queue* hlt_thread_queue_new(int writers, int batch_size, int max_batches)
{
hlt_thread_queue *queue = (hlt_thread_queue *) hlt_malloc(sizeof(hlt_thread_queue));
if ( ! queue )
_fatal_error("out of memory");
queue->writers = writers;
queue->batch_size = batch_size;
queue->max_batches = max_batches;
queue->reader_head = 0;
queue->reader_pos = 0;
queue->reader_num_read = 0;
queue->reader_num_terminated = 0;
queue->reader_stats = (hlt_thread_queue_stats*) hlt_malloc(sizeof(hlt_thread_queue_stats));
memset(queue->reader_stats, 0, sizeof(hlt_thread_queue_stats));
queue->writer_batches = (batch**) hlt_malloc(sizeof(batch*) * writers);
queue->writer_num_written = (uint64_t*) hlt_malloc(sizeof(uint64_t) * writers);
queue->writer_stats = (hlt_thread_queue_stats*) hlt_malloc(sizeof(hlt_thread_queue_stats) * writers);
queue->lock_num_pending = 0;
queue->lock_pending_head = 0;
queue->lock_pending_tail = 0;
queue->lock_block = 0;
queue->lock_writers_terminated = (int*) hlt_malloc(sizeof(int) * writers);
memset(queue->writer_stats, 0, sizeof(hlt_thread_queue_stats) * writers);
for ( int i = 0; i < writers; ++i ) {
queue->writer_batches[i] = 0;
queue->writer_num_written[i] = 0;
queue->lock_writers_terminated[i] = 0;
}
queue->need_flush = 0;
if ( PTHREAD_SPIN_INIT(&queue->lock) != 0 )
_fatal_error("cannot init lock");
return queue;
}
// Does not ref copied element.
static inline void* _to_voidp(const hlt_type_info* type, void* data)
{
void* z = hlt_malloc(type->size);
memcpy(z, data, type->size);
return z;
}
