void
TestBatchRunner_Destroy_IMP(TestBatchRunner *self) {
DECREF(self->formatter);
SUPER_DESTROY(self, TESTBATCHRUNNER);
}
Vector*
QueryLexer_Tokenize_IMP(QueryLexer *self, String *query_string) {
QueryLexerIVARS *const ivars = QueryLexer_IVARS(self);
Vector *elems = Vec_new(0);
if (!query_string) { return elems; }
StringIterator *iter = Str_Top(query_string);
while (StrIter_Has_Next(iter)) {
ParserElem *elem = NULL;
if (StrIter_Skip_Whitespace(iter)) {
// Fast-forward past whitespace.
continue;
}
if (ivars->heed_colons) {
ParserElem *elem = S_consume_field(iter);
if (elem) {
Vec_Push(elems, (Obj*)elem);
}
}
int32_t code_point = StrIter_Next(iter);
switch (code_point) {
case '(':
elem = ParserElem_new(TOKEN_OPEN_PAREN, NULL);
break;
case ')':
elem = ParserElem_new(TOKEN_CLOSE_PAREN, NULL);
break;
case '+':
if (StrIter_Has_Next(iter)
&& !StrIter_Skip_Whitespace(iter)
) {
elem = ParserElem_new(TOKEN_PLUS, NULL);
}
else {
elem = ParserElem_new(TOKEN_STRING, (Obj*)Str_newf("+"));
}
break;
case '-':
if (StrIter_Has_Next(iter)
&& !StrIter_Skip_Whitespace(iter)
) {
elem = ParserElem_new(TOKEN_MINUS, NULL);
}
else {
elem = ParserElem_new(TOKEN_STRING, (Obj*)Str_newf("-"));
}
break;
case '"':
StrIter_Recede(iter, 1);
elem = S_consume_quoted_string(iter);
break;
case 'O':
StrIter_Recede(iter, 1);
elem = S_consume_keyword(iter, "OR", 2, TOKEN_OR);
if (!elem) {
elem = S_consume_text(iter);
}
break;
case 'A':
StrIter_Recede(iter, 1);
elem = S_consume_keyword(iter, "AND", 3, TOKEN_AND);
if (!elem) {
elem = S_consume_text(iter);
}
break;
case 'N':
StrIter_Recede(iter, 1);
elem = S_consume_keyword(iter, "NOT", 3, TOKEN_NOT);
if (!elem) {
elem = S_consume_text(iter);
}
break;
default:
StrIter_Recede(iter, 1);
elem = S_consume_text(iter);
break;
}
Vec_Push(elems, (Obj*)elem);
}
DECREF(iter);
return elems;
}
void
DefLexReader_Destroy_IMP(DefaultLexiconReader *self) {
DefaultLexiconReaderIVARS *const ivars = DefLexReader_IVARS(self);
DECREF(ivars->lexicons);
SUPER_DESTROY(self, DEFAULTLEXICONREADER);
}
Class*
Class_singleton(String *class_name, Class *parent) {
if (Class_registry == NULL) {
Class_init_registry();
}
Class *singleton = (Class*)LFReg_fetch(Class_registry, class_name);
if (singleton == NULL) {
Vector *fresh_host_methods;
size_t num_fresh;
if (parent == NULL) {
String *parent_class = Class_find_parent_class(class_name);
if (parent_class == NULL) {
THROW(ERR, "Class '%o' doesn't descend from %o", class_name,
OBJ->name);
}
else {
parent = Class_singleton(parent_class, NULL);
DECREF(parent_class);
}
}
singleton = S_simple_subclass(parent, class_name);
// Allow host methods to override.
fresh_host_methods = Class_fresh_host_methods(class_name);
num_fresh = Vec_Get_Size(fresh_host_methods);
if (num_fresh) {
Hash *meths = Hash_new(num_fresh);
for (size_t i = 0; i < num_fresh; i++) {
String *meth = (String*)Vec_Fetch(fresh_host_methods, i);
Hash_Store(meths, meth, (Obj*)CFISH_TRUE);
}
for (Class *klass = parent; klass; klass = klass->parent) {
for (size_t i = 0; klass->methods[i]; i++) {
Method *method = klass->methods[i];
if (method->callback_func) {
String *name = Method_Host_Name(method);
if (Hash_Fetch(meths, name)) {
Class_Override(singleton, method->callback_func,
method->offset);
}
DECREF(name);
}
}
}
DECREF(meths);
}
DECREF(fresh_host_methods);
// Register the new class, both locally and with host.
if (Class_add_to_registry(singleton)) {
// Doing this after registering is racy, but hard to fix. :(
Class_register_with_host(singleton, parent);
}
else {
DECREF(singleton);
singleton = (Class*)LFReg_fetch(Class_registry, class_name);
if (!singleton) {
THROW(ERR, "Failed to either insert or fetch Class for '%o'",
class_name);
}
}
}
return singleton;
}
void
TestQPSyntax_run_tests(Folder *index)
{
uint32_t i;
TestBatch *batch = TestBatch_new(58);
IndexSearcher *searcher = IxSearcher_new((Obj*)index);
QueryParser *qparser = QParser_new(IxSearcher_Get_Schema(searcher),
NULL, NULL, NULL);
QParser_Set_Heed_Colons(qparser, true);
TestBatch_Plan(batch);
for (i = 0; leaf_test_funcs[i] != NULL; i++) {
kino_TestQPSyntax_test_t test_func = leaf_test_funcs[i];
TestQueryParser *test_case = test_func();
Query *tree = QParser_Tree(qparser, test_case->query_string);
Query *expanded = QParser_Expand_Leaf(qparser, test_case->tree);
Query *parsed = QParser_Parse(qparser, test_case->query_string);
Hits *hits = IxSearcher_Hits(searcher, (Obj*)parsed, 0, 10, NULL);
TEST_TRUE(batch, Query_Equals(tree, (Obj*)test_case->tree),
"tree() %s", (char*)CB_Get_Ptr8(test_case->query_string));
TEST_TRUE(batch, Query_Equals(expanded, (Obj*)test_case->expanded),
"expand_leaf() %s", (char*)CB_Get_Ptr8(test_case->query_string));
TEST_INT_EQ(batch, Hits_Total_Hits(hits), test_case->num_hits,
"hits: %s", (char*)CB_Get_Ptr8(test_case->query_string));
DECREF(hits);
DECREF(parsed);
DECREF(expanded);
DECREF(tree);
DECREF(test_case);
}
for (i = 0; syntax_test_funcs[i] != NULL; i++) {
kino_TestQPSyntax_test_t test_func = syntax_test_funcs[i];
TestQueryParser *test_case = test_func();
Query *tree = QParser_Tree(qparser, test_case->query_string);
Query *parsed = QParser_Parse(qparser, test_case->query_string);
Hits *hits = IxSearcher_Hits(searcher, (Obj*)parsed, 0, 10, NULL);
TEST_TRUE(batch, Query_Equals(tree, (Obj*)test_case->tree),
"tree() %s", (char*)CB_Get_Ptr8(test_case->query_string));
TEST_INT_EQ(batch, Hits_Total_Hits(hits), test_case->num_hits,
"hits: %s", (char*)CB_Get_Ptr8(test_case->query_string));
DECREF(hits);
DECREF(parsed);
DECREF(tree);
DECREF(test_case);
}
DECREF(batch);
DECREF(searcher);
DECREF(qparser);
}
Obj*
Freezer_deserialize(Obj *obj, InStream *instream) {
if (Obj_is_a(obj, STRING)) {
obj = (Obj*)Freezer_deserialize_string((String*)obj, instream);
}
else if (Obj_is_a(obj, BLOB)) {
obj = (Obj*)Freezer_deserialize_blob((Blob*)obj, instream);
}
else if (Obj_is_a(obj, VECTOR)) {
obj = (Obj*)Freezer_deserialize_varray((Vector*)obj, instream);
}
else if (Obj_is_a(obj, HASH)) {
obj = (Obj*)Freezer_deserialize_hash((Hash*)obj, instream);
}
else if (Obj_is_a(obj, INTEGER)) {
int64_t value = (int64_t)InStream_Read_C64(instream);
obj = (Obj*)Int_init((Integer*)obj, value);
}
else if (Obj_is_a(obj, FLOAT)) {
double value = InStream_Read_F64(instream);
obj = (Obj*)Float_init((Float*)obj, value);
}
else if (Obj_is_a(obj, BOOLEAN)) {
bool value = !!InStream_Read_U8(instream);
Obj *result = value ? INCREF(CFISH_TRUE) : INCREF(CFISH_FALSE);
// FIXME: This DECREF is essentially a no-op causing a
// memory leak.
DECREF(obj);
obj = result;
}
else if (Obj_is_a(obj, QUERY)) {
obj = (Obj*)Query_Deserialize((Query*)obj, instream);
}
else if (Obj_is_a(obj, DOC)) {
obj = (Obj*)Doc_Deserialize((Doc*)obj, instream);
}
else if (Obj_is_a(obj, DOCVECTOR)) {
obj = (Obj*)DocVec_Deserialize((DocVector*)obj, instream);
}
else if (Obj_is_a(obj, TERMVECTOR)) {
obj = (Obj*)TV_Deserialize((TermVector*)obj, instream);
}
else if (Obj_is_a(obj, SIMILARITY)) {
obj = (Obj*)Sim_Deserialize((Similarity*)obj, instream);
}
else if (Obj_is_a(obj, MATCHDOC)) {
obj = (Obj*)MatchDoc_Deserialize((MatchDoc*)obj, instream);
}
else if (Obj_is_a(obj, TOPDOCS)) {
obj = (Obj*)TopDocs_Deserialize((TopDocs*)obj, instream);
}
else if (Obj_is_a(obj, SORTSPEC)) {
obj = (Obj*)SortSpec_Deserialize((SortSpec*)obj, instream);
}
else if (Obj_is_a(obj, SORTRULE)) {
obj = (Obj*)SortRule_Deserialize((SortRule*)obj, instream);
}
else {
THROW(ERR, "Don't know how to deserialize a %o",
Obj_get_class_name(obj));
}
return obj;
}
void
StrIter_Destroy_IMP(StringIterator *self) {
DECREF(self->string);
SUPER_DESTROY(self, STRINGITERATOR);
}
void
BlobSortEx_Destroy_IMP(BlobSortEx *self) {
BlobSortExIVARS *const ivars = BlobSortEx_IVARS(self);
DECREF(ivars->external);
SUPER_DESTROY(self, BLOBSORTEX);
}
void
TestQPLogic_Run_IMP(TestQueryParserLogic *self, TestBatchRunner *runner) {
TestBatchRunner_Plan(runner, (TestBatch*)self, 258);
uint32_t i;
Folder *folder = S_create_index();
IndexSearcher *searcher = IxSearcher_new((Obj*)folder);
QueryParser *or_parser = QParser_new(IxSearcher_Get_Schema(searcher),
NULL, NULL, NULL);
String *AND = SSTR_WRAP_C("AND");
QueryParser *and_parser = QParser_new(IxSearcher_Get_Schema(searcher),
NULL, AND, NULL);
QParser_Set_Heed_Colons(or_parser, true);
QParser_Set_Heed_Colons(and_parser, true);
// Run logical tests with default boolop of OR.
for (i = 0; logical_test_funcs[i] != NULL; i++) {
LUCY_TestQPLogic_Logical_Test_t test_func = logical_test_funcs[i];
TestQueryParser *test_case_obj = test_func(BOOLOP_OR);
TestQueryParserIVARS *test_case = TestQP_IVARS(test_case_obj);
Query *tree = QParser_Tree(or_parser, test_case->query_string);
Query *parsed = QParser_Parse(or_parser, test_case->query_string);
Hits *hits = IxSearcher_Hits(searcher, (Obj*)parsed, 0, 10, NULL);
char *qstr = Str_To_Utf8(test_case->query_string);
TEST_TRUE(runner, Query_Equals(tree, (Obj*)test_case->tree),
"tree() OR %s", qstr);
TEST_INT_EQ(runner, Hits_Total_Hits(hits), test_case->num_hits,
"hits: OR %s", qstr);
free(qstr);
DECREF(hits);
DECREF(parsed);
DECREF(tree);
DECREF(test_case_obj);
}
// Run logical tests with default boolop of AND.
for (i = 0; logical_test_funcs[i] != NULL; i++) {
LUCY_TestQPLogic_Logical_Test_t test_func = logical_test_funcs[i];
TestQueryParser *test_case_obj = test_func(BOOLOP_AND);
TestQueryParserIVARS *test_case = TestQP_IVARS(test_case_obj);
Query *tree = QParser_Tree(and_parser, test_case->query_string);
Query *parsed = QParser_Parse(and_parser, test_case->query_string);
Hits *hits = IxSearcher_Hits(searcher, (Obj*)parsed, 0, 10, NULL);
char *qstr = Str_To_Utf8(test_case->query_string);
TEST_TRUE(runner, Query_Equals(tree, (Obj*)test_case->tree),
"tree() AND %s", qstr);
TEST_INT_EQ(runner, Hits_Total_Hits(hits), test_case->num_hits,
"hits: AND %s", qstr);
free(qstr);
DECREF(hits);
DECREF(parsed);
DECREF(tree);
DECREF(test_case_obj);
}
// Run tests for QParser_Prune().
for (i = 0; prune_test_funcs[i] != NULL; i++) {
LUCY_TestQPLogic_Prune_Test_t test_func = prune_test_funcs[i];
TestQueryParser *test_case_obj = test_func();
TestQueryParserIVARS *test_case = TestQP_IVARS(test_case_obj);
String *qstring = test_case->tree
? Query_To_String(test_case->tree)
: Str_new_from_trusted_utf8("(NULL)", 6);
char *qstr = Str_To_Utf8(qstring);
Query *tree = test_case->tree;
Query *wanted = test_case->expanded;
Query *pruned = QParser_Prune(or_parser, tree);
Query *expanded;
Hits *hits;
TEST_TRUE(runner, Query_Equals(pruned, (Obj*)wanted),
"prune() %s", qstr);
expanded = QParser_Expand(or_parser, pruned);
hits = IxSearcher_Hits(searcher, (Obj*)expanded, 0, 10, NULL);
TEST_INT_EQ(runner, Hits_Total_Hits(hits), test_case->num_hits,
"hits: %s", qstr);
free(qstr);
DECREF(hits);
DECREF(expanded);
DECREF(pruned);
DECREF(qstring);
DECREF(test_case_obj);
}
DECREF(and_parser);
DECREF(or_parser);
DECREF(searcher);
DECREF(folder);
}
void
Err_warn_mess(CharBuf *message) {
fprintf(stderr, "%s", CB_Get_Ptr8(message));
DECREF(message);
}
void
RAMFileDes_destroy(RAMFileDes *self)
{
DECREF(self->buffers);
SUPER_DESTROY(self, RAMFILEDES);
}
void
SegWriter_Set_Del_Writer_IMP(SegWriter *self, DeletionsWriter *del_writer) {
SegWriterIVARS *const ivars = SegWriter_IVARS(self);
DECREF(ivars->del_writer);
ivars->del_writer = (DeletionsWriter*)INCREF(del_writer);
}
void
HeatMap_Destroy_IMP(HeatMap *self) {
HeatMapIVARS *const ivars = HeatMap_IVARS(self);
DECREF(ivars->spans);
SUPER_DESTROY(self, HEATMAP);
}
void
SnowStop_Destroy_IMP(SnowballStopFilter *self) {
SnowballStopFilterIVARS *const ivars = SnowStop_IVARS(self);
DECREF(ivars->stoplist);
SUPER_DESTROY(self, SNOWBALLSTOPFILTER);
}
static void _varintDestructor(void * privdata, void * value)
{
DICT_NOTUSED(privdata);
DECREF((varint_t*)value);
}
VArray*
PhraseCompiler_highlight_spans(PhraseCompiler *self, Searcher *searcher,
DocVector *doc_vec, const CharBuf *field)
{
PhraseQuery *const parent = (PhraseQuery*)self->parent;
VArray *const terms = parent->terms;
VArray *const spans = VA_new(0);
VArray *term_vectors;
BitVector *posit_vec;
BitVector *other_posit_vec;
uint32_t i;
const uint32_t num_terms = VA_Get_Size(terms);
uint32_t num_tvs;
UNUSED_VAR(searcher);
// Bail if no terms or field doesn't match.
if (!num_terms) { return spans; }
if (!CB_Equals(field, (Obj*)parent->field)) { return spans; }
term_vectors = VA_new(num_terms);
posit_vec = BitVec_new(0);
other_posit_vec = BitVec_new(0);
for (i = 0; i < num_terms; i++) {
Obj *term = VA_Fetch(terms, i);
TermVector *term_vector
= DocVec_Term_Vector(doc_vec, field, (CharBuf*)term);
// Bail if any term is missing.
if (!term_vector)
break;
VA_Push(term_vectors, (Obj*)term_vector);
if (i == 0) {
// Set initial positions from first term.
uint32_t j;
I32Array *positions = TV_Get_Positions(term_vector);
for (j = I32Arr_Get_Size(positions); j > 0; j--) {
BitVec_Set(posit_vec, I32Arr_Get(positions, j - 1));
}
}
else {
// Filter positions using logical "and".
uint32_t j;
I32Array *positions = TV_Get_Positions(term_vector);
BitVec_Clear_All(other_posit_vec);
for (j = I32Arr_Get_Size(positions); j > 0; j--) {
int32_t pos = I32Arr_Get(positions, j - 1) - i;
if (pos >= 0) {
BitVec_Set(other_posit_vec, pos);
}
}
BitVec_And(posit_vec, other_posit_vec);
}
}
// Proceed only if all terms are present.
num_tvs = VA_Get_Size(term_vectors);
if (num_tvs == num_terms) {
TermVector *first_tv = (TermVector*)VA_Fetch(term_vectors, 0);
TermVector *last_tv
= (TermVector*)VA_Fetch(term_vectors, num_tvs - 1);
I32Array *tv_start_positions = TV_Get_Positions(first_tv);
I32Array *tv_end_positions = TV_Get_Positions(last_tv);
I32Array *tv_start_offsets = TV_Get_Start_Offsets(first_tv);
I32Array *tv_end_offsets = TV_Get_End_Offsets(last_tv);
uint32_t terms_max = num_terms - 1;
I32Array *valid_posits = BitVec_To_Array(posit_vec);
uint32_t num_valid_posits = I32Arr_Get_Size(valid_posits);
uint32_t j = 0;
uint32_t posit_tick;
float weight = PhraseCompiler_Get_Weight(self);
i = 0;
// Add only those starts/ends that belong to a valid position.
for (posit_tick = 0; posit_tick < num_valid_posits; posit_tick++) {
int32_t valid_start_posit = I32Arr_Get(valid_posits, posit_tick);
int32_t valid_end_posit = valid_start_posit + terms_max;
int32_t start_offset = 0, end_offset = 0;
uint32_t max;
for (max = I32Arr_Get_Size(tv_start_positions); i < max; i++) {
if (I32Arr_Get(tv_start_positions, i) == valid_start_posit) {
start_offset = I32Arr_Get(tv_start_offsets, i);
break;
}
}
for (max = I32Arr_Get_Size(tv_end_positions); j < max; j++) {
if (I32Arr_Get(tv_end_positions, j) == valid_end_posit) {
end_offset = I32Arr_Get(tv_end_offsets, j);
break;
}
}
VA_Push(spans, (Obj*)Span_new(start_offset,
end_offset - start_offset, weight) );
i++, j++;
}
DECREF(valid_posits);
}
DECREF(other_posit_vec);
DECREF(posit_vec);
DECREF(term_vectors);
return spans;
}
void
MatchPost_destroy(MatchPosting *self)
{
DECREF(self->sim);
FREE_OBJ(self);
}
void
Folder_set_path(Folder *self, const CharBuf *path) {
FolderIVARS *const ivars = Folder_IVARS(self);
DECREF(ivars->path);
ivars->path = CB_Clone(path);
}
static SortCache*
S_lazy_init_sort_cache(DefaultSortReader *self, String *field) {
DefaultSortReaderIVARS *const ivars = DefSortReader_IVARS(self);
// See if we have any values.
Obj *count_obj = Hash_Fetch(ivars->counts, (Obj*)field);
int32_t count = count_obj ? (int32_t)Obj_To_I64(count_obj) : 0;
if (!count) { return NULL; }
// Get a FieldType and sanity check that the field is sortable.
Schema *schema = DefSortReader_Get_Schema(self);
FieldType *type = Schema_Fetch_Type(schema, field);
if (!type || !FType_Sortable(type)) {
THROW(ERR, "'%o' isn't a sortable field", field);
}
// Open streams.
Folder *folder = DefSortReader_Get_Folder(self);
Segment *segment = DefSortReader_Get_Segment(self);
String *seg_name = Seg_Get_Name(segment);
int32_t field_num = Seg_Field_Num(segment, field);
int8_t prim_id = FType_Primitive_ID(type);
bool var_width = (prim_id == FType_TEXT || prim_id == FType_BLOB)
? true
: false;
String *ord_path = Str_newf("%o/sort-%i32.ord", seg_name, field_num);
InStream *ord_in = Folder_Open_In(folder, ord_path);
DECREF(ord_path);
if (!ord_in) {
THROW(ERR, "Error building sort cache for '%o': %o",
field, Err_get_error());
}
InStream *ix_in = NULL;
if (var_width) {
String *ix_path = Str_newf("%o/sort-%i32.ix", seg_name, field_num);
ix_in = Folder_Open_In(folder, ix_path);
DECREF(ix_path);
if (!ix_in) {
THROW(ERR, "Error building sort cache for '%o': %o",
field, Err_get_error());
}
}
String *dat_path = Str_newf("%o/sort-%i32.dat", seg_name, field_num);
InStream *dat_in = Folder_Open_In(folder, dat_path);
DECREF(dat_path);
if (!dat_in) {
THROW(ERR, "Error building sort cache for '%o': %o",
field, Err_get_error());
}
Obj *null_ord_obj = Hash_Fetch(ivars->null_ords, (Obj*)field);
int32_t null_ord = null_ord_obj ? (int32_t)Obj_To_I64(null_ord_obj) : -1;
Obj *ord_width_obj = Hash_Fetch(ivars->ord_widths, (Obj*)field);
int32_t ord_width = ord_width_obj
? (int32_t)Obj_To_I64(ord_width_obj)
: S_calc_ord_width(count);
int32_t doc_max = (int32_t)Seg_Get_Count(segment);
SortCache *cache = NULL;
switch (prim_id & FType_PRIMITIVE_ID_MASK) {
case FType_TEXT:
cache = (SortCache*)TextSortCache_new(field, type, count, doc_max,
null_ord, ord_width, ord_in,
ix_in, dat_in);
break;
case FType_INT32:
cache = (SortCache*)I32SortCache_new(field, type, count, doc_max,
null_ord, ord_width, ord_in,
dat_in);
break;
case FType_INT64:
cache = (SortCache*)I64SortCache_new(field, type, count, doc_max,
null_ord, ord_width, ord_in,
dat_in);
break;
case FType_FLOAT32:
cache = (SortCache*)F32SortCache_new(field, type, count, doc_max,
null_ord, ord_width, ord_in,
dat_in);
break;
case FType_FLOAT64:
cache = (SortCache*)F64SortCache_new(field, type, count, doc_max,
null_ord, ord_width, ord_in,
dat_in);
break;
default:
THROW(ERR, "No SortCache class for %o", type);
}
Hash_Store(ivars->caches, (Obj*)field, (Obj*)cache);
if (ivars->format == 2) { // bug compatibility
SortCache_Set_Native_Ords(cache, true);
}
DECREF(ord_in);
DECREF(ix_in);
DECREF(dat_in);
return cache;
}
/*
* code for the beginning of a function; a is an array of
* indices in symtab for the arguments; n is the number
* On m16k, space is allocated on stack for register arguments,
* arguments are moved to the stack and symtab is updated accordingly.
*/
void
bfcode(struct symtab **a, int n)
{
struct symtab *s;
int i, r0l, r0h, a0, r2, sz, hasch, stk;
int argoff = ARGINIT;
if (cftnsp->stype == STRTY+FTN || cftnsp->stype == UNIONTY+FTN) {
/* Function returns struct, adjust arg offset */
for (i = 0; i < n; i++)
a[i]->soffset += SZPOINT(INT);
}
/* first check if there are 1-byte parameters */
for (hasch = i = 0; i < n && i < 6; i++)
if (DEUNSIGN(a[i]->stype) == CHAR)
hasch = 1;
stk = r0l = r0h = a0 = r2 = 0;
for (i = 0; i < n; i++) {
s = a[i];
sz = tsize(s->stype, s->sdf, s->ssue);
if (ISPTR(s->stype) && ISFTN(DECREF(s->stype)))
sz = SZLONG; /* function pointers are always 32 */
if (stk == 0)
switch (sz) {
case SZCHAR:
if (r0l) {
if (r0h)
break;
argmove(s, 1);
r0h = 1;
} else {
argmove(s, 0);
r0l = 1;
}
continue;
case SZINT:
if (s->stype > BTMASK) {
/* is a pointer */
if (a0) {
if (r0l || hasch) {
if (r2)
break;
argmove(s, R2);
r2 = 1;
} else {
argmove(s, R0);
r0l = r0h = 1;
}
} else {
argmove(s, A0);
a0 = 1;
}
} else if (r0l || hasch) {
if (r2) {
if (a0)
break;
argmove(s, A0);
a0 = 1;
} else {
argmove(s, R2);
r2 = 1;
}
} else {
argmove(s, R0);
r0l = r0h = 1;
}
continue;
case SZLONG:
if (r0l||r0h||r2)
break;
argmove(s, R0);
r0l = r0h = r2 = 1;
continue;
default:
break;
}
stk = 1;
s->soffset = argoff;
argoff += sz;
}
}
FSFileHandle*
FSFH_do_open(FSFileHandle *self, const CharBuf *path, uint32_t flags)
{
FH_do_open((FileHandle*)self, path, flags);
if (!path || !CB_Get_Size(path)) {
Err_set_error(Err_new(CB_newf("Missing required param 'path'")));
DECREF(self);
return NULL;
}
// Attempt to open file.
if (flags & FH_WRITE_ONLY) {
self->fd = open((char*)CB_Get_Ptr8(path), SI_posix_flags(flags), 0666);
if (self->fd == -1) {
self->fd = 0;
Err_set_error(Err_new(CB_newf("Attempt to open '%o' failed: %s",
path, strerror(errno))));
DECREF(self);
return NULL;
}
if (flags & FH_EXCLUSIVE) {
self->len = 0;
}
else {
// Derive length.
self->len = lseek64(self->fd, I64_C(0), SEEK_END);
if (self->len == -1) {
Err_set_error(Err_new(CB_newf("lseek64 on %o failed: %s",
self->path, strerror(errno))));
DECREF(self);
return NULL;
}
else {
int64_t check_val = lseek64(self->fd, I64_C(0), SEEK_SET);
if (check_val == -1) {
Err_set_error(Err_new(CB_newf("lseek64 on %o failed: %s",
self->path, strerror(errno))));
DECREF(self);
return NULL;
}
}
}
}
else if (flags & FH_READ_ONLY) {
if (SI_init_read_only(self)) {
// On 64-bit systems, map the whole file up-front.
if (IS_64_BIT && self->len) {
self->buf = (char*)SI_map(self, 0, self->len);
if (!self->buf) {
// An error occurred during SI_map, which has set
// Err_error for us already.
DECREF(self);
return NULL;
}
}
}
else {
DECREF(self);
return NULL;
}
}
else {
Err_set_error(Err_new(CB_newf(
"Must specify FH_READ_ONLY or FH_WRITE_ONLY to open '%o'",
path)));
DECREF(self);
return NULL;
}
return self;
}
/*
* code for the beginning of a function; a is an array of
* indices in symtab for the arguments; n is the number
*/
void
bfcode(struct symtab **s, int cnt)
{
union arglist *al;
struct symtab *sp;
NODE *p, *r;
TWORD t;
int i, rno, typ;
/* recalculate the arg offset and create TEMP moves */
/* Always do this for reg, even if not optimizing, to free arg regs */
nsse = ngpr = 0;
nrsp = ARGINIT;
if (cftnsp->stype == STRTY+FTN || cftnsp->stype == UNIONTY+FTN) {
sp = cftnsp;
if (argtyp(DECREF(sp->stype), sp->sdf, sp->sap) == STRMEM) {
r = block(REG, NIL, NIL, LONG, 0, MKAP(LONG));
regno(r) = argregsi[ngpr++];
p = tempnode(0, r->n_type, r->n_df, r->n_ap);
stroffset = regno(p);
ecomp(buildtree(ASSIGN, p, r));
}
}
for (i = 0; i < cnt; i++) {
sp = s[i];
if (sp == NULL)
continue; /* XXX when happens this? */
switch (typ = argtyp(sp->stype, sp->sdf, sp->sap)) {
case INTEGER:
case SSE:
if (typ == SSE)
rno = XMM0 + nsse++;
else
rno = argregsi[ngpr++];
r = block(REG, NIL, NIL, sp->stype, sp->sdf, sp->sap);
regno(r) = rno;
p = tempnode(0, sp->stype, sp->sdf, sp->sap);
sp->soffset = regno(p);
sp->sflags |= STNODE;
ecomp(buildtree(ASSIGN, p, r));
break;
case SSEMEM:
sp->soffset = nrsp;
nrsp += SZDOUBLE;
if (xtemps) {
p = tempnode(0, sp->stype, sp->sdf, sp->sap);
p = buildtree(ASSIGN, p, nametree(sp));
sp->soffset = regno(p->n_left);
sp->sflags |= STNODE;
ecomp(p);
}
break;
case INTMEM:
sp->soffset = nrsp;
nrsp += SZLONG;
if (xtemps) {
p = tempnode(0, sp->stype, sp->sdf, sp->sap);
p = buildtree(ASSIGN, p, nametree(sp));
sp->soffset = regno(p->n_left);
sp->sflags |= STNODE;
ecomp(p);
}
break;
case STRMEM: /* Struct in memory */
sp->soffset = nrsp;
nrsp += tsize(sp->stype, sp->sdf, sp->sap);
break;
case X87: /* long double args */
sp->soffset = nrsp;
nrsp += SZLDOUBLE;
break;
case STRCPX:
case STRREG: /* Struct in register */
/* Allocate space on stack for the struct */
/* For simplicity always fetch two longwords */
autooff += (2*SZLONG);
if (typ == STRCPX) {
t = DOUBLE;
rno = XMM0 + nsse++;
} else {
t = LONG;
rno = argregsi[ngpr++];
}
r = block(REG, NIL, NIL, t, 0, MKAP(t));
regno(r) = rno;
ecomp(movtomem(r, -autooff, FPREG));
if (tsize(sp->stype, sp->sdf, sp->sap) > SZLONG) {
r = block(REG, NIL, NIL, t, 0, MKAP(t));
regno(r) = (typ == STRCPX ?
XMM0 + nsse++ : argregsi[ngpr++]);
ecomp(movtomem(r, -autooff+SZLONG, FPREG));
}
sp->soffset = -autooff;
break;
default:
cerror("bfcode: %d", typ);
}
}
/* Check if there are varargs */
if (cftnsp->sdf == NULL || cftnsp->sdf->dfun == NULL)
return; /* no prototype */
al = cftnsp->sdf->dfun;
for (; al->type != TELLIPSIS; al++) {
t = al->type;
if (t == TNULL)
return;
if (BTYPE(t) == STRTY || BTYPE(t) == UNIONTY)
al++;
for (; t > BTMASK; t = DECREF(t))
if (ISARY(t) || ISFTN(t))
al++;
}
/* fix stack offset */
SETOFF(autooff, ALMAX);
/* Save reg arguments in the reg save area */
p = NIL;
for (i = ngpr; i < 6; i++) {
r = block(REG, NIL, NIL, LONG, 0, MKAP(LONG));
regno(r) = argregsi[i];
r = movtomem(r, -RSALONGOFF(i)-autooff, FPREG);
p = (p == NIL ? r : block(COMOP, p, r, INT, 0, MKAP(INT)));
}
for (i = nsse; i < 8; i++) {
r = block(REG, NIL, NIL, DOUBLE, 0, MKAP(DOUBLE));
regno(r) = i + XMM0;
r = movtomem(r, -RSADBLOFF(i)-autooff, FPREG);
p = (p == NIL ? r : block(COMOP, p, r, INT, 0, MKAP(INT)));
}
autooff += RSASZ;
rsaoff = autooff;
thissse = nsse;
thisgpr = ngpr;
thisrsp = nrsp;
ecomp(p);
}
void
Class_bootstrap(const cfish_ParcelSpec *parcel_spec) {
const ClassSpec *specs = parcel_spec->class_specs;
const NovelMethSpec *novel_specs = parcel_spec->novel_specs;
const OverriddenMethSpec *overridden_specs = parcel_spec->overridden_specs;
const InheritedMethSpec *inherited_specs = parcel_spec->inherited_specs;
uint32_t num_classes = parcel_spec->num_classes;
/* Pass 1:
* - Allocate memory.
* - Initialize global Class pointers.
*/
for (uint32_t i = 0; i < num_classes; ++i) {
const ClassSpec *spec = &specs[i];
Class *parent = NULL;
if (spec->parent) {
parent = *spec->parent;
if (!parent) {
// Wrong order of class specs or inheritance cycle.
fprintf(stderr, "Parent class of '%s' not initialized\n",
spec->name);
abort();
}
}
uint32_t novel_offset = parent
? parent->class_alloc_size
: offsetof(Class, vtable);
uint32_t class_alloc_size = novel_offset
+ spec->num_novel_meths
* sizeof(cfish_method_t);
Class *klass = (Class*)CALLOCATE(class_alloc_size, 1);
// Needed to calculate size of subclasses.
klass->class_alloc_size = class_alloc_size;
// Initialize the global pointer to the Class.
if (!Atomic_cas_ptr((void**)spec->klass, NULL, klass)) {
// Another thread beat us to it.
FREEMEM(klass);
}
}
/* Pass 2:
* - Initialize IVARS_OFFSET.
* - Initialize 'klass' ivar and refcount by calling Init_Obj.
* - Initialize parent, flags, obj_alloc_size, class_alloc_size.
* - Assign parcel_spec.
* - Initialize method pointers and offsets.
*/
uint32_t num_novel = 0;
uint32_t num_overridden = 0;
uint32_t num_inherited = 0;
for (uint32_t i = 0; i < num_classes; ++i) {
const ClassSpec *spec = &specs[i];
Class *klass = *spec->klass;
Class *parent = spec->parent ? *spec->parent : NULL;
uint32_t ivars_offset = 0;
if (spec->ivars_offset_ptr != NULL) {
if (parent) {
Class *ancestor = parent;
while (ancestor && ancestor->parcel_spec == parcel_spec) {
ancestor = ancestor->parent;
}
ivars_offset = ancestor ? ancestor->obj_alloc_size : 0;
*spec->ivars_offset_ptr = ivars_offset;
}
else {
*spec->ivars_offset_ptr = 0;
}
}
// CLASS->obj_alloc_size is always 0, so Init_Obj doesn't clear any
// values set in the previous pass or by another thread.
Class_Init_Obj_IMP(CLASS, klass);
klass->parent = parent;
klass->parcel_spec = parcel_spec;
// CLASS->obj_alloc_size must stay at 0.
if (klass != CLASS) {
klass->obj_alloc_size = ivars_offset + spec->ivars_size;
}
if (cfish_Class_bootstrap_hook1 != NULL) {
cfish_Class_bootstrap_hook1(klass);
}
klass->flags = 0;
if (klass == CLASS
|| klass == METHOD
|| klass == BOOLEAN
|| klass == STRING
) {
klass->flags |= CFISH_fREFCOUNTSPECIAL;
}
if (spec->flags & cfish_ClassSpec_FINAL) {
klass->flags |= CFISH_fFINAL;
}
if (parent) {
// Copy parent vtable.
uint32_t parent_vt_size = parent->class_alloc_size
- offsetof(Class, vtable);
memcpy(klass->vtable, parent->vtable, parent_vt_size);
}
for (size_t i = 0; i < spec->num_inherited_meths; ++i) {
const InheritedMethSpec *mspec = &inherited_specs[num_inherited++];
*mspec->offset = *mspec->parent_offset;
}
for (size_t i = 0; i < spec->num_overridden_meths; ++i) {
const OverriddenMethSpec *mspec
= &overridden_specs[num_overridden++];
*mspec->offset = *mspec->parent_offset;
Class_Override_IMP(klass, mspec->func, *mspec->offset);
}
uint32_t novel_offset = parent
? parent->class_alloc_size
: offsetof(Class, vtable);
for (size_t i = 0; i < spec->num_novel_meths; ++i) {
const NovelMethSpec *mspec = &novel_specs[num_novel++];
*mspec->offset = novel_offset;
novel_offset += sizeof(cfish_method_t);
Class_Override_IMP(klass, mspec->func, *mspec->offset);
}
}
/* Now it's safe to call methods.
*
* Pass 3:
* - Inititalize name and method array.
* - Register class.
*/
num_novel = 0;
num_overridden = 0;
num_inherited = 0;
for (uint32_t i = 0; i < num_classes; ++i) {
const ClassSpec *spec = &specs[i];
Class *klass = *spec->klass;
String *name_internal
= Str_new_from_trusted_utf8(spec->name, strlen(spec->name));
if (!Atomic_cas_ptr((void**)&klass->name_internal, NULL,
name_internal)
) {
DECREF(name_internal);
name_internal = klass->name_internal;
}
String *name = Str_new_wrap_trusted_utf8(Str_Get_Ptr8(name_internal),
Str_Get_Size(name_internal));
if (!Atomic_cas_ptr((void**)&klass->name, NULL, name)) {
DECREF(name);
name = klass->name;
}
Method **methods = (Method**)MALLOCATE((spec->num_novel_meths + 1)
* sizeof(Method*));
// Only store novel methods for now.
for (size_t i = 0; i < spec->num_novel_meths; ++i) {
const NovelMethSpec *mspec = &novel_specs[num_novel++];
String *name = SSTR_WRAP_C(mspec->name);
Method *method = Method_new(name, mspec->callback_func,
*mspec->offset);
methods[i] = method;
}
methods[spec->num_novel_meths] = NULL;
if (!Atomic_cas_ptr((void**)&klass->methods, NULL, methods)) {
// Another thread beat us to it.
for (size_t i = 0; i < spec->num_novel_meths; ++i) {
Method_Destroy(methods[i]);
}
FREEMEM(methods);
}
Class_add_to_registry(klass);
}
}
/*
* Define everything needed to print out some data (or text).
* This means segment, alignment, visibility, etc.
*/
void
defloc(struct symtab *sp)
{
extern char *nextsect;
static char *loctbl[] = { "text", "data", "section .rodata" };
extern int tbss;
char *name;
TWORD t;
int s;
if (sp == NULL) {
lastloc = -1;
return;
}
if (kflag) {
#ifdef MACHOABI
loctbl[DATA] = "section .data.rel.rw,\"aw\"";
loctbl[RDATA] = "section .data.rel.ro,\"aw\"";
#else
loctbl[DATA] = "section .data.rel.rw,\"aw\",@progbits";
loctbl[RDATA] = "section .data.rel.ro,\"aw\",@progbits";
#endif
}
t = sp->stype;
s = ISFTN(t) ? PROG : ISCON(cqual(t, sp->squal)) ? RDATA : DATA;
if ((name = sp->soname) == NULL)
name = exname(sp->sname);
if (sp->sflags & STLS) {
if (s != DATA)
cerror("non-data symbol in tls section");
if (tbss)
nextsect = ".tbss,\"awT\",@nobits";
else
nextsect = ".tdata,\"awT\",@progbits";
tbss = 0;
lastloc = -1;
}
varattrib(name, sp->sap);
if (nextsect) {
printf(" .section %s\n", nextsect);
nextsect = NULL;
s = -1;
} else if (s != lastloc)
printf(" .%s\n", loctbl[s]);
lastloc = s;
while (ISARY(t))
t = DECREF(t);
s = ISFTN(t) ? ALINT : talign(t, sp->sap);
if (s > ALCHAR)
printf(" .align %d\n", s/ALCHAR);
if (sp->sclass == EXTDEF) {
printf("\t.globl %s\n", name);
#ifndef MACHOABI
printf("\t.type %s,@%s\n", name,
ISFTN(t)? "function" : "object");
#endif
}
if (sp->slevel == 0)
printf("%s:\n", name);
else
printf(LABFMT ":\n", sp->soffset);
}
void
FullTextType_destroy(FullTextType *self) {
DECREF(self->analyzer);
SUPER_DESTROY(self, FULLTEXTTYPE);
}
void
BBSortEx_destroy(BBSortEx *self) {
DECREF(self->external);
SUPER_DESTROY(self, BBSORTEX);
}
void
DefLexReader_Close_IMP(DefaultLexiconReader *self) {
DefaultLexiconReaderIVARS *const ivars = DefLexReader_IVARS(self);
DECREF(ivars->lexicons);
ivars->lexicons = NULL;
}
void
PolyAnalyzer_Destroy_IMP(PolyAnalyzer *self) {
PolyAnalyzerIVARS *const ivars = PolyAnalyzer_IVARS(self);
DECREF(ivars->analyzers);
SUPER_DESTROY(self, POLYANALYZER);
}
void
CaseFolder_Destroy_IMP(CaseFolder *self) {
CaseFolderIVARS *const ivars = CaseFolder_IVARS(self);
DECREF(ivars->normalizer);
SUPER_DESTROY(self, CASEFOLDER);
}
static void
test_flatten_spans(TestBatchRunner *runner) {
VArray *spans = VA_new(8);
VArray *wanted = VA_new(8);
HeatMap *heat_map = HeatMap_new(spans, 133);
VArray *flattened, *boosts;
VA_Push(spans, (Obj*)Span_new(10, 10, 1.0f));
VA_Push(spans, (Obj*)Span_new(16, 14, 2.0f));
flattened = HeatMap_Flatten_Spans(heat_map, spans);
VA_Push(wanted, (Obj*)Span_new(10, 6, 1.0f));
VA_Push(wanted, (Obj*)Span_new(16, 4, 3.0f));
VA_Push(wanted, (Obj*)Span_new(20, 10, 2.0f));
TEST_TRUE(runner, VA_Equals(flattened, (Obj*)wanted),
"flatten two overlapping spans");
VA_Clear(wanted);
boosts = HeatMap_Generate_Proximity_Boosts(heat_map, spans);
VA_Push(wanted, (Obj*)Span_new(10, 20, 3.0f));
TEST_TRUE(runner, VA_Equals(boosts, (Obj*)wanted),
"prox boosts for overlap");
VA_Clear(wanted);
VA_Clear(spans);
DECREF(boosts);
DECREF(flattened);
VA_Push(spans, (Obj*)Span_new(10, 10, 1.0f));
VA_Push(spans, (Obj*)Span_new(16, 14, 2.0f));
VA_Push(spans, (Obj*)Span_new(50, 1, 1.0f));
flattened = HeatMap_Flatten_Spans(heat_map, spans);
VA_Push(wanted, (Obj*)Span_new(10, 6, 1.0f));
VA_Push(wanted, (Obj*)Span_new(16, 4, 3.0f));
VA_Push(wanted, (Obj*)Span_new(20, 10, 2.0f));
VA_Push(wanted, (Obj*)Span_new(50, 1, 1.0f));
TEST_TRUE(runner, VA_Equals(flattened, (Obj*)wanted),
"flatten two overlapping spans, leave hole, then third span");
VA_Clear(wanted);
boosts = HeatMap_Generate_Proximity_Boosts(heat_map, spans);
TEST_TRUE(runner, VA_Get_Size(boosts) == 2 + 1,
"boosts generated for each unique pair, since all were in range");
VA_Clear(spans);
DECREF(boosts);
DECREF(flattened);
VA_Push(spans, (Obj*)Span_new(10, 10, 1.0f));
VA_Push(spans, (Obj*)Span_new(14, 4, 4.0f));
VA_Push(spans, (Obj*)Span_new(16, 14, 2.0f));
flattened = HeatMap_Flatten_Spans(heat_map, spans);
VA_Push(wanted, (Obj*)Span_new(10, 4, 1.0f));
VA_Push(wanted, (Obj*)Span_new(14, 2, 5.0f));
VA_Push(wanted, (Obj*)Span_new(16, 2, 7.0f));
VA_Push(wanted, (Obj*)Span_new(18, 2, 3.0f));
VA_Push(wanted, (Obj*)Span_new(20, 10, 2.0f));
TEST_TRUE(runner, VA_Equals(flattened, (Obj*)wanted),
"flatten three overlapping spans");
VA_Clear(wanted);
boosts = HeatMap_Generate_Proximity_Boosts(heat_map, spans);
TEST_TRUE(runner, VA_Get_Size(boosts) == 2 + 1,
"boosts generated for each unique pair, since all were in range");
VA_Clear(spans);
DECREF(boosts);
DECREF(flattened);
VA_Push(spans, (Obj*)Span_new(10, 10, 1.0f));
VA_Push(spans, (Obj*)Span_new(16, 14, 4.0f));
VA_Push(spans, (Obj*)Span_new(16, 14, 2.0f));
VA_Push(spans, (Obj*)Span_new(30, 10, 10.0f));
flattened = HeatMap_Flatten_Spans(heat_map, spans);
VA_Push(wanted, (Obj*)Span_new(10, 6, 1.0f));
VA_Push(wanted, (Obj*)Span_new(16, 4, 7.0f));
VA_Push(wanted, (Obj*)Span_new(20, 10, 6.0f));
VA_Push(wanted, (Obj*)Span_new(30, 10, 10.0f));
TEST_TRUE(runner, VA_Equals(flattened, (Obj*)wanted),
"flatten 4 spans, middle two have identical range");
VA_Clear(wanted);
boosts = HeatMap_Generate_Proximity_Boosts(heat_map, spans);
TEST_TRUE(runner, VA_Get_Size(boosts) == 3 + 2 + 1,
"boosts generated for each unique pair, since all were in range");
VA_Clear(spans);
DECREF(boosts);
DECREF(flattened);
VA_Push(spans, (Obj*)Span_new( 10, 10, 1.0f));
VA_Push(spans, (Obj*)Span_new( 16, 4, 4.0f));
VA_Push(spans, (Obj*)Span_new( 16, 14, 2.0f));
VA_Push(spans, (Obj*)Span_new(230, 10, 10.0f));
flattened = HeatMap_Flatten_Spans(heat_map, spans);
VA_Push(wanted, (Obj*)Span_new( 10, 6, 1.0f));
VA_Push(wanted, (Obj*)Span_new( 16, 4, 7.0f));
VA_Push(wanted, (Obj*)Span_new( 20, 10, 2.0f));
VA_Push(wanted, (Obj*)Span_new(230, 10, 10.0f));
TEST_TRUE(runner, VA_Equals(flattened, (Obj*)wanted),
"flatten 4 spans, middle two have identical starts but different ends");
VA_Clear(wanted);
boosts = HeatMap_Generate_Proximity_Boosts(heat_map, spans);
TEST_TRUE(runner, VA_Get_Size(boosts) == 2 + 1,
"boosts not generated for out of range span");
VA_Clear(spans);
DECREF(boosts);
DECREF(flattened);
DECREF(heat_map);
DECREF(wanted);
DECREF(spans);
}
