Normalizer*
Normalizer_load(Normalizer *self, Obj *dump) {
Normalizer_Load_t super_load
= SUPER_METHOD_PTR(NORMALIZER, Lucy_Normalizer_Load);
Normalizer *loaded = super_load(self, dump);
Hash *source = (Hash*)CERTIFY(dump, HASH);
Obj *obj = Hash_Fetch_Str(source, "normalization_form", 18);
CharBuf *form = (CharBuf*)CERTIFY(obj, CHARBUF);
obj = Hash_Fetch_Str(source, "case_fold", 9);
bool_t case_fold = Bool_Get_Value((BoolNum*)CERTIFY(obj, BOOLNUM));
obj = Hash_Fetch_Str(source, "strip_accents", 13);
bool_t strip_accents = Bool_Get_Value((BoolNum*)CERTIFY(obj, BOOLNUM));
return Normalizer_init(loaded, form, case_fold, strip_accents);
}
void
Freezer_serialize(Obj *obj, OutStream *outstream) {
if (Obj_is_a(obj, STRING)) {
Freezer_serialize_string((String*)obj, outstream);
}
else if (Obj_is_a(obj, BLOB)) {
Freezer_serialize_blob((Blob*)obj, outstream);
}
else if (Obj_is_a(obj, VECTOR)) {
Freezer_serialize_varray((Vector*)obj, outstream);
}
else if (Obj_is_a(obj, HASH)) {
Freezer_serialize_hash((Hash*)obj, outstream);
}
else if (Obj_is_a(obj, INTEGER)) {
int64_t val = Int_Get_Value((Integer*)obj);
OutStream_Write_C64(outstream, (uint64_t)val);
}
else if (Obj_is_a(obj, FLOAT)) {
double val = Float_Get_Value((Float*)obj);
OutStream_Write_F64(outstream, val);
}
else if (Obj_is_a(obj, BOOLEAN)) {
bool val = Bool_Get_Value((Boolean*)obj);
OutStream_Write_U8(outstream, (uint8_t)val);
}
else if (Obj_is_a(obj, QUERY)) {
Query_Serialize((Query*)obj, outstream);
}
else if (Obj_is_a(obj, DOC)) {
Doc_Serialize((Doc*)obj, outstream);
}
else if (Obj_is_a(obj, DOCVECTOR)) {
DocVec_Serialize((DocVector*)obj, outstream);
}
else if (Obj_is_a(obj, TERMVECTOR)) {
TV_Serialize((TermVector*)obj, outstream);
}
else if (Obj_is_a(obj, SIMILARITY)) {
Sim_Serialize((Similarity*)obj, outstream);
}
else if (Obj_is_a(obj, MATCHDOC)) {
MatchDoc_Serialize((MatchDoc*)obj, outstream);
}
else if (Obj_is_a(obj, TOPDOCS)) {
TopDocs_Serialize((TopDocs*)obj, outstream);
}
else if (Obj_is_a(obj, SORTSPEC)) {
SortSpec_Serialize((SortSpec*)obj, outstream);
}
else if (Obj_is_a(obj, SORTRULE)) {
SortRule_Serialize((SortRule*)obj, outstream);
}
else {
THROW(ERR, "Don't know how to serialize a %o",
Obj_get_class_name(obj));
}
}
static void
test_normalization(TestBatchRunner *runner) {
FSFolder *modules_folder = TestUtils_modules_folder();
if (modules_folder == NULL) {
SKIP(runner, 13, "Can't locate test data");
return;
}
String *path = Str_newf("unicode/utf8proc/tests.json");
Vector *tests = (Vector*)Json_slurp_json((Folder*)modules_folder, path);
if (!tests) { RETHROW(Err_get_error()); }
for (uint32_t i = 0, max = Vec_Get_Size(tests); i < max; i++) {
Hash *test = (Hash*)Vec_Fetch(tests, i);
String *form = (String*)Hash_Fetch_Utf8(
test, "normalization_form", 18);
bool case_fold = Bool_Get_Value((Boolean*)Hash_Fetch_Utf8(
test, "case_fold", 9));
bool strip_accents = Bool_Get_Value((Boolean*)Hash_Fetch_Utf8(
test, "strip_accents", 13));
Normalizer *normalizer = Normalizer_new(form, case_fold, strip_accents);
Vector *words = (Vector*)Hash_Fetch_Utf8(test, "words", 5);
Vector *norms = (Vector*)Hash_Fetch_Utf8(test, "norms", 5);
for (uint32_t j = 0, max = Vec_Get_Size(words); j < max; j++) {
String *word = (String*)Vec_Fetch(words, j);
Vector *got = Normalizer_Split(normalizer, word);
String *norm = (String*)Vec_Fetch(got, 0);
TEST_TRUE(runner,
norm
&& Str_is_a(norm, STRING)
&& Str_Equals(norm, Vec_Fetch(norms, j)),
"Normalize %s %d %d: %s", Str_Get_Ptr8(form),
case_fold, strip_accents, Str_Get_Ptr8(word)
);
DECREF(got);
}
DECREF(normalizer);
}
DECREF(tests);
DECREF(modules_folder);
DECREF(path);
}
static void
test_accessors(TestBatch *batch) {
Float32 *f32 = Float32_new(1.0);
Float64 *f64 = Float64_new(1.0);
Integer32 *i32 = Int32_new(1);
Integer64 *i64 = Int64_new(1);
float wanted32 = 1.33f;
double wanted64 = 1.33;
float got32;
double got64;
Float32_Set_Value(f32, 1.33f);
TEST_FLOAT_EQ(batch, Float32_Get_Value(f32), 1.33f,
"F32 Set_Value Get_Value");
Float64_Set_Value(f64, 1.33);
got64 = Float64_Get_Value(f64);
TEST_TRUE(batch, *(int64_t*)&got64 == *(int64_t*)&wanted64,
"F64 Set_Value Get_Value");
TEST_TRUE(batch, Float32_To_I64(f32) == 1, "Float32_To_I64");
TEST_TRUE(batch, Float64_To_I64(f64) == 1, "Float64_To_I64");
got32 = (float)Float32_To_F64(f32);
TEST_TRUE(batch, *(int32_t*)&got32 == *(int32_t*)&wanted32,
"Float32_To_F64");
got64 = Float64_To_F64(f64);
TEST_TRUE(batch, *(int64_t*)&got64 == *(int64_t*)&wanted64,
"Float64_To_F64");
Int32_Set_Value(i32, I32_MIN);
TEST_INT_EQ(batch, Int32_Get_Value(i32), I32_MIN,
"I32 Set_Value Get_Value");
Int64_Set_Value(i64, I64_MIN);
TEST_TRUE(batch, Int64_Get_Value(i64) == I64_MIN,
"I64 Set_Value Get_Value");
Int32_Set_Value(i32, -1);
Int64_Set_Value(i64, -1);
TEST_TRUE(batch, Int32_To_F64(i32) == -1, "Int32_To_F64");
TEST_TRUE(batch, Int64_To_F64(i64) == -1, "Int64_To_F64");
TEST_INT_EQ(batch, Bool_Get_Value(CFISH_TRUE), true,
"Bool_Get_Value [true]");
TEST_INT_EQ(batch, Bool_Get_Value(CFISH_FALSE), false,
"Bool_Get_Value [false]");
TEST_TRUE(batch, Bool_To_I64(CFISH_TRUE) == true,
"Bool_To_I64 [true]");
TEST_TRUE(batch, Bool_To_I64(CFISH_FALSE) == false,
"Bool_To_I64 [false]");
TEST_TRUE(batch, Bool_To_F64(CFISH_TRUE) == 1.0,
"Bool_To_F64 [true]");
TEST_TRUE(batch, Bool_To_F64(CFISH_FALSE) == 0.0,
"Bool_To_F64 [false]");
DECREF(i64);
DECREF(i32);
DECREF(f64);
DECREF(f32);
}
