void GeneTopoList::core(){
if ( this->TipLabels.size() == 2 ) return;
for ( size_t tip_i = 2; tip_i < this->TipLabels.size() ; tip_i++ ){
this->TreeList_tmp.clear();
for ( size_t i = 0 ; i < this->TreeList.size(); i++ ){
this->str_tmp = this->TreeList[i];
for ( size_t i_str_len = 1; i_str_len < this->str_tmp.size(); ){
if ( isalpha( this->str_tmp[i_str_len] ) || isdigit( this->str_tmp[i_str_len]) ){
string label = extract_label( this->str_tmp, i_str_len);
string new_topo = add_new_taxa_at_tip( str_tmp, i_str_len, this->TipLabels[tip_i], label);
TreeList_tmp.push_back(new_topo);
i_str_len = label.size() + i_str_len;
}
else{
if ( this->str_tmp[i_str_len]==')' ){
string new_topo = add_new_taxa_at_int( str_tmp, i_str_len, this->TipLabels[tip_i]);
TreeList_tmp.push_back( new_topo );
}
i_str_len++;
}
}
}
this->TreeList.clear();
this->TreeList = this->TreeList_tmp;
}
}
int
get_reminder ( S710_Driver *d, S710_Packet_Index which, reminder_t *reminder )
{
packet_t *p;
int ok = 0;
if ( which < S710_GET_REMINDER_1 ||
which > S710_GET_REMINDER_7 )
return 0;
p = get_response ( which, d );
if ( p != NULL) {
reminder->which = p->data[0] + 1;
extract_label ( &p->data[6], &reminder->label, 7 );
memset(&reminder->date,0,sizeof(reminder->date));
reminder->date.tm_sec = 0;
reminder->date.tm_min = BCD(p->data[1]);
reminder->date.tm_hour = BCD(p->data[2]);
reminder->date.tm_mday = BCD(p->data[3]);
reminder->date.tm_mon = LNIB(p->data[5]) - 1;
reminder->date.tm_year = BCD(p->data[4]) + 100;
reminder->date.tm_wday = 0;
reminder->on = (UNIB(p->data[5]) & 0x01) ? S710_ON : S710_OFF;
reminder->exercise = LNIB(p->data[13]);
reminder->repeat = UNIB(p->data[13]);
free ( p );
ok = 1;
}
return ok;
}
void GeneTopoList::extract_TipLabels_from_TreeStr( string &in_str ){
for ( size_t i = 1; i < in_str.size(); ){
if ( isalpha(in_str[i]) || isdigit(in_str[i]) ){
string label = extract_label( in_str, i );
GeneTopoListdout << label << endl;
this->TipLabels.push_back(label);
i += label.size();
} else {
i++;
}
}
}
static int extract_certificate_type(char **s, char *what)
{
int type;
char *str_type;
if (isdigit(**s)) {
type = extract_integer(s, what);
if (type >= 1 && type <= 8)
return type;
if (type == 253 || type == 254)
return type;
if (type >= 65280 && type <= 65534)
return type;
if (type < 0 || type > 65535) {
bitch("bad certificate type %d", type);
return -1;
}
if (type == 0 || type == 255 || type == 65535) {
bitch("certificate type %d is reserved by IANA", type);
return -1;
}
bitch("certificate type %d is unassigned", type);
return -1;
} else {
str_type = extract_label(s, what, "temporary");
if (!str_type) return -1;
if (strcmp(str_type, "pkix") == 0)
return 1;
if (strcmp(str_type, "spki") == 0)
return 2;
if (strcmp(str_type, "pgp") == 0)
return 3;
if (strcmp(str_type, "ipkix") == 0)
return 4;
if (strcmp(str_type, "ispki") == 0)
return 5;
if (strcmp(str_type, "ipgp") == 0)
return 6;
if (strcmp(str_type, "acpkix") == 0)
return 7;
if (strcmp(str_type, "iacpkix") == 0)
return 8;
if (strcmp(str_type, "uri") == 0)
return 253;
if (strcmp(str_type, "oid") == 0)
return 254;
bitch("bad certificate type %s", str_type);
return -1;
}
}
static struct rr* nsec3_parse(char *name, long ttl, int type, char *s)
{
struct rr_nsec3 *rr = getmem(sizeof(*rr));
struct rr *ret_rr;
struct binary_data bitmap;
int i;
int opt_out = 0;
char *str_type = NULL;
int ltype;
i = extract_integer(&s, "hash algorithm");
if (i < 0)
return NULL;
if (i > 255)
return bitch("bad hash algorithm value");
if (i != 1)
return bitch("unrecognized or unsupported hash algorithm");
rr->hash_algorithm = i;
i = extract_integer(&s, "flags");
if (i < 0)
return NULL;
if (i > 255)
return bitch("bad flags value");
if (!(i == 0 || i == 1))
return bitch("unsupported flags value");
if (i == 1)
opt_out = 1;
rr->flags = i;
i = extract_integer(&s, "iterations");
if (i < 0)
return NULL;
if (i > 2500)
return bitch("bad iterations value");
rr->iterations = i;
/* TODO validate iteration count according to key size,
* as per http://tools.ietf.org/html/rfc5155#section-10.3 */
if (*s == '-') {
rr->salt.length = 0;
rr->salt.data = NULL;
s++;
if (*s && !isspace(*s) && *s != ';' && *s != ')')
return bitch("salt is not valid");
s = skip_white_space(s);
} else {
rr->salt = extract_hex_binary_data(&s, "salt", EXTRACT_DONT_EAT_WHITESPACE);
if (rr->salt.length <= 0)
return NULL;
if (rr->salt.length > 255)
return bitch("salt is too long");
}
rr->next_hashed_owner = extract_base32hex_binary_data(&s, "next hashed owner");
bitmap = new_set();
while (s && *s) {
str_type = extract_label(&s, "type list", "temporary");
if (!str_type) return NULL;
ltype = str2rdtype(str_type);
add_bit_to_set(&bitmap, ltype);
}
if (!s)
return NULL;
rr->type_bitmap = compressed_set(&bitmap);
ret_rr = store_record(type, name, ttl, rr);
if (ret_rr) {
G.nsec3_present = 1;
if (opt_out)
G.nsec3_opt_out_present = 1;
}
return ret_rr;
}
static VIO_Status read_one_tag(
FILE *file,
int n_volumes,
VIO_Real tags_volume1_ptr[],
VIO_Real tags_volume2_ptr[],
VIO_Real *weight_ptr,
int *structure_id_ptr,
int *patient_id_ptr,
VIO_STR *label_ptr )
{
VIO_Status status;
VIO_STR line;
VIO_BOOL last_was_blank, in_quotes;
int n_strings, pos, i;
VIO_Real x1 = 0.0, y1 = 0.0, z1 = 0.0, x2 = 0.0, y2 = 0.0, z2 = 0.0;
int structure_id, patient_id;
VIO_Real weight;
VIO_STR label;
/* parameter checking */
if( file == NULL )
{
print_error( "read_one_tag(): passed NULL FILE ptr.\n");
return( VIO_ERROR );
}
status = mni_input_real( file, &x1 );
if( status == VIO_OK )
{
if( mni_input_real( file, &y1 ) != VIO_OK ||
mni_input_real( file, &z1 ) != VIO_OK ||
(n_volumes == 2 &&
(mni_input_real( file, &x2 ) != VIO_OK ||
mni_input_real( file, &y2 ) != VIO_OK ||
mni_input_real( file, &z2 ) != VIO_OK)) )
{
print_error( "read_one_tag(): error reading tag point\n" );
return( VIO_ERROR );
}
if( tags_volume1_ptr != NULL )
{
tags_volume1_ptr[VIO_X] = x1;
tags_volume1_ptr[VIO_Y] = y1;
tags_volume1_ptr[VIO_Z] = z1;
}
if( n_volumes == 2 && tags_volume2_ptr != NULL )
{
tags_volume2_ptr[VIO_X] = x2;
tags_volume2_ptr[VIO_Y] = y2;
tags_volume2_ptr[VIO_Z] = z2;
}
label = NULL;
weight = 0.0;
structure_id = -1;
patient_id = -1;
n_strings = 0;
if( mni_input_line( file, &line ) == VIO_OK )
{
i = 0;
last_was_blank = TRUE;
in_quotes = FALSE;
while( line[i] != VIO_END_OF_STRING )
{
if( line[i] == ' ' || line[i] == '\t' )
{
last_was_blank = TRUE;
}
else
{
if( last_was_blank && !in_quotes )
++n_strings;
last_was_blank = FALSE;
if( line[i] == '\"' )
in_quotes = !in_quotes;
}
++i;
}
while( i > 0 &&
(line[i] == ' ' || line[i] == '\t' ||
line[i] == VIO_END_OF_STRING) )
--i;
if( line[i] == ';' )
{
(void) unget_character( file, (char) ';' );
line[i] = VIO_END_OF_STRING;
}
}
if( n_strings != 0 )
{
if( n_strings == 1 )
{
label = extract_label( line );
}
else if( n_strings < 3 || n_strings > 4 ||
sscanf( line, "%lf %d %d %n", &weight, &structure_id,
&patient_id, &pos ) != 3 )
{
print_error( "input_tag_points(): error reading tag point\n" );
return( VIO_ERROR );
}
else if( n_strings == 4 )
{
label = extract_label( &line[pos] );
}
}
delete_string( line );
if( weight_ptr != NULL )
*weight_ptr = weight;
if( structure_id_ptr != NULL )
*structure_id_ptr = structure_id;
if( patient_id_ptr != NULL )
*patient_id_ptr = patient_id;
if( label_ptr != NULL )
*label_ptr = label;
else
delete_string( label );
}
if( status == VIO_ERROR ) /* --- found no more tag points, should now find ; */
{
if( mni_skip_expected_character( file, (char) ';' ) != VIO_OK )
status = VIO_ERROR;
else
status = VIO_END_OF_FILE;
}
return( status );
}
void ldig_detect(const std::string &modelpath, const std::string &outputpath, const std::vector<std::string> &files, LdigFloat margin) {
cybozu::ldig::Model model;
loadModel(model, modelpath);
const size_t K = model.K;
std::cout << "labels : " << K << std::endl;
std::cout << "features : " << model.M << std::endl;
std::ofstream ofs;
if (outputpath.length() > 0) {
ofs.open(outputpath, std::ios::binary);
if (!ofs.is_open()) {
throw cybozu::ldig::Exception("cannot open output file");
}
ofs.precision(3);
ofs << std::fixed;
}
char buf[BUFSIZE];
std::map<size_t, std::map<size_t, size_t> > predicted;
LdigFloat log_likelihood = 0;
for (auto i=files.begin(), ie=files.end(); i!=ie; ++i) {
std::cout << "loading... " << *i << std::endl;
std::ifstream ifs(*i);
if (!ifs.is_open()) {
throw cybozu::ldig::Exception("cannot open a test file");
}
while (!ifs.eof()) {
ifs.getline(buf, BUFSIZE);
std::string line(buf);
std::string label;
size_t label_k = extract_label(label, line, model.labelmap);
size_t i = line.rfind("\t");
cybozu::String text;
if (i!=std::string::npos) {
cybozu::ldig::normalize(text, line.substr(i+1, line.length() - i - 1));
} else {
cybozu::ldig::normalize(text, line);
}
if (text.length()<=0) continue;
text = "\x01" + text + "\x01";
//std::cout << "THE TEXT IS: " << text << std::endl;
//std::cout << "EXTRACTING FEATURES..." << std::endl;
//size_t predict_k = model.predict(y, text);
cybozu::ldig::Events events;
model.trie.extract_features(events, text);
if (margin > 0 && events.size() < 10) continue;
//std::cout << "PREDICTING..." << std::endl;
//std::vector<LdigFloat> y(K);
size_t predict_k = model.predict(y, events);
if (label_k != (size_t)-1) {
const size_t label_k = model.labelmap.at(label);
predicted[label_k][(y[predict_k] >= 0.6)? predict_k : -1] += 1;
if (y[label_k] > 0) log_likelihood -= log(y[label_k]);
}
//std::cout << "GOT PREDICTION: " << predict_k << std::endl;
const std::string &predict_label = model.label(predict_k);
LdigFloat score = y[predict_k];
//std::cout << "THE SCORE: " << score << std::endl;
if (margin > 0) {
LdigFloat top = 0, second = 0;
for (auto i=y.begin(), ie=y.end(); i!=ie; ++i) {
if (*i>top) {
second = top;
top = *i;
} else if (*i>second) {
second = *i;
}
}
score = top - second;
if (score > margin) continue;
}
//std::cout << "THE SCORE WAS: " << score << " " << " LABEL: " << predict_label << std::endl;
ofs << score << "\t" << predict_label << "\t" << line << std::endl;
}
//std::cout << "CLOSING infile stream..." << std::endl;
ifs.close();
}
size_t cor = 0, sum = 0;
for (auto k=predicted.begin(), ke=predicted.end(); k!=ke; ++k) {
size_t s = 0;
std::ostringstream buf;
for (auto j=k->second.begin(), je=k->second.end(); j!=je; ++j) {
s += j->second;
buf << " " << model.label(j->first) << ":" << j->second;
}
size_t c = 0;
auto l = k->second.find(k->first);
if (l!=k->second.end()) c = l->second;
std::cout << model.label(k->first) << " " << c << " / " << s << " = " << (LdigFloat)(c) / s << " (" << buf.str() << " )" << std::endl;
cor += c;
sum += s;
}
if (sum>0) {
std::cout << "total : " << cor << " / " << sum << " = " << (LdigFloat)cor/sum << ", neg log likelihood " << log_likelihood << std::endl;
}
}
