void Graph::addNode(std::string dataKeys,std::string prefix){
std::string temp = str_node;
std::string id = prefix+"-Node-"+intToStr(GenerateNodeId());
find_replace(temp, str_nodeid, id);
find_replace(temp,str_datakeys,dataKeys);
nodes.push_back(ID(id,temp));
}
static void PrintOutputInterface(std::wostream& os, Grammar& g)
{
std::wstring ws(STR_HEADER);
std::wstring upperfile = MakeUpper(g.GetModuleName() + L".h");
FindReplace(upperfile, L".", L"_");
PrintGeneratedFileHeader(os);
os << L"#ifndef " << upperfile << "" << std::endl;
os << L"#define " << upperfile << "" << std::endl;
os << std::endl;
os << std::endl;
find_replace(ws, L"{DATE}", __WDATE__);
find_replace(ws, L"{GRAMMAR}", g.GetLanguageName());
os << L"#include \"Result.h\"\n";
os << L"#include " << "\"" << g.GetLanguageName() << ".h\"\n";
std::wostringstream ss;
PrintActions(ss, g, false);
find_replace(ws, L"{ACTIONS}", ss.str());
os << ws;
os << std::endl;
os << L"#endif" << std::endl;
}
static void PrintActions(std::wostream& os, Grammar& g, bool virt)
{
#if 0
os << L"/*Forward declarations for actions*/\n";
os << L"struct Eval_Context;\n";
os << L"\n";
os << L" " << g.GetLanguageName() << L"_Actions{\n";
const wchar_t * psz0 =
L" int (*{ACTION})( {LANG}_Context* ctx);\n";
std::wstring ws0(psz0);
find_replace(ws0, L"{LANG}", g.GetLanguageName());
for (int i = 0; i < g.GetNumberOfActions(); i++)
{
std::wstring ws(ws0);
find_replace(ws, L"{ACTION}", g.GetActionName(i));
os << ws;
}
os << L"};\n";
#else
os << L"typedef enum \n";
os << L"{\n";
for (int i = 0; i < g.GetNumberOfActions(); i++)
{
os << L" " << g.GetActionName(i) << L",\n";
}
os << g.GetLanguageName() << L"_OnError\n";
os << L"}" << g.GetLanguageName() << L"_Actions;\n";
#endif
}
void Graph::addKey(std::string keyId, std::string keyFor,std::string attrName, std::string attrType){
std::string temp = str_gkey;
find_replace(temp, str_keyid,keyId);
find_replace(temp, str_keyfor,keyFor);
find_replace(temp, str_attrname,attrName);
find_replace(temp, str_attrtype,attrType);
keys.push_back(ID(keyId,temp));
}
void Graph::addEdge(std::string target, std::string source, std::string label,std::string prefix){
std::string temp = str_edge;
std::string id = prefix+"-Edge-"+intToStr(GenerateNodeId());
find_replace(temp, str_edgeid, id);
find_replace(temp, str_source, source);
find_replace(temp, str_target, target);
find_replace(temp, str_label, label);
edges.push_back(ID(id,temp));
}
// replace with existing file with different case if possible
// lol at brute force :)
char* fix_path(const char* name, void* p) {
char* rp = malloc(0x400); //malloc(strlen(p)+1);
char pathbuf[0x400]; // buffer overflow, yay?
char* path = (char *)p;
strcpy(rp, path); // syscalls rely on this...
int didfix = 0;
//printf("%s: (%d) %s\n", name, qa(rp), rp);
// do the magic
if (rp[0] != '/') {
getcwd(rp, 0x1000);
strcat(rp, "/");
strcat(rp, path);
}
char* pathptr = rp+1;
while (pathptr=strchr(pathptr,'/')) {
memcpy(pathbuf, rp, pathptr-rp);
pathbuf[pathptr-rp] = '\0';
//printf("checking %s: %d\n", pathbuf, qa(pathbuf));
if (qa(pathbuf) == 0) {
if (find_replace(pathbuf) == 0) {
goto done;
} else {
//printf("replacing...%s %s %d\n", rp, pathbuf, strlen(pathbuf));
memcpy(rp, pathbuf, strlen(pathbuf));
didfix = 1;
}
}
pathptr++;
}
if (qa(rp) == 0) {
if (find_replace(rp) == 1) {
didfix = 1;
}
}
done:
if (didfix == 1) {
//printf("%s: %s -> %s\n", name, path, rp);
// logging
if (cc_logfile != NULL) {
fprintf(cc_logfile, "%s: %s -> %s\n", name, path, rp);
fflush(cc_logfile);
}
}
return rp;
}
void CreateObjectDataText(String* text, short id) {
Resource rsrc("text", "txt", kShipDataTextID);
String data(macroman::decode(rsrc.data()));
const baseObjectType& baseObject = gBaseObjectData.get()[id];
StringList keys(kShipDataKeyStringID);
StringList values(kShipDataNameID);
// *** Replace place-holders in text with real data, using the fabulous find_replace routine
// an object or a ship?
if ( baseObject.attributes & kCanThink) {
const StringSlice& name = values.at(0);
find_replace(data, 0, keys.at(kShipOrObjectStringNum), name);
} else {
const StringSlice& name = values.at(1);
find_replace(data, 0, keys.at(kShipOrObjectStringNum), name);
}
// ship name
{
StringList names(5000);
const StringSlice& name = names.at(id);
find_replace(data, 0, keys.at(kShipTypeStringNum), name);
}
// ship mass
find_replace(data, 0, keys.at(kMassStringNum), fixed(baseObject.mass));
// ship shields
find_replace(data, 0, keys.at(kShieldStringNum), baseObject.health);
// light speed
find_replace(data, 0, keys.at(kHasLightStringNum), baseObject.warpSpeed);
// max velocity
find_replace(data, 0, keys.at(kMaxSpeedStringNum), fixed(baseObject.maxVelocity));
// thrust
find_replace(data, 0, keys.at(kThrustStringNum), fixed(baseObject.maxThrust));
// par turn
find_replace(data, 0, keys.at(kTurnStringNum),
fixed(baseObject.frame.rotation.turnAcceleration));
// now, check for weapons!
CreateWeaponDataText(&data, baseObject.pulse, values.at(kShipDataPulseStringNum));
CreateWeaponDataText(&data, baseObject.beam, values.at(kShipDataBeamStringNum));
CreateWeaponDataText(&data, baseObject.special, values.at(kShipDataSpecialStringNum));
print(*text, data);
}
void replace_image_to_label(char *input_path, char *output_path) {
//find_replace(input_path, "/images/", "/labels/", output_path); // COCO
find_replace(input_path, "/images/train2014/", "/labels/train2014/", output_path); // COCO
find_replace(output_path, "/images/val2014/", "/labels/val2014/", output_path); // COCO
find_replace(output_path, "/JPEGImages/", "/labels/", output_path); // PascalVOC
//find_replace(output_path, "/VOC2007/JPEGImages/", "/VOC2007/labels/", output_path); // PascalVOC
//find_replace(output_path, "/VOC2012/JPEGImages/", "/VOC2012/labels/", output_path); // PascalVOC
//find_replace(output_path, "/raw/", "/labels/", output_path);
// replace only ext of files
find_replace_extension(output_path, ".jpg", ".txt", output_path);
find_replace_extension(output_path, ".JPG", ".txt", output_path); // error
find_replace_extension(output_path, ".jpeg", ".txt", output_path);
find_replace_extension(output_path, ".JPEG", ".txt", output_path);
find_replace_extension(output_path, ".png", ".txt", output_path);
find_replace_extension(output_path, ".PNG", ".txt", output_path);
find_replace_extension(output_path, ".bmp", ".txt", output_path);
find_replace_extension(output_path, ".BMP", ".txt", output_path);
find_replace_extension(output_path, ".ppm", ".txt", output_path);
find_replace_extension(output_path, ".PPM", ".txt", output_path);
}
void Replace_KeyCode_Strings_With_Actual_Key_Names(String* text, short resID, size_t padTo) {
StringList keys(kHelpScreenKeyStringID);
StringList values(resID);
for (int i = 0; i < kKeyExtendedControlNum; ++i) {
const StringSlice& search = keys.at(i);
String replace(values.at(Preferences::preferences()->key(i) - 1));
if (replace.size() < padTo) {
replace.resize(padTo, ' ');
}
while (find_replace(*text, 0, search, replace) > 0) {
// DO NOTHING
};
}
}
S3Params S3BucketReader::constructReaderParams(BucketContent& key) {
S3Params readerParams = this->params;
// encode the key name but leave the "/"
// "/encoded_path/encoded_name"
string keyEncoded = uri_encode(key.getName());
find_replace(keyEncoded, "%2F", "/");
readerParams.setKeyUrl(this->getKeyURL(keyEncoded));
readerParams.setRegion(this->region);
readerParams.setKeySize(key.getSize());
S3DEBUG("key: %s, size: %" PRIu64, readerParams.getKeyUrl().c_str(), readerParams.getKeySize());
return readerParams;
}
TEST(Utils, find_replace) {
string str1 = "This is a simple & short test.";
find_replace(str1, "simple", "");
EXPECT_STREQ("This is a & short test.", str1.c_str());
find_replace(str1, "short ", "");
EXPECT_STREQ("This is a & test.", str1.c_str());
find_replace(str1, "test.", "");
EXPECT_STREQ("This is a & ", str1.c_str());
find_replace(str1, "This", "");
EXPECT_STREQ(" is a & ", str1.c_str());
find_replace(str1, "is a", "abcdefghijklmn");
EXPECT_STREQ(" abcdefghijklmn & ", str1.c_str());
find_replace(str1, " a", "a");
EXPECT_STREQ("abcdefghijklmn & ", str1.c_str());
find_replace(str1, "abc", "abcabc");
EXPECT_STREQ("abcabcdefghijklmn & ", str1.c_str());
}
void validate_detector_recall(char *cfgfile, char *weightfile)
{
network net = parse_network_cfg(cfgfile);
if(weightfile){
load_weights(&net, weightfile);
}
set_batch_network(&net, 1);
fprintf(stderr, "Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
srand(time(0));
list *plist = get_paths("data/voc.2007.test");
char **paths = (char **)list_to_array(plist);
layer l = net.layers[net.n-1];
int classes = l.classes;
int j, k;
box *boxes = calloc(l.w*l.h*l.n, sizeof(box));
float **probs = calloc(l.w*l.h*l.n, sizeof(float *));
for(j = 0; j < l.w*l.h*l.n; ++j) probs[j] = calloc(classes, sizeof(float *));
int m = plist->size;
int i=0;
float thresh = .001;
float iou_thresh = .5;
float nms = .4;
int total = 0;
int correct = 0;
int proposals = 0;
float avg_iou = 0;
for(i = 0; i < m; ++i){
char *path = paths[i];
image orig = load_image_color(path, 0, 0);
image sized = resize_image(orig, net.w, net.h);
char *id = basecfg(path);
network_predict(net, sized.data);
get_region_boxes(l, 1, 1, thresh, probs, boxes, 1, 0, .5);
if (nms) do_nms(boxes, probs, l.w*l.h*l.n, 1, nms);
char labelpath[4096];
find_replace(path, "images", "labels", labelpath);
find_replace(labelpath, "JPEGImages", "labels", labelpath);
find_replace(labelpath, ".jpg", ".txt", labelpath);
find_replace(labelpath, ".JPEG", ".txt", labelpath);
int num_labels = 0;
box_label *truth = read_boxes(labelpath, &num_labels);
for(k = 0; k < l.w*l.h*l.n; ++k){
if(probs[k][0] > thresh){
++proposals;
}
}
for (j = 0; j < num_labels; ++j) {
++total;
box t = {truth[j].x, truth[j].y, truth[j].w, truth[j].h};
float best_iou = 0;
for(k = 0; k < l.w*l.h*l.n; ++k){
float iou = box_iou(boxes[k], t);
if(probs[k][0] > thresh && iou > best_iou){
best_iou = iou;
}
}
avg_iou += best_iou;
if(best_iou > iou_thresh){
++correct;
}
}
fprintf(stderr, "%5d %5d %5d\tRPs/Img: %.2f\tIOU: %.2f%%\tRecall:%.2f%%\n", i, correct, total, (float)proposals/(i+1), avg_iou*100/total, 100.*correct/total);
free(id);
free_image(orig);
free_image(sized);
}
}
void GenerateDescRecC(std::wostream& os,
Grammar& g,
MMap& map,
const std::wstring& tokenPrefix,
const std::wstring& parserFileSuffix)
{
if (map.empty())
{
return;
}
PrintGeneratedFileHeader(os);
PrintGeneratedFileLicense(os);
os << L"\n";
os << L"#include \"stdafx.h\"\n";
os << L"#include <assert.h>\n";
os << L"\n";
os << L"#include \"" << g.GetModuleName() << L"Lex.h\"\n";
os << L"#include \"" << g.GetModuleName() << parserFileSuffix << L".h\"\n";
os << L"\n";
os << L"\n";
//os << L"#include \"sstream.h\"\n";
//os << L"#include \"errors.h\"\n";
os << L"\n";
os << L"\n";
PrintActionsNames(os, g, false);
//PrintActions(os, g, false);
os << L"\n";
PrintFowardDeclarations(os, g, map);
std::wstring ws(SourceCode);
find_replace(ws, L"{GRAMMAR}", g.GetLanguageName());
find_replace(ws, L"{MODULE}", g.GetLanguageName());
os << ws;
int i = 0;
int currentRuleIndex = -1;
auto it = map.begin();
int rulecount = 0;
for (; it != map.end();)
{
int currentRuleIndex = it->m_pNotTerminal->GetIndex();
//Faz todos desta regra (até ela mudar)
os << L"Result " << GetFunctionName(g, it->m_pNotTerminal->GetName()) << L"( " << g.GetLanguageName() + L"_Context* ctx)\n";
os << L"{\n";
os << TAB_1 << L"Result result = RESULT_OK;\n";
os << TAB_1 << L"" << g.GetLanguageName() << L"_Tokens token = ctx->token; \n";
os << L"\n";
int sub = 0;
rulecount = 0;
while (it->m_pNotTerminal->GetIndex() == currentRuleIndex)
{
int currentResultIndex = it->m_RuleIndex;
//faz todos que resultam na mesma producao
int count = 0;
while (currentResultIndex == it->m_RuleIndex)
{
if (count == 0)
{
os << TAB_1;
if (rulecount > 0)
{
os << L"else ";
}
os << L"if (token == " << tokenPrefix << it->m_pTerminal->GetName();
}
else
{
os << L" ||\n";
os << TAB_1 << L" token == " << tokenPrefix << it->m_pTerminal->GetName();
}
auto itcopy = it;
it++;
count++;
if (it == map.end() ||
currentResultIndex != it->m_RuleIndex)
{
os << L")\n"; //fecha if
const Production& production = g.GetProduction(itcopy->m_RuleIndex);
os << TAB_1 << L"{\n";
os << TAB__2;
os << L"/*";
Print(os, production);
os << L"*/\n";
PrintProduction(os, production, g, tokenPrefix, TAB__2);
os << TAB_1 << L"}\n";
break;
}
if (it == map.end())
{
break;
}
}
/* if (it != map.end())
{
it++;
}
*/
if (it == map.end())
{
break;
}
rulecount++;
} //regra
os << TAB_1 << L"else\n";
os << TAB_1 << L"{\n";
os << TAB__2 << g.GetLanguageName() << L"_OnAction(ctx, " << g.GetLanguageName() << L"_OnError); \n";
os << TAB__2 << L"return RESULT_FAIL;\n";
os << TAB_1 << L"}\n";
os << L"\n";
os << TAB_1 << L"return result;\n";
os << L"}\n\n";
}
}
static bool decode(const Node& node, PlfsConf& pconf) {
set_default_confs(&pconf);
if(node["global_params"] && (node["global_params"].IsNull() ||
node["global_params"].as<string>() == "")) {
if(node["num_hostdirs"]) {
if(!conv(node["num_hostdirs"],pconf.num_hostdirs) ||
pconf.num_hostdirs > MAX_HOSTDIRS ||
pconf.num_hostdirs <= 0)
pconf.err_msg = new string ("Illegal num_hostdirs");
}
if(node["threadpool_size"]) {
if(!conv(node["threadpool_size"], pconf.threadpool_size) ||
pconf.threadpool_size < 1)
pconf.err_msg = new string ("Illegal threadpool_size");
pconf.threadpool_size = max(pconf.threadpool_size, 1);
}
if(node["lazy_stat"]) {
if(!conv(node["lazy_stat"], pconf.lazy_stat))
pconf.err_msg = new string ("Illegal lazy_stat");
}
if(node["lazy_droppings"]) {
if(!conv(node["lazy_droppings"], pconf.lazy_droppings))
pconf.err_msg = new string ("Illegal lazy_droppings");
}
if(node["compress_contiguous"]) {
if(!conv(node["compress_contiguous"],pconf.compress_contiguous))
pconf.err_msg = new string ("Illegal compress_contiguous");
}
if(node["index_buffer_mbs"]) {
if(!conv(node["index_buffer_mbs"], pconf.buffer_mbs) ||
pconf.buffer_mbs < 0)
pconf.err_msg = new string ("Illegal index_buffer_mbs");
}
if(node["read_buffer_mbs"]) {
if(!conv(node["read_buffer_mbs"],pconf.read_buffer_mbs) ||
pconf.read_buffer_mbs < 0)
pconf.err_msg = new string ("Illegal read_buffer_mbs");
}
if(node["global_summary_dir"]) {
string temp;
if(!conv(node["global_summary_dir"],temp) || temp.c_str()[0] != '/')
pconf.err_msg = new string ("Illegal global_summary_dir");
else {
pconf.global_summary_dir = strdup(temp.c_str());
pconf.global_sum_io.prefix = strdup(temp.c_str());
}
}
if(node["test_metalink"]) {
if(!conv(node["test_metalink"],pconf.test_metalink))
pconf.err_msg = new string ("Illegal test_metalink");
}
if(node["mlog_stderr"]) {
bool temp;
if(!conv(node["mlog_stderr"],temp))
pconf.err_msg = new string ("Illegal mlog_stderr");
else {
if (temp)
pconf.mlog_flags |= MLOG_STDERR;
else
pconf.mlog_flags &= ~MLOG_STDERR;
}
}
if(node["mlog_ucon"]) {
bool temp;
if(!conv(node["mlog_ucon"],temp))
pconf.err_msg = new string ("Illegal mlog_ucon");
else {
if (temp)
pconf.mlog_flags |= (MLOG_UCON_ON|MLOG_UCON_ENV);
else
pconf.mlog_flags &= ~(MLOG_UCON_ON|MLOG_UCON_ENV);
}
}
if(node["mlog_syslog"]) {
bool temp;
if(!conv(node["mlog_syslog"],temp))
pconf.err_msg = new string ("Illegal mlog_syslog");
else {
if (temp)
pconf.mlog_flags |= MLOG_SYSLOG;
else
pconf.mlog_flags &= ~MLOG_SYSLOG;
}
}
if(node["mlog_defmask"]) {
string temp;
if(!conv(node["mlog_defmask"],temp))
pconf.err_msg = new string ("Illegal mlog_defmask");
else {
pconf.mlog_defmask = mlog_str2pri(temp.c_str());
if(pconf.mlog_defmask < 0)
pconf.err_msg = new string ("Bad mlog_defmask value");
}
}
if(node["mlog_stderrmask"]) {
string temp;
if(!conv(node["mlog_stderrmask"],temp))
pconf.err_msg = new string ("Illegal mlog_stderrmask");
else {
pconf.mlog_stderrmask = mlog_str2pri(temp.c_str());
if(pconf.mlog_stderrmask < 0)
pconf.err_msg = new string ("Bad mlog_stderrmask value");
}
}
if(node["mlog_file"]) {
string temp;
if(!conv(node["mlog_file"],temp))
pconf.err_msg = new string ("Illegal mlog_file");
else {
if(!(strchr(temp.c_str(),'%') != NULL))
pconf.mlog_file = strdup(temp.c_str());
else {
pconf.mlog_file_base = strdup(temp.c_str());
pconf.mlog_file =
strdup(expand_macros(temp.c_str()).c_str());
}
}
}
if(node["mlog_msgbuf_size"]) {
if(!conv(node["mlog_msgbuf_size"],pconf.mlog_msgbuf_size) ||
pconf.mlog_msgbuf_size < 256)
pconf.err_msg = new string ("Illegal mlog_msgbuf_size");
}
if(node["mlog_syslogfac"]) {
int temp =
atoi(&node["mlog_syslogfac"].as<string>().c_str()[5]);
switch (temp) {
case 0:
pconf.mlog_syslogfac = LOG_LOCAL0;
break;
case 1:
pconf.mlog_syslogfac = LOG_LOCAL1;
break;
case 2:
pconf.mlog_syslogfac = LOG_LOCAL2;
break;
case 3:
pconf.mlog_syslogfac = LOG_LOCAL3;
break;
case 4:
pconf.mlog_syslogfac = LOG_LOCAL4;
break;
case 5:
pconf.mlog_syslogfac = LOG_LOCAL5;
break;
case 6:
pconf.mlog_syslogfac = LOG_LOCAL6;
break;
case 7:
pconf.mlog_syslogfac = LOG_LOCAL7;
break;
default:
pconf.err_msg =
new string("bad mlog_syslogfac value");
}
}
if(node["mlog_setmasks"]) {
string temp;
if(!conv(node["mlog_setmasks"],temp))
pconf.err_msg = new string ("Illegal mlog_setmasks");
else {
find_replace(temp, " ", ",");
pconf.mlog_setmasks = strdup(temp.c_str());
}
}
if(node["fuse_crash_log"]) {
if(!conv(node["fuse_crash_log"],pconf.fuse_crash_log))
pconf.err_msg = new string ("Illegal fuse_crash_log");
}
return true;
}
return false;
}
std::string Graph::getDataKey(std::string keyId, std::string value){
std::string temp = str_datakey;
find_replace(temp, str_key, keyId);
find_replace(temp, str_value, value);
return temp;
}
void validate_recall(char *cfgfile, char *weightfile)
{
network net = parse_network_cfg(cfgfile);
if(weightfile){
load_weights(&net, weightfile);
}
set_batch_network(&net, 1);
fprintf(stderr, "Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
srand(time(0));
char *val_images = "/home/pjreddie/data/voc/test/2007_test.txt";
list *plist = get_paths(val_images);
char **paths = (char **)list_to_array(plist);
layer l = net.layers[net.n - 1];
int num_boxes = l.side;
int num = l.n;
int classes = l.classes;
int j;
box *boxes = calloc(num_boxes*num_boxes*num, sizeof(box));
float **probs = calloc(num_boxes*num_boxes*num, sizeof(float *));
for(j = 0; j < num_boxes*num_boxes*num; ++j) probs[j] = calloc(classes+1, sizeof(float *));
int N = plist->size;
int i=0;
int k;
float iou_thresh = .5;
float thresh = .1;
int total = 0;
int correct = 0;
float avg_iou = 0;
int nms = 1;
int proposals = 0;
int save = 1;
for (i = 0; i < N; ++i) {
char *path = paths[i];
image orig = load_image_color(path, 0, 0);
image resized = resize_image(orig, net.w, net.h);
float *X = resized.data;
float *predictions = network_predict(net, X);
get_boxes(predictions+1+classes, num, num_boxes, 5+classes, boxes);
get_probs(predictions, num*num_boxes*num_boxes, classes, 5+classes, probs);
if (nms) do_nms(boxes, probs, num*num_boxes*num_boxes, (classes>0) ? classes : 1, iou_thresh);
char *labelpath = find_replace(path, "images", "labels");
labelpath = find_replace(labelpath, "JPEGImages", "labels");
labelpath = find_replace(labelpath, ".jpg", ".txt");
labelpath = find_replace(labelpath, ".JPEG", ".txt");
int num_labels = 0;
box_label *truth = read_boxes(labelpath, &num_labels);
for(k = 0; k < num_boxes*num_boxes*num; ++k){
if(probs[k][0] > thresh){
++proposals;
if(save){
char buff[256];
sprintf(buff, "/data/extracted/nms_preds/%d", proposals);
int dx = (boxes[k].x - boxes[k].w/2) * orig.w;
int dy = (boxes[k].y - boxes[k].h/2) * orig.h;
int w = boxes[k].w * orig.w;
int h = boxes[k].h * orig.h;
image cropped = crop_image(orig, dx, dy, w, h);
image sized = resize_image(cropped, 224, 224);
#ifdef OPENCV
save_image_jpg(sized, buff);
#endif
free_image(sized);
free_image(cropped);
sprintf(buff, "/data/extracted/nms_pred_boxes/%d.txt", proposals);
char *im_id = basecfg(path);
FILE *fp = fopen(buff, "w");
fprintf(fp, "%s %d %d %d %d\n", im_id, dx, dy, dx+w, dy+h);
fclose(fp);
free(im_id);
}
}
}
for (j = 0; j < num_labels; ++j) {
++total;
box t = {truth[j].x, truth[j].y, truth[j].w, truth[j].h};
float best_iou = 0;
for(k = 0; k < num_boxes*num_boxes*num; ++k){
float iou = box_iou(boxes[k], t);
if(probs[k][0] > thresh && iou > best_iou){
best_iou = iou;
}
}
avg_iou += best_iou;
if(best_iou > iou_thresh){
++correct;
}
}
free(truth);
free_image(orig);
free_image(resized);
fprintf(stderr, "%5d %5d %5d\tRPs/Img: %.2f\tIOU: %.2f%%\tRecall:%.2f%%\n", i, correct, total, (float)proposals/(i+1), avg_iou*100/total, 100.*correct/total);
}
}
void extract_boxes(char *cfgfile, char *weightfile)
{
network net = parse_network_cfg(cfgfile);
if(weightfile){
load_weights(&net, weightfile);
}
set_batch_network(&net, 1);
fprintf(stderr, "Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
srand(time(0));
char *val_images = "/home/pjreddie/data/voc/test/train.txt";
list *plist = get_paths(val_images);
char **paths = (char **)list_to_array(plist);
layer l = net.layers[net.n - 1];
int num_boxes = l.side;
int num = l.n;
int classes = l.classes;
int j;
box *boxes = calloc(num_boxes*num_boxes*num, sizeof(box));
float **probs = calloc(num_boxes*num_boxes*num, sizeof(float *));
for(j = 0; j < num_boxes*num_boxes*num; ++j) probs[j] = calloc(classes+1, sizeof(float *));
int N = plist->size;
int i=0;
int k;
int count = 0;
float iou_thresh = .3;
for (i = 0; i < N; ++i) {
fprintf(stderr, "%5d %5d\n", i, count);
char *path = paths[i];
image orig = load_image_color(path, 0, 0);
image resized = resize_image(orig, net.w, net.h);
float *X = resized.data;
float *predictions = network_predict(net, X);
get_boxes(predictions+1+classes, num, num_boxes, 5+classes, boxes);
get_probs(predictions, num*num_boxes*num_boxes, classes, 5+classes, probs);
char *labelpath = find_replace(path, "images", "labels");
labelpath = find_replace(labelpath, "JPEGImages", "labels");
labelpath = find_replace(labelpath, ".jpg", ".txt");
labelpath = find_replace(labelpath, ".JPEG", ".txt");
int num_labels = 0;
box_label *truth = read_boxes(labelpath, &num_labels);
FILE *label = stdin;
for(k = 0; k < num_boxes*num_boxes*num; ++k){
int overlaps = 0;
for (j = 0; j < num_labels; ++j) {
box t = {truth[j].x, truth[j].y, truth[j].w, truth[j].h};
float iou = box_iou(boxes[k], t);
if (iou > iou_thresh){
if (!overlaps) {
char buff[256];
sprintf(buff, "/data/extracted/labels/%d.txt", count);
label = fopen(buff, "w");
overlaps = 1;
}
fprintf(label, "%d %f\n", truth[j].id, iou);
}
}
if (overlaps) {
char buff[256];
sprintf(buff, "/data/extracted/imgs/%d", count++);
int dx = (boxes[k].x - boxes[k].w/2) * orig.w;
int dy = (boxes[k].y - boxes[k].h/2) * orig.h;
int w = boxes[k].w * orig.w;
int h = boxes[k].h * orig.h;
image cropped = crop_image(orig, dx, dy, w, h);
image sized = resize_image(cropped, 224, 224);
#ifdef OPENCV
save_image_jpg(sized, buff);
#endif
free_image(sized);
free_image(cropped);
fclose(label);
}
}
free(truth);
free_image(orig);
free_image(resized);
}
}
void validate_yolo_classify(char *datacfg, char *cfgfile, char *weightfile)
{
list *options = read_data_cfg(datacfg);
//char *train_list = option_find_str(options, "train", "data/train_list.txt");
//char *test_list = option_find_str(options, "test", "data/test_list.txt");
char *valid_list = option_find_str(options, "valid", "data/valid_list.txt");
//char *backup_directory = option_find_str(options, "backup", "/backup/");
//char *label_list = option_find_str(options, "labels", "data/labels_list.txt");
//int classes = option_find_int(options, "classes", 2);
//char **labels = get_labels(label_list);
network net = parse_network_cfg(cfgfile);
if(weightfile){
load_weights(&net, weightfile);
}
set_batch_network(&net, 1);
fprintf(stderr, "Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
srand(time(0));
char *base = "results/comp4_det_test_";
//list *plist = get_paths("data/voc.2007.test");
list *plist = get_paths(valid_list);
char **paths = (char **)list_to_array(plist);
layer l = net.layers[net.n-1];
int classes = l.classes;
int square = l.sqrt;
int side = l.side;
int j, k;
FILE **fps = calloc(classes, sizeof(FILE *));
for(j = 0; j < classes; ++j){
char buff[1024];
snprintf(buff, 1024, "%s%s.txt", base, voc_names[j]);
fps[j] = fopen(buff, "w");
}
box *boxes = calloc(side*side*l.n, sizeof(box));
float **probs = calloc(side*side*l.n, sizeof(float *));
for(j = 0; j < side*side*l.n; ++j) probs[j] = calloc(classes, sizeof(float *));
int m = plist->size;
int i=0;
//float thresh = .001;
float thresh = .2;
float iou_thresh = .5;
//float nms = 0;
float nms = 0.5;
int total = 0;
int correct = 0;
int class_correct = 0;
int proposals = 0;
float avg_iou = 0;
for(i = 0; i < m; ++i){
char *path = paths[i];
image orig = load_image_color(path, 0, 0);
image sized = resize_image(orig, net.w, net.h);
char *id = basecfg(path);
float *predictions = network_predict(net, sized.data);
convert_yolo_detections(predictions, classes, l.n, square, side, 1, 1, thresh, probs, boxes, 0);
//if (nms) do_nms(boxes, probs, side*side*l.n, 1, nms);
if (nms) do_nms(boxes, probs, side*side*l.n, classes, nms);
char *labelpath = find_replace(path, "images", "labels");
labelpath = find_replace(labelpath, "JPEGImages", "labels");
labelpath = find_replace(labelpath, ".jpg", ".txt");
labelpath = find_replace(labelpath, ".JPEG", ".txt");
labelpath = find_replace(labelpath, ".bmp", ".txt");
labelpath = find_replace(labelpath, ".dib", ".txt");
labelpath = find_replace(labelpath, ".jpe", ".txt");
labelpath = find_replace(labelpath, ".jp2", ".txt");
labelpath = find_replace(labelpath, ".png", ".txt");
labelpath = find_replace(labelpath, ".pbm", ".txt");
labelpath = find_replace(labelpath, ".pgm", ".txt");
labelpath = find_replace(labelpath, ".ppm", ".txt");
labelpath = find_replace(labelpath, ".sr", ".txt");
labelpath = find_replace(labelpath, ".ras", ".txt");
labelpath = find_replace(labelpath, ".tiff", ".txt");
labelpath = find_replace(labelpath, ".tif", ".txt");
int num_labels = 0;
box_label *truth = read_boxes(labelpath, &num_labels);
for(k = 0; k < side*side*l.n; ++k){
int class = max_index(probs[k], classes);
float prob = probs[k][class];
//fprintf(stderr, "path=%s\t\tk=%d\tprob=%f\tclass=%d\n", path, k, prob, class);
if(prob > thresh){
++proposals;
}
}
for (j = 0; j < num_labels; ++j) {
++total;
box t = {truth[j].x, truth[j].y, truth[j].w, truth[j].h};
float best_iou = 0;
int pre_class = -1;
for(k = 0; k < side*side*l.n; ++k){
float iou = box_iou(boxes[k], t);
int class = max_index(probs[k], classes);
float prob = probs[k][class];
//fprintf(stderr, "path=%s\t\tk=%d\tprob=%f\tclass=%d\n", path, k, prob, class);
if(prob > thresh && iou > best_iou){
best_iou = iou;
pre_class = class;
}
}
avg_iou += best_iou;
if(best_iou > iou_thresh){
++correct;
}
if(pre_class == truth[j].id){
++class_correct;
}
//fprintf(stderr, "true_class=%d\tpre_class=%d\n", truth[j].id, pre_class);
}
fprintf(stderr, "%5d %5d %5d\tRPs/Img: %.2f\tIOU: %.2f%%\tRecall:%.2f%%\t\tClassify:%.2f%%\n", i, correct, total, (float)proposals/(i+1), avg_iou*100/total, 100.*correct/total, 100.*class_correct/total);
free(id);
free_image(orig);
free_image(sized);
}
void validate_yolo_recall(char *cfg, char *weights)
{
network *net = load_network(cfg, weights, 0);
set_batch_network(net, 1);
fprintf(stderr, "Learning Rate: %g, Momentum: %g, Decay: %g\n", net->learning_rate, net->momentum, net->decay);
srand(time(0));
char *base = "results/comp4_det_test_";
list *plist = get_paths("data/voc.2007.test");
char **paths = (char **)list_to_array(plist);
layer l = net->layers[net->n-1];
int classes = l.classes;
int side = l.side;
int j, k;
FILE **fps = calloc(classes, sizeof(FILE *));
for(j = 0; j < classes; ++j){
char buff[1024];
snprintf(buff, 1024, "%s%s.txt", base, voc_names[j]);
fps[j] = fopen(buff, "w");
}
int m = plist->size;
int i=0;
float thresh = .001;
float iou_thresh = .5;
float nms = 0;
int total = 0;
int correct = 0;
int proposals = 0;
float avg_iou = 0;
for(i = 0; i < m; ++i){
char *path = paths[i];
image orig = load_image_color(path, 0, 0);
image sized = resize_image(orig, net->w, net->h);
char *id = basecfg(path);
network_predict(net, sized.data);
int nboxes = 0;
detection *dets = get_network_boxes(net, orig.w, orig.h, thresh, 0, 0, 1, &nboxes);
if (nms) do_nms_obj(dets, side*side*l.n, 1, nms);
char labelpath[4096];
find_replace(path, "images", "labels", labelpath);
find_replace(labelpath, "JPEGImages", "labels", labelpath);
find_replace(labelpath, ".jpg", ".txt", labelpath);
find_replace(labelpath, ".JPEG", ".txt", labelpath);
int num_labels = 0;
box_label *truth = read_boxes(labelpath, &num_labels);
for(k = 0; k < side*side*l.n; ++k){
if(dets[k].objectness > thresh){
++proposals;
}
}
for (j = 0; j < num_labels; ++j) {
++total;
box t = {truth[j].x, truth[j].y, truth[j].w, truth[j].h};
float best_iou = 0;
for(k = 0; k < side*side*l.n; ++k){
float iou = box_iou(dets[k].bbox, t);
if(dets[k].objectness > thresh && iou > best_iou){
best_iou = iou;
}
}
avg_iou += best_iou;
if(best_iou > iou_thresh){
++correct;
}
}
fprintf(stderr, "%5d %5d %5d\tRPs/Img: %.2f\tIOU: %.2f%%\tRecall:%.2f%%\n", i, correct, total, (float)proposals/(i+1), avg_iou*100/total, 100.*correct/total);
free_detections(dets, nboxes);
free(id);
free_image(orig);
free_image(sized);
}
free_network( net );
}
void CreateWeaponDataText(String* text, long whichWeapon, const StringSlice& weaponName) {
baseObjectType *weaponObject, *missileObject;
long mostDamage, actionNum;
objectActionType *action;
bool isGuided = false;
if (whichWeapon == kNoShip) {
return;
}
weaponObject = gBaseObjectData.get() + whichWeapon;
// TODO(sfiera): catch exception.
Resource rsrc("text", "txt", kWeaponDataTextID);
String data(macroman::decode(rsrc.data()));
// damage; this is tricky--we have to guess by walking through activate actions,
// and for all the createObject actions, see which creates the most damaging
// object. We calc this first so we can use isGuided
mostDamage = 0;
isGuided = false;
if ( weaponObject->activateActionNum > 0)
{
action = gObjectActionData.get() + weaponObject->activateAction;
for ( actionNum = 0; actionNum < weaponObject->activateActionNum; actionNum++)
{
if (( action->verb == kCreateObject) || ( action->verb == kCreateObjectSetDest))
{
missileObject = gBaseObjectData.get() +
action->argument.createObject.whichBaseType;
if ( missileObject->attributes & kIsGuided) isGuided = true;
if ( missileObject->damage > mostDamage) mostDamage = missileObject->damage;
}
action++;
}
}
StringList keys(kShipDataKeyStringID);
StringList values(kShipDataNameID);
// weapon name #
find_replace(data, 0, keys.at(kWeaponNumberStringNum), weaponName);
// weapon name
{
StringList names(5000);
const StringSlice& name = names.at(whichWeapon);
find_replace(data, 0, keys.at(kWeaponNameStringNum), name);
}
const StringSlice& yes = values.at(kShipDataYesStringNum);
const StringSlice& no = values.at(kShipDataNoStringNum);
const StringSlice& dash = values.at(kShipDataDashStringNum);
// is guided
if (isGuided) {
find_replace(data, 0, keys.at(kWeaponGuidedStringNum), yes);
} else {
find_replace(data, 0, keys.at(kWeaponGuidedStringNum), no);
}
// is autotarget
if (weaponObject->attributes & kAutoTarget) {
find_replace(data, 0, keys.at(kWeaponAutoTargetStringNum), yes);
} else {
find_replace(data, 0, keys.at(kWeaponAutoTargetStringNum), no);
}
// range
find_replace(data, 0, keys.at(kWeaponRangeStringNum),
lsqrt(weaponObject->frame.weapon.range));
if (mostDamage > 0) {
find_replace(data, 0, keys.at(kWeaponDamageStringNum), mostDamage);
} else {
find_replace(data, 0, keys.at(kWeaponDamageStringNum), dash);
}
print(*text, data);
}
void validate_coco_recall(char *cfgfile, char *weightfile)
{
network net = parse_network_cfg(cfgfile);
if(weightfile){
load_weights(&net, weightfile);
}
set_batch_network(&net, 1);
fprintf(stderr, "Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
srand(time(0));
char *base = "results/comp4_det_test_";
list *plist = get_paths("/home/pjreddie/data/voc/test/2007_test.txt");
char **paths = (char **)list_to_array(plist);
layer l = net.layers[net.n-1];
int classes = l.classes;
int square = l.sqrt;
int side = l.side;
int j, k;
FILE **fps = calloc(classes, sizeof(FILE *));
for(j = 0; j < classes; ++j){
char buff[1024];
_snprintf(buff, 1024, "%s%s.txt", base, coco_classes[j]);
fps[j] = fopen(buff, "w");
}
box *boxes = calloc(side*side*l.n, sizeof(box));
float **probs = calloc(side*side*l.n, sizeof(float *));
for(j = 0; j < side*side*l.n; ++j) probs[j] = calloc(classes, sizeof(float *));
int m = plist->size;
int i=0;
float thresh = .001;
int nms = 0;
float iou_thresh = .5;
float nms_thresh = .5;
int total = 0;
int correct = 0;
int proposals = 0;
float avg_iou = 0;
for(i = 0; i < m; ++i){
char *path = paths[i];
image orig = load_image_color(path, 0, 0);
image sized = resize_image(orig, net.w, net.h);
char *id = basecfg(path);
float *predictions = network_predict(net, sized.data);
convert_detections(predictions, classes, l.n, square, side, 1, 1, thresh, probs, boxes, 1);
if (nms) do_nms(boxes, probs, side*side*l.n, 1, nms_thresh);
char *labelpath = find_replace(path, "images", "labels");
labelpath = find_replace(labelpath, "JPEGImages", "labels");
labelpath = find_replace(labelpath, ".jpg", ".txt");
labelpath = find_replace(labelpath, ".JPEG", ".txt");
int num_labels = 0;
box_label *truth = read_boxes(labelpath, &num_labels);
for(k = 0; k < side*side*l.n; ++k){
if(probs[k][0] > thresh){
++proposals;
}
}
for (j = 0; j < num_labels; ++j) {
++total;
box t = {truth[j].x, truth[j].y, truth[j].w, truth[j].h};
float best_iou = 0;
for(k = 0; k < side*side*l.n; ++k){
float iou = box_iou(boxes[k], t);
if(probs[k][0] > thresh && iou > best_iou){
best_iou = iou;
}
}
avg_iou += best_iou;
if(best_iou > iou_thresh){
++correct;
}
}
fprintf(stderr, "%5d %5d %5d\tRPs/Img: %.2f\tIOU: %.2f%%\tRecall:%.2f%%\n", i, correct, total, (float)proposals/(i+1), avg_iou*100/total, 100.*correct/total);
free(id);
free_image(orig);
free_image(sized);
}
}
