JNIEXPORT void JNICALL Java_org_opencv_contrib_FaceRecognizer_save_10
(JNIEnv* env, jclass , jlong self, jstring filename)
{
static const char method_name[] = "contrib::save_10()";
try {
LOGD("%s", method_name);
FaceRecognizer* me = (FaceRecognizer*) self; //TODO: check for NULL
const char* utf_filename = env->GetStringUTFChars(filename, 0); std::string n_filename( utf_filename ? utf_filename : "" ); env->ReleaseStringUTFChars(filename, utf_filename);
me->save( n_filename );
return;
} catch(const std::exception &e) {
throwJavaException(env, &e, method_name);
} catch (...) {
throwJavaException(env, 0, method_name);
}
return;
}
JNIEXPORT void JNICALL Java_org_opencv_contrib_FaceRecognizer_update_10
(JNIEnv* env, jclass , jlong self, jlong src_mat_nativeObj, jlong labels_nativeObj)
{
static const char method_name[] = "contrib::update_10()";
try {
LOGD("%s", method_name);
vector<Mat> src;
Mat& src_mat = *((Mat*)src_mat_nativeObj);
Mat_to_vector_Mat( src_mat, src );
FaceRecognizer* me = (FaceRecognizer*) self; //TODO: check for NULL
Mat& labels = *((Mat*)labels_nativeObj);
me->update( src, labels );
return;
} catch(const std::exception &e) {
throwJavaException(env, &e, method_name);
} catch (...) {
throwJavaException(env, 0, method_name);
}
return;
}
JNIEXPORT void JNICALL Java_org_opencv_contrib_FaceRecognizer_predict_10
(JNIEnv* env, jclass , jlong self, jlong src_nativeObj, jdoubleArray label_out, jdoubleArray confidence_out)
{
static const char method_name[] = "contrib::predict_10()";
try {
LOGD("%s", method_name);
FaceRecognizer* me = (FaceRecognizer*) self; //TODO: check for NULL
Mat& src = *((Mat*)src_nativeObj);
int label;
double confidence;
me->predict( src, label, confidence );
jdouble tmp_label[1] = {label}; env->SetDoubleArrayRegion(label_out, 0, 1, tmp_label); jdouble tmp_confidence[1] = {confidence}; env->SetDoubleArrayRegion(confidence_out, 0, 1, tmp_confidence);
return;
} catch(const std::exception &e) {
throwJavaException(env, &e, method_name);
} catch (...) {
throwJavaException(env, 0, method_name);
}
return;
}
bool srvLearnPerson(ed_perception::LearnPerson::Request& req, ed_perception::LearnPerson::Response& res)
{
std::cout << "Learn person " << req.person_name << std::endl;
rgbd::ImagePtr image = rgbd_client.nextImage();
if (!image)
{
res.error_msg = "Could not capture image";
ROS_ERROR("%s", res.error_msg.c_str());
return true;
}
fr.train(image->getRGBImage(), req.person_name);
// visualizeResult(*image);
return true;
}
bool srvRecognizePerson(ed_perception::RecognizePerson::Request& req, ed_perception::RecognizePerson::Response& res)
{
std::cout << "Recognize person" << std::endl;
rgbd::ImagePtr image = rgbd_client.nextImage();
if (!image)
{
res.error_msg = "Could not capture image";
ROS_ERROR("%s", res.error_msg.c_str());
return true;
}
std::vector<FaceRecognitionResult> detections;
fr.find(image->getRGBImage(), detections);
res.person_detections.resize(detections.size());
for(unsigned int i = 0; i < detections.size(); ++i)
{
const FaceRecognitionResult& det = detections[i];
ed_perception::PersonDetection& det_msg = res.person_detections[i];
det_msg.name = det.name;
det_msg.name_score = det.name_score;
if (det.gender == MALE)
det_msg.gender = ed_perception::PersonDetection::MALE;
else if (det.gender == FEMALE)
det_msg.gender = ed_perception::PersonDetection::FEMALE;
det_msg.age = det.age;
geo::Pose3D pose = geo::Pose3D::identity();
pose.t.z = 3;
det_msg.pose.header.frame_id = image->getFrameId();
geo::convert(pose, det_msg.pose.pose);
}
return true;
}
Graph::Graph(const int *cartan,
const std::vector<Word>& gens, //namesake
const std::vector<Word>& v_cogens,
const std::vector<Word>& e_gens,
const std::vector<Word>& f_gens,
const Vect& weights)
{
//define symmetry group relations
std::vector<Word> words = words_from_cartan(cartan);
{
const Logging::fake_ostream& os = logger.debug();
os << "relations =";
for (int w=0; w<6; ++w) {
Word& word = words[w];
os << "\n ";
for (unsigned i=0; i<word.size(); ++i) {
os << word[i];
}
}
os |0;
}
//check vertex stabilizer generators
{
const Logging::fake_ostream& os = logger.debug();
os << "v_cogens =";
for (unsigned w=0; w<v_cogens.size(); ++w) {
const Word& jenn = v_cogens[w]; //namesake
os << "\n ";
for (unsigned t=0; t<jenn.size(); ++t) {
int j = jenn[t];
os << j;
Assert (0<=j and j<4,
"generator out of range: letter w["
<< w << "][" << t << "] = " << j );
}
}
os |0;
}
//check edge generators
{
const Logging::fake_ostream& os = logger.debug();
os << "e_gens =";
for (unsigned w=0; w<e_gens.size(); ++w) {
const Word& edge = e_gens[w];
os << "\n ";
for (unsigned t=0; t<edge.size(); ++t) {
int j = edge[t];
os << j;
Assert (0<=j and j<4,
"generator out of range: letter w["
<< w << "][" << t << "] = " << j );
}
}
os |0;
}
//check face generators
{
const Logging::fake_ostream& os = logger.debug();
os << "f_gens =";
for (unsigned w=0; w<f_gens.size(); ++w) {
const Word& face = f_gens[w];
os << "\n ";
for (unsigned t=0; t<face.size(); ++t) {
int j = face[t];
os << j;
Assert (0<=j and j<4,
"generator out of range: letter w["
<< w << "][" << t << "] = " << j );
}
}
os |0;
}
//build symmetry group
Group group(words);
logger.debug() << "group.ord = " << group.ord |0;
//build subgroup
std::vector<int> subgroup; subgroup.push_back(0);
std::set<int> in_subgroup; in_subgroup.insert(0);
for (unsigned g=0; g<subgroup.size(); ++g) {
int g0 = subgroup[g];
for (unsigned j=0; j<gens.size(); ++j) {
int g1 = group.left(g0,gens[j]);
if (in_subgroup.find(g1) != in_subgroup.end()) continue;
subgroup.push_back(g1);
in_subgroup.insert(g1);
}
}
logger.debug() << "subgroup.ord = " << subgroup.size() |0;
//build cosets and count ord
std::map<int,int> coset; //maps group elements to cosets
ord = 0; //used as coset number
for (unsigned g=0; g<subgroup.size(); ++g) {
int g0 = subgroup[g];
if (coset.find(g0) != coset.end()) continue;
int c0 = ord++;
coset[g0] = c0;
std::vector<int> members(1, g0);
std::vector<int> others(0);
for (unsigned i=0; i<members.size(); ++i) {
int g1 = members[i];
for (unsigned w=0; w<v_cogens.size(); ++w) {
int g2 = group.left(g1, v_cogens[w]);
if (coset.find(g2) != coset.end()) continue;
coset[g2] = c0;
members.push_back(g2);
}
}
}
logger.info() << "cosets table built: " << " ord = " << ord |0;
//build edge lists
std::vector<std::set<int> > neigh(ord);
for (unsigned g=0; g<subgroup.size(); ++g) {
int g0 = subgroup[g];
int c0 = coset[g0];
for (unsigned w=0; w<e_gens.size(); ++w) {
int g1 = group.left(g0, e_gens[w]);
Assert (in_subgroup.find(g1) != in_subgroup.end(),
"edge leaves subgroup");
int c1 = coset[g1];
if (c0 != c1) neigh[c0].insert(c1);
}
}
// make symmetric
for (int c0=0; c0<ord; ++c0) {
const std::set<int>& n = neigh[c0];
for (std::set<int>::iterator c1=n.begin(); c1!=n.end(); ++c1) {
neigh[*c1].insert(c0);
}
}
// build edge table
adj.resize(ord);
for (int c=0; c<ord; ++c) {
adj[c].insert(adj[c].begin(), neigh[c].begin(), neigh[c].end());
}
neigh.clear();
deg = adj[0].size();
logger.info() << "edge table built: deg = " << deg |0;
//define faces
for (unsigned g=0; g<f_gens.size(); ++g) {
const Word& face = f_gens[g];
logger.debug() << "defining faces on " << face |0;
Logging::IndentBlock block;
//define basic face in group
Ring basic(1,0);
// g = 0;
int g0 = 0;
for (unsigned c=0; true; ++c) {
g0 = group.left(g0, face[c%face.size()]);
if (c >= face.size() and g0 == 0) break;
if (in_subgroup.find(g0) != in_subgroup.end() and g0 != basic.back()) {
basic.push_back(g0);
}
}
for (unsigned c=0; c<basic.size(); ++c) {
logger.debug() << " corner: " << basic[c] |0;
}
logger.debug() << "sides/face (free) = " << basic.size() |0;
//build orbit of basic face
std::vector<Ring> faces_g; faces_g.push_back(basic);
FaceRecognizer recognized; recognized(basic);
for (unsigned i=0; i<faces_g.size(); ++i) {
const Ring f = faces_g[i];
for (unsigned j=0; j<gens.size(); ++j) {
//right action of group on faces
Ring f_j(f.size());
for (unsigned c=0; c<f.size(); ++c) {
f_j[c] = group.right(f[c],gens[j]);
}
//add face
if (not recognized(f_j)) {
faces_g.push_back(f_j);
//logger.debug() << "new face: " << f_j |0;
} else {
//logger.debug() << "old face: " << f_j|0;
}
}
}
//hom face down to quotient graph
recognized.clear();
for (unsigned f=0; f<faces_g.size(); ++f) {
const Ring face_g = faces_g[f];
Ring face;
face.push_back(coset[face_g[0]]);
for (unsigned i=1; i<face_g.size(); ++i) {
int c = coset[face_g[i]];
if (c != face.back() and c != face[0]) {
face.push_back(c);
}
}
if (face.size() < 3) continue;
if (not recognized(face)) {
faces.push_back(face);
}
}
}
ord_f = faces.size();
logger.info() << "faces defined: order = " << ord_f |0;
//define vertex coset
std::vector<Word> vertex_coset;
for (unsigned g=0; g<subgroup.size(); ++g) {
int g0 = subgroup[g];
if (coset[g0]==0) vertex_coset.push_back(group.parse(g0));
}
//build geometry
std::vector<Mat> gen_reps(gens.size());
points.resize(ord);
build_geom(cartan, vertex_coset, gens, v_cogens, weights,
gen_reps, points[0]);
std::vector<int> pointed(ord,0);
pointed[0] = true;
logger.debug() << "geometry built" |0;
//build point sets
std::vector<int> reached(1,0);
std::set<int> is_reached;
is_reached.insert(0);
for (unsigned g=0; g<subgroup.size(); ++g) {
int g0 = reached[g];
for (unsigned j=0; j<gens.size(); ++j) {
int g1 = group.right(g0,gens[j]);
if (is_reached.find(g1) == is_reached.end()) {
if (not pointed[coset[g1]]) {
vect_mult(gen_reps[j], points[coset[g0]],
points[coset[g1]]);
pointed[coset[g1]] = true;
}
reached.push_back(g1);
is_reached.insert(g1);
}
}
}
logger.debug() << "point set built." |0;
//build face normals
normals.resize(ord_f);
for (int f=0; f<ord_f; ++f) {
Ring& face = faces[f];
Vect &a = points[face[0]];
Vect &b = points[face[1]];
Vect &c = points[face[2]];
Vect &n = normals[f];
cross4(a,b,c, n);
normalize(n);
/*
Assert1(fabs(inner(a,n)) < 1e-6,
"bad normal: <n,a> = " << fabs(inner(a,n)));
Assert1(fabs(inner(b,n)) < 1e-6,
"bad normal: <n,b> = " << fabs(inner(b,n)));
Assert1(fabs(inner(c,n)) < 1e-6,
"bad normal: <n,b> = " << fabs(inner(c,n)));
*/
}
logger.debug() << "face normals built." |0;
}
bool clearPersons(std_srvs::Empty::Request& req, std_srvs::Empty::Response& res)
{
fr.clear();
return true;
}
int main(int argc, char **argv)
{
std::string image_filename;
std::string config_filename;
if (argc < 3)
{
std::cout << "Please provide --file or --config" << std::endl;
return 1;
}
for(unsigned int i = 1 ; i + 1 < argc; i += 2)
{
std::string arg = argv[i];
std::cout << arg << std::endl;
if (arg[0] == '_')
break;
if (arg == "--file")
image_filename = argv[i + 1];
else if (arg == "--config")
config_filename = argv[i + 1];
else
{
std::cerr << "Unknown option: " << arg << std::endl;
return 1;
}
}
tue::Configuration config;
if (!config_filename.empty())
{
tue::config::loadFromYAMLFile(config_filename, config);
fr.configure(config);
}
if (!image_filename.empty())
{
rgbd::ImagePtr image = loadImage(image_filename);
if (!image)
{
std::cerr << "Image could not be loaded" << std::endl;
return 1;
}
fr.configure(config);
if (config.hasError())
{
std::cerr << "Error in configuration: " << config.error().c_str() << std::endl;
return 1;
}
std::vector<FaceRecognitionResult> detections;
fr.find(image->getRGBImage(), detections);
visualizeResult(*image, detections);
}
else
{
// No image filename given, so run as rosnode
ros::init(argc, argv, "face_recognition");
ros::NodeHandle nh;
if (argc == 1)
{
ROS_ERROR("[FACE RECOGNITION] Please provide config file");
return 1;
}
if (!config.hasError())
{
std::string topic;
if (config.value("camera_topic", topic))
rgbd_client.intialize(topic);
fr.configure(config);
}
if (config.hasError())
{
ROS_ERROR("[FACE RECOGNITION] %s", config.error().c_str());
return 1;
}
ros::ServiceServer srv_learn_person = nh.advertiseService("learn_person", srvLearnPerson);
ros::ServiceServer srv_recognize_person = nh.advertiseService("recognize_person", srvRecognizePerson);
ros::ServiceServer srv_clear_persons = nh.advertiseService("clear_persons", clearPersons);
ros::spin();
}
if (argc == 1 || std::string(argv[1]) != "--file")
{
}
else
{
// Filename as argument, so run as test
if (argc < 3)
{
std::cout << "Please provide rgbd filename" << std::endl;
return 1;
}
std::string image_filename = argv[2];
}
return 0;
}
int main()
{
try
{
cv::VideoCapture cap(0);
cv::namedWindow( "Face window", cv::WINDOW_AUTOSIZE);
image_window win;
OpenFace of("resources/shape_predictor_68_face_landmarks.dat",
"src/openface/forward_nn.lua", "resources/nn4.v2.t7");
FaceDetector fd("resources/haarcascade_frontalface_alt.xml",
"");
FaceRecognizer fr;
try {
fr.load("facedatabase.dat");
}
catch(serialization_error& err) {
cout << "No learned database found." << endl;
cout << "Please create a database with the ./examples/database_builder" << endl;
cout << "Afterwards learn the decision function using ./examples/database_processor" << endl;
cout << endl << err.what() << endl;
}
//Grab and process frames until the main window is closed by the user.
while(!win.is_closed())
{
// Grab a frame
cv::Mat temp;
cap >> temp;
Image img(temp);
Detection detect = fd.dlib_detect(img);
if (detect.face.width() == 0) {
win.set_image(img.asDLIBImage());
continue;
}
//fa.align(detect);
FaceNetEmbed rep = of.facenet(detect);
auto df = fr.df();
auto result = df.predict(rep);
if (result.second > 0.5) {
std::stringstream ss;
ss << result.first << ", " << result.second;
std::string label = ss.str(); //fr.recognize(rep);
// Display it all on the screen
win.clear_overlay();
win.set_image(img.asDLIBImage());
win.add_overlay(detect.rect.asDLIBRect(),rgb_pixel(255,0,0), label);
}
else {
// Display it all on the screen
win.clear_overlay();
win.set_image(img.asDLIBImage());
win.add_overlay(detect.rect.asDLIBRect(),rgb_pixel(255,0,0), std::to_string(result.second));
}
}
}
catch(serialization_error& e)
{
cout << "You need dlib's default face landmarking model file to run this example." << endl;
cout << "You can get it from the following URL: " << endl;
cout << " http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2" << endl;
cout << endl << e.what() << endl;
}
catch(exception& e)
{
cout << e.what() << endl;
}
}
