bool reservoirHandler::nodeTrainSD() { sCurrentNode = "nodeTrainSD"; sCurrentGrammarFile = nameGrammarNodeTrainSD; ostringstream osError; // Error message osError << "Error in reservoirHandler | "<< sCurrentNode << " :: "; cout << endl << "In " << sCurrentNode << endl << endl; Bottle bOutput; bool fGetaReply = false; Bottle bSpeechRecognized, //recceived FROM speech recog with transfer information (1/0 (bAnswer) ACK/NACK) bMessenger, //to be send TO speech recog bAnswer, //response from speech recog without transfer information, including raw sentence bSemantic, // semantic information of the content of the recognition bSendReasoning, // send the information of recall to the abmReasoning bSpeak, // bottle for tts bTemp; bMessenger.addString("recog"); bMessenger.addString("grammarXML"); bMessenger.addString(grammarToString(sCurrentGrammarFile).c_str()); //to be replace by Say while (!fGetaReply) { Port2SpeechRecog.write(bMessenger,bSpeechRecognized); cout << "Reply from Speech Recog : " << bSpeechRecognized.toString() << endl; if (bSpeechRecognized.toString() == "NACK" || bSpeechRecognized.size() != 3) { osError << "Check " << sCurrentGrammarFile; bOutput.addString(osError.str()); cout << osError.str() << endl; return false; } if (bSpeechRecognized.get(0).toString() == "0") { osError << "Grammar not recognized"; bOutput.addString(osError.str()); cout << osError.str() << endl; return false; } bAnswer = *bSpeechRecognized.get(1).asList(); if (bAnswer.toString() != "" && !bAnswer.isNull()) { fGetaReply = true; } } bSemantic = *bAnswer.get(1).asList()->get(1).asList(); string sQuestionKind = bAnswer.get(1).asList()->get(0).toString(); if(sCurrentType == "train") { // Do you know any ... if (sQuestionKind == "sentence") { cout << "Human : 'Let me see, I have finished...'" << endl; cout << "iCub says : 'What is the corresponding sentence '" << bAnswer.get(0).asString() << " " << bAnswer.toString() << endl ; /* * bAnswer.get(0).asString() => the circle is to the left of of the cross * bAnswer.toString() => * "the circle is to the left of of the cross" (sentence (sentence1 ((object "the circle") (relative_complete ((spatial_relative ((relative to) (spatial "the left"))) (object "the cross")))))) */ cout << "iCub says : 'I have recognized '" << bAnswer.get(0).asString() << endl ; cout << "iCub says : 'Is it ok ? ... 'continue or exit" << endl; string sSentence = bAnswer.get(0).asString(); nodeTrainSD(); } else if (sQuestionKind == "follow") { string continueExit = bSemantic.check("mode", Value("none")).asString(); if (continueExit == "continue the interaction") { cout << "lMeaningsSentences " << endl; lMeaningsSentences.push_back(sSentence+ sSentence_type); nodeTrainSD(); } else if (continueExit == "exit") { trainSaveMeaningSentence(fileSRinputM.c_str()); nodeType(); } } } else if (sCurrentType == "test") { iCub->say("Ok, Set your initial situation, and show me your object of focus !", false); while (!nodeYesNo()) {} iCub->opc->update(); std::list<Entity*> PresentObjects = iCub->opc->EntitiesCache(); std::vector<RTObject> PresentRtoBefore; for(std::list<Entity*>::iterator itE = PresentObjects.begin() ; itE != PresentObjects.end(); itE++) { if ((*itE)->isType(EFAA_OPC_ENTITY_RTOBJECT)) { RTObject rto; rto.fromBottle((*itE)->asBottle()); if (rto.m_present)PresentRtoBefore.push_back(rto) ; } } if (PresentObjects.size() < 2 && PresentObjects.size() > 3) { iCub->say("Dude, I was expecting 2 or 3 objects... Star again !",false); return nodeTrainSD(); } //get the focus object double maxSalience = 0; string sObjectFocus = "none"; for (std::vector<RTObject>::iterator itRTO = PresentRtoBefore.begin() ; itRTO != PresentRtoBefore.end() ; itRTO++) { if (itRTO->m_saliency > maxSalience) { maxSalience = itRTO->m_saliency; sObjectFocus = itRTO->name(); } } if (maxSalience == 0.) { iCub->say("I think I didn't get your focus object dude...",false); return nodeTrainSD(); } string sSentence = "Ok, so you decided to focus on " + sObjectFocus; iCub->say(sSentence,false); if (PresentRtoBefore.size()==2) { double deltaX = 0.0; double deltaY = 0.0; int iFactor; (PresentRtoBefore[0].name() == sObjectFocus) ? iFactor = 1 : iFactor = -1; deltaX = iFactor*(PresentRtoBefore[1].m_ego_position[0] - PresentRtoBefore[0].m_ego_position[0]); deltaY = iFactor*(PresentRtoBefore[1].m_ego_position[1] - PresentRtoBefore[0].m_ego_position[1]); string sLocation; (deltaY>0)? sLocation = "right" : sLocation = "left"; string sRelative; (iFactor==1)? sRelative = (PresentRtoBefore[1].name()) : sRelative =(PresentRtoBefore[0].name()); cout << "I understood :" << endl << sObjectFocus << "\t" << sLocation << "\t" << sRelative << endl ; //TODO send to xavier get response } else // case of 3 objects { RTObject rtFocus, rtRelative1, rtRelative2; bool bFirstRelative = true; for (unsigned int i = 0 ; i < 3 ; i++) { if (PresentRtoBefore[i].name() != sObjectFocus ) { bFirstRelative? rtRelative1 = PresentRtoBefore[i] : rtRelative2 =PresentRtoBefore[i]; bFirstRelative = false; } else { rtFocus = PresentRtoBefore[i]; } } iCub->say("Thinking of the situation",false); double deltaX1 ; // difference btw focus and relative1 double deltaX2 ; // difference btw focus and relative2 deltaX1 = rtRelative1.m_ego_position[1] - rtFocus.m_ego_position[1]; deltaX2 = rtRelative2.m_ego_position[1] - rtFocus.m_ego_position[1]; string sLocation1; string sLocation2; string sRelative1 = rtRelative1.name(); string sRelative2 = rtRelative2.name(); (deltaX1>0)? sLocation1 = "right" : sLocation1 = "left"; (deltaX2>0)? sLocation2 = "right" : sLocation2 = "left"; cout << "I understood : " << sLocation1 << "\t" << sObjectFocus << "\t" << sRelative1 << endl; cout << "and : " << sLocation2 << "\t" << sObjectFocus << "\t" << sRelative2 << endl; if (sSentence_type.size()==0) { sSentence_type = " :C"; } sdataTestSD = sLocation1 + " " + sObjectFocus + " " + sRelative1 + ", " + sLocation2 + " " + sObjectFocus + " " + sRelative2 + sSentence_type; // TO SEND TO XAVIER copyPastFile(fileXavierTrain.c_str(), fileSRinputM.c_str()); cout << fileXavierTrain << endl; cout << fileSRinputM << endl; createTestwithTrainData(fileSRinputM.c_str(), sdataTestSD); callReservoir(pythonPath + fileSD); string result = openResult(fileSRoutputS.c_str()); iCub->say(result,false); cout << "iCub says : 'I have understood '" << result << endl ; } return nodeType(); } if (bAnswer.get(0).asString() == "return the interaction") { nodeModality(); } return true; }
bool LRH::spatialRelation(string sObjectFocus) { iCub->opc->update(); iCub->opc->checkout(); std::list<Entity*> PresentObjects = iCub->opc->EntitiesCache(); std::vector<Object> PresentRtoBefore; yDebug() << "before loop into present object"; for (std::list<Entity*>::iterator itE = PresentObjects.begin(); itE != PresentObjects.end(); itE++) { if ((*itE)->isType(EFAA_OPC_ENTITY_OBJECT)) { Object rto; rto.fromBottle((*itE)->asBottle()); if (rto.m_present)PresentRtoBefore.push_back(rto); } } yDebug() << "check if at least 2 obj present"; if (PresentObjects.size() < 2) { iCub->say("Dude, I was expecting more than 3 objects... "); return false; } //get the focus object //string sObjectFocus = "circle"; //double maxSalience = 0.0; //string sObjectFocus = "none"; if (sobjectFocusChanged.empty()) { double maxSalience = 0.4; yDebug() << "obj focus is empty"; for (std::vector<Object>::iterator itRTO = PresentRtoBefore.begin(); itRTO != PresentRtoBefore.end(); itRTO++) { yDebug() << "loop in present RTO"; if (itRTO->m_saliency > maxSalience) { yDebug() << "new max sliency"; maxSalience = itRTO->m_saliency; sObjectFocus = itRTO->name(); } } if (maxSalience == 0.) { return false; } } else{ sObjectFocus = sobjectFocusChanged; } cout << "In spatialRelation" << endl; if (PresentRtoBefore.size() == 2) { yDebug() << "== than 2 obj present"; double deltaX = 0.0; double deltaY = 0.0; int iFactor; (PresentRtoBefore[0].name() == sObjectFocus) ? iFactor = 1 : iFactor = -1; yDebug() << "weird if done"; deltaX = iFactor*(PresentRtoBefore[1].m_ego_position[0] - PresentRtoBefore[0].m_ego_position[0]); deltaY = iFactor*(PresentRtoBefore[1].m_ego_position[1] - PresentRtoBefore[0].m_ego_position[1]); string sLocation; (deltaY > 0) ? sLocation = "right" : sLocation = "left"; yDebug() << "second weird if done"; string sRelative; (iFactor == 1) ? sRelative = (PresentRtoBefore[1].name()) : sRelative = (PresentRtoBefore[0].name()); cout << "I understood :" << endl << sObjectFocus << "\t" << sLocation << "\t" << sRelative << endl; sdataTestSD = sLocation + " " + sObjectFocus + " " + sRelative; } else // case of 3 objects { Object rtFocus, rtRelative1, rtRelative2; bool bFirstRelative = true; for (unsigned int i = 0; i < 3; i++) { yDebug() << "in loop because more than 3 obj"; if (PresentRtoBefore[i].name() != sObjectFocus) { yDebug() << "if 1"; bFirstRelative ? rtRelative1 = PresentRtoBefore[i] : rtRelative2 = PresentRtoBefore[i]; bFirstRelative = false; } else { yDebug() << "else 1"; rtFocus = PresentRtoBefore[i]; } } double deltaX1; // difference btw focus and relative1 double deltaX2; // difference btw focus and relative2 deltaX1 = rtRelative1.m_ego_position[1] - rtFocus.m_ego_position[1]; deltaX2 = rtRelative2.m_ego_position[1] - rtFocus.m_ego_position[1]; string sLocation1; string sLocation2; string sRelative1 = rtRelative1.name(); string sRelative2 = rtRelative2.name(); (deltaX1 > 0) ? sLocation1 = "right" : sLocation1 = "left"; (deltaX2 > 0) ? sLocation2 = "right" : sLocation2 = "left"; cout << "I understood : " << sLocation1 << "\t" << sObjectFocus << "\t" << sRelative1 << endl; cout << "and : " << sLocation2 << "\t" << sObjectFocus << "\t" << sRelative2 << endl; string sfocusHierarchy = "<o> [A-P-O-_-_-_-_-_-_][A-_-_-P-O-_-_-_-_] <o>"; sdataTestSD = sLocation1 + " " + sObjectFocus + " " + sRelative1 + "," + sLocation2 + " " + sObjectFocus + " " + sRelative2 + " " + sfocusHierarchy; // spatial location cout << "sdataTestSD : " << sdataTestSD << endl; meaningToSentence(sdataTestSD); string result = openResult(sfileResult.c_str()); iCub->say("I have discoverd a new object " + sObjectFocus); iCub->say("And now I can say you that"); iCub->say(result); // distance //the objectFocus is closer/further the object1 than the object2 string sproximity; (deltaX1 > deltaX2) ? sproximity = "closer" : sproximity = "further"; sfocusHierarchy = "<o> [A-P-O-R-_-_-_-_-_][_-_-_-_-_-_-_-_-_] <o>"; sdataTestSD = sproximity + " " + sObjectFocus + " " + sRelative1 + " " + sRelative2 + " " + sfocusHierarchy; cout << "sdataTestSD : " << sdataTestSD << endl; meaningToSentence(sdataTestSD); result = openResult(sfileResult.c_str()); iCub->say("I learned also that"); iCub->say(result); } return true; }
bool reservoirHandler::nodeTestAP() { sCurrentNode = "nodeTestAP"; sCurrentGrammarFile = nameGrammarNodeTestAP; ostringstream osError; // Error message osError << "Error in reservoirHandler | "<< sCurrentNode << " :: "; Bottle bOutput; bool fGetaReply = false; Bottle bSpeechRecognized, //recceived FROM speech recog with transfer information (1/0 (bAnswer) ACK/NACK) bMessenger, //to be send TO speech recog bAnswer, //response from speech recog without transfer information, including raw sentence bSemantic, // semantic information of the content of the recognition bSendReasoning, // send the information of recall to the abmReasoning bSpeak, // bottle for tts bTemp; bMessenger.addString("recog"); bMessenger.addString("grammarXML"); bMessenger.addString(grammarToString(sCurrentGrammarFile).c_str()); //to be replace by Say while (!fGetaReply) { Port2SpeechRecog.write(bMessenger,bSpeechRecognized); cout << "Reply from Speech Recog : " << bSpeechRecognized.toString() << endl; if (bSpeechRecognized.toString() == "NACK" || bSpeechRecognized.size() != 3) { osError << "Check " << sCurrentGrammarFile; bOutput.addString(osError.str()); cout << osError.str() << endl; return false; } if (bSpeechRecognized.get(0).toString() == "0") { osError << "Grammar not recognized"; bOutput.addString(osError.str()); cout << osError.str() << endl; return false; } bAnswer = *bSpeechRecognized.get(1).asList(); if (bAnswer.toString() != "" && !bAnswer.isNull()) { fGetaReply = true; } } if (bAnswer.get(0).asString() == "stop the interaction") { iCurrentInstance = -1; osError.str(""); osError << " | STOP called"; bOutput.addString(osError.str()); cout << osError.str() << endl; return false; } bSemantic = *bAnswer.get(1).asList()->get(1).asList(); string sQuestionKind = bAnswer.get(1).asList()->get(0).toString(); cout << "I'm here ...................." << endl; iCub->say("Say a sentence",false); // Do you know any ... if (sQuestionKind == "sentence") { /* * bAnswer.get(0).asString() => the circle is to the left of of the cross * bAnswer.toString() => * "the circle is to the left of of the cross" (sentence (sentence1 ((object "the circle") (relative_complete ((spatial_relative ((relative to) (spatial "the left"))) (object "the cross")))))) */ cout << "iCub says : 'I have understood '" << bAnswer.get(0).asString() << endl ; iCub->say("I have understood ....", false); sentence += bAnswer.get(0).asString() + " "; iCub->say(bAnswer.get(0).asString(), false); iCub->say("Is it ok ?", false); cout << "iCub says : 'Is it ok ? ... 'No or Yes" << endl; return nodeTestAP(); } else if (sQuestionKind == "yesno") { string yesNo = bSemantic.check("agree", Value("none")).asString(); if (yesNo == "yes") { copyPastFile(fileXavierTrainAP.c_str(), fileAPimputS.c_str()); cout << fileXavierTrainAP << endl; cout << fileAPimputS << endl; createTestwithTrainData(fileAPimputS.c_str(), sentence); callReservoir(fileAPimputS); string result = openResult(fileAPoutputM.c_str()); iCub->say(result,false); cout << result << endl; int id = result.find(","); int idf = result.size()-id; cout << id << endl; if (id != 0) { cout << result.substr(0, id) << endl; cout << result.substr(id +1,idf) << endl; string firstCommand = result.substr(0, id); string secondCommand = result.substr(id +1,idf); iCub->say("I will do the actions", false); AREactions(extractVocabulary(firstCommand)); extractVocabulary(secondCommand); AREactions(extractVocabulary(secondCommand)); return nodeTestAP(); } else { iCub->say("I will do the actions", false); AREactions(extractVocabulary(result)); sentence = " "; return nodeTestAP(); } cout << "iCub do the action..." << endl; } else if(yesNo == "no") { sentence = " "; } else if (sQuestionKind == "follow") { string continueExit = bSemantic.check("mode", Value("none")).asString(); if (continueExit == "continue the interaction") { cout << "Humain say a command ...." << endl; sentence = " "; return nodeTestAP(); } else if (continueExit == "exit") { return nodeType(); } } } return true; }