bool reservoirHandler::nodeTrainSD()
{
sCurrentNode = "nodeTrainSD";
sCurrentGrammarFile = nameGrammarNodeTrainSD;
ostringstream osError; // Error message
osError << "Error in reservoirHandler | "<< sCurrentNode << " :: ";
cout << endl << "In " << sCurrentNode << endl << endl;
Bottle bOutput;
bool fGetaReply = false;
Bottle bSpeechRecognized, //recceived FROM speech recog with transfer information (1/0 (bAnswer) ACK/NACK)
bMessenger, //to be send TO speech recog
bAnswer, //response from speech recog without transfer information, including raw sentence
bSemantic, // semantic information of the content of the recognition
bSendReasoning, // send the information of recall to the abmReasoning
bSpeak, // bottle for tts
bTemp;
bMessenger.addString("recog");
bMessenger.addString("grammarXML");
bMessenger.addString(grammarToString(sCurrentGrammarFile).c_str());
//to be replace by Say
while (!fGetaReply)
{
Port2SpeechRecog.write(bMessenger,bSpeechRecognized);
cout << "Reply from Speech Recog : " << bSpeechRecognized.toString() << endl;
if (bSpeechRecognized.toString() == "NACK" || bSpeechRecognized.size() != 3)
{
osError << "Check " << sCurrentGrammarFile;
bOutput.addString(osError.str());
cout << osError.str() << endl;
return false;
}
if (bSpeechRecognized.get(0).toString() == "0")
{
osError << "Grammar not recognized";
bOutput.addString(osError.str());
cout << osError.str() << endl;
return false;
}
bAnswer = *bSpeechRecognized.get(1).asList();
if (bAnswer.toString() != "" && !bAnswer.isNull())
{
fGetaReply = true;
}
}
bSemantic = *bAnswer.get(1).asList()->get(1).asList();
string sQuestionKind = bAnswer.get(1).asList()->get(0).toString();
if(sCurrentType == "train")
{
// Do you know any ...
if (sQuestionKind == "sentence")
{
cout << "Human : 'Let me see, I have finished...'" << endl;
cout << "iCub says : 'What is the corresponding sentence '" << bAnswer.get(0).asString() << " " << bAnswer.toString() << endl ;
/*
* bAnswer.get(0).asString() => the circle is to the left of of the cross
* bAnswer.toString() =>
* "the circle is to the left of of the cross" (sentence (sentence1 ((object "the circle") (relative_complete ((spatial_relative ((relative to) (spatial "the left"))) (object "the cross"))))))
*/
cout << "iCub says : 'I have recognized '" << bAnswer.get(0).asString() << endl ;
cout << "iCub says : 'Is it ok ? ... 'continue or exit" << endl;
string sSentence = bAnswer.get(0).asString();
nodeTrainSD();
}
else if (sQuestionKind == "follow")
{
string continueExit = bSemantic.check("mode", Value("none")).asString();
if (continueExit == "continue the interaction")
{
cout << "lMeaningsSentences " << endl;
lMeaningsSentences.push_back(sSentence+ sSentence_type);
nodeTrainSD();
}
else if (continueExit == "exit")
{
trainSaveMeaningSentence(fileSRinputM.c_str());
nodeType();
}
}
}
else if (sCurrentType == "test")
{
iCub->say("Ok, Set your initial situation, and show me your object of focus !", false);
while (!nodeYesNo())
{}
iCub->opc->update();
std::list<Entity*> PresentObjects = iCub->opc->EntitiesCache();
std::vector<RTObject> PresentRtoBefore;
for(std::list<Entity*>::iterator itE = PresentObjects.begin() ; itE != PresentObjects.end(); itE++)
{
if ((*itE)->isType(EFAA_OPC_ENTITY_RTOBJECT))
{
RTObject rto;
rto.fromBottle((*itE)->asBottle());
if (rto.m_present)PresentRtoBefore.push_back(rto) ;
}
}
if (PresentObjects.size() < 2 && PresentObjects.size() > 3)
{
iCub->say("Dude, I was expecting 2 or 3 objects... Star again !",false);
return nodeTrainSD();
}
//get the focus object
double maxSalience = 0;
string sObjectFocus = "none";
for (std::vector<RTObject>::iterator itRTO = PresentRtoBefore.begin() ; itRTO != PresentRtoBefore.end() ; itRTO++)
{
if (itRTO->m_saliency > maxSalience)
{
maxSalience = itRTO->m_saliency;
sObjectFocus = itRTO->name();
}
}
if (maxSalience == 0.)
{
iCub->say("I think I didn't get your focus object dude...",false);
return nodeTrainSD();
}
string sSentence = "Ok, so you decided to focus on " + sObjectFocus;
iCub->say(sSentence,false);
if (PresentRtoBefore.size()==2)
{
double deltaX = 0.0;
double deltaY = 0.0;
int iFactor;
(PresentRtoBefore[0].name() == sObjectFocus) ? iFactor = 1 : iFactor = -1;
deltaX = iFactor*(PresentRtoBefore[1].m_ego_position[0] - PresentRtoBefore[0].m_ego_position[0]);
deltaY = iFactor*(PresentRtoBefore[1].m_ego_position[1] - PresentRtoBefore[0].m_ego_position[1]);
string sLocation;
(deltaY>0)? sLocation = "right" : sLocation = "left";
string sRelative;
(iFactor==1)? sRelative = (PresentRtoBefore[1].name()) : sRelative =(PresentRtoBefore[0].name());
cout << "I understood :" << endl << sObjectFocus << "\t" << sLocation << "\t" << sRelative << endl ;
//TODO send to xavier get response
}
else // case of 3 objects
{
RTObject rtFocus,
rtRelative1,
rtRelative2;
bool bFirstRelative = true;
for (unsigned int i = 0 ; i < 3 ; i++)
{
if (PresentRtoBefore[i].name() != sObjectFocus )
{
bFirstRelative? rtRelative1 = PresentRtoBefore[i] : rtRelative2 =PresentRtoBefore[i];
bFirstRelative = false;
}
else
{
rtFocus = PresentRtoBefore[i];
}
}
iCub->say("Thinking of the situation",false);
double deltaX1 ; // difference btw focus and relative1
double deltaX2 ; // difference btw focus and relative2
deltaX1 = rtRelative1.m_ego_position[1] - rtFocus.m_ego_position[1];
deltaX2 = rtRelative2.m_ego_position[1] - rtFocus.m_ego_position[1];
string sLocation1;
string sLocation2;
string sRelative1 = rtRelative1.name();
string sRelative2 = rtRelative2.name();
(deltaX1>0)? sLocation1 = "right" : sLocation1 = "left";
(deltaX2>0)? sLocation2 = "right" : sLocation2 = "left";
cout << "I understood : " << sLocation1 << "\t" << sObjectFocus << "\t" << sRelative1 << endl;
cout << "and : " << sLocation2 << "\t" << sObjectFocus << "\t" << sRelative2 << endl;
if (sSentence_type.size()==0)
{
sSentence_type = " :C";
}
sdataTestSD = sLocation1 + " " + sObjectFocus + " " + sRelative1 + ", " + sLocation2 + " " + sObjectFocus + " " + sRelative2 + sSentence_type;
// TO SEND TO XAVIER
copyPastFile(fileXavierTrain.c_str(), fileSRinputM.c_str());
cout << fileXavierTrain << endl;
cout << fileSRinputM << endl;
createTestwithTrainData(fileSRinputM.c_str(), sdataTestSD);
callReservoir(pythonPath + fileSD);
string result = openResult(fileSRoutputS.c_str());
iCub->say(result,false);
cout << "iCub says : 'I have understood '" << result << endl ;
}
return nodeType();
}
if (bAnswer.get(0).asString() == "return the interaction")
{
nodeModality();
}
return true;
}
bool LRH::spatialRelation(string sObjectFocus)
{
iCub->opc->update();
iCub->opc->checkout();
std::list<Entity*> PresentObjects = iCub->opc->EntitiesCache();
std::vector<Object> PresentRtoBefore;
yDebug() << "before loop into present object";
for (std::list<Entity*>::iterator itE = PresentObjects.begin(); itE != PresentObjects.end(); itE++)
{
if ((*itE)->isType(EFAA_OPC_ENTITY_OBJECT))
{
Object rto;
rto.fromBottle((*itE)->asBottle());
if (rto.m_present)PresentRtoBefore.push_back(rto);
}
}
yDebug() << "check if at least 2 obj present";
if (PresentObjects.size() < 2)
{
iCub->say("Dude, I was expecting more than 3 objects... ");
return false;
}
//get the focus object
//string sObjectFocus = "circle";
//double maxSalience = 0.0;
//string sObjectFocus = "none";
if (sobjectFocusChanged.empty())
{
double maxSalience = 0.4;
yDebug() << "obj focus is empty";
for (std::vector<Object>::iterator itRTO = PresentRtoBefore.begin(); itRTO != PresentRtoBefore.end(); itRTO++)
{
yDebug() << "loop in present RTO";
if (itRTO->m_saliency > maxSalience)
{
yDebug() << "new max sliency";
maxSalience = itRTO->m_saliency;
sObjectFocus = itRTO->name();
}
}
if (maxSalience == 0.)
{
return false;
}
}
else{
sObjectFocus = sobjectFocusChanged;
}
cout << "In spatialRelation" << endl;
if (PresentRtoBefore.size() == 2)
{
yDebug() << "== than 2 obj present";
double deltaX = 0.0;
double deltaY = 0.0;
int iFactor;
(PresentRtoBefore[0].name() == sObjectFocus) ? iFactor = 1 : iFactor = -1;
yDebug() << "weird if done";
deltaX = iFactor*(PresentRtoBefore[1].m_ego_position[0] - PresentRtoBefore[0].m_ego_position[0]);
deltaY = iFactor*(PresentRtoBefore[1].m_ego_position[1] - PresentRtoBefore[0].m_ego_position[1]);
string sLocation;
(deltaY > 0) ? sLocation = "right" : sLocation = "left";
yDebug() << "second weird if done";
string sRelative;
(iFactor == 1) ? sRelative = (PresentRtoBefore[1].name()) : sRelative = (PresentRtoBefore[0].name());
cout << "I understood :" << endl << sObjectFocus << "\t" << sLocation << "\t" << sRelative << endl;
sdataTestSD = sLocation + " " + sObjectFocus + " " + sRelative;
}
else // case of 3 objects
{
Object rtFocus,
rtRelative1,
rtRelative2;
bool bFirstRelative = true;
for (unsigned int i = 0; i < 3; i++)
{
yDebug() << "in loop because more than 3 obj";
if (PresentRtoBefore[i].name() != sObjectFocus)
{
yDebug() << "if 1";
bFirstRelative ? rtRelative1 = PresentRtoBefore[i] : rtRelative2 = PresentRtoBefore[i];
bFirstRelative = false;
}
else
{
yDebug() << "else 1";
rtFocus = PresentRtoBefore[i];
}
}
double deltaX1; // difference btw focus and relative1
double deltaX2; // difference btw focus and relative2
deltaX1 = rtRelative1.m_ego_position[1] - rtFocus.m_ego_position[1];
deltaX2 = rtRelative2.m_ego_position[1] - rtFocus.m_ego_position[1];
string sLocation1;
string sLocation2;
string sRelative1 = rtRelative1.name();
string sRelative2 = rtRelative2.name();
(deltaX1 > 0) ? sLocation1 = "right" : sLocation1 = "left";
(deltaX2 > 0) ? sLocation2 = "right" : sLocation2 = "left";
cout << "I understood : " << sLocation1 << "\t" << sObjectFocus << "\t" << sRelative1 << endl;
cout << "and : " << sLocation2 << "\t" << sObjectFocus << "\t" << sRelative2 << endl;
string sfocusHierarchy = "<o> [A-P-O-_-_-_-_-_-_][A-_-_-P-O-_-_-_-_] <o>";
sdataTestSD = sLocation1 + " " + sObjectFocus + " " + sRelative1 + "," + sLocation2 + " " + sObjectFocus + " " + sRelative2 + " " + sfocusHierarchy;
// spatial location
cout << "sdataTestSD : " << sdataTestSD << endl;
meaningToSentence(sdataTestSD);
string result = openResult(sfileResult.c_str());
iCub->say("I have discoverd a new object " + sObjectFocus);
iCub->say("And now I can say you that");
iCub->say(result);
// distance
//the objectFocus is closer/further the object1 than the object2
string sproximity;
(deltaX1 > deltaX2) ? sproximity = "closer" : sproximity = "further";
sfocusHierarchy = "<o> [A-P-O-R-_-_-_-_-_][_-_-_-_-_-_-_-_-_] <o>";
sdataTestSD = sproximity + " " + sObjectFocus + " " + sRelative1 + " " + sRelative2 + " " + sfocusHierarchy;
cout << "sdataTestSD : " << sdataTestSD << endl;
meaningToSentence(sdataTestSD);
result = openResult(sfileResult.c_str());
iCub->say("I learned also that");
iCub->say(result);
}
return true;
}
bool reservoirHandler::nodeTestAP()
{
sCurrentNode = "nodeTestAP";
sCurrentGrammarFile = nameGrammarNodeTestAP;
ostringstream osError; // Error message
osError << "Error in reservoirHandler | "<< sCurrentNode << " :: ";
Bottle bOutput;
bool fGetaReply = false;
Bottle bSpeechRecognized, //recceived FROM speech recog with transfer information (1/0 (bAnswer) ACK/NACK)
bMessenger, //to be send TO speech recog
bAnswer, //response from speech recog without transfer information, including raw sentence
bSemantic, // semantic information of the content of the recognition
bSendReasoning, // send the information of recall to the abmReasoning
bSpeak, // bottle for tts
bTemp;
bMessenger.addString("recog");
bMessenger.addString("grammarXML");
bMessenger.addString(grammarToString(sCurrentGrammarFile).c_str());
//to be replace by Say
while (!fGetaReply)
{
Port2SpeechRecog.write(bMessenger,bSpeechRecognized);
cout << "Reply from Speech Recog : " << bSpeechRecognized.toString() << endl;
if (bSpeechRecognized.toString() == "NACK" || bSpeechRecognized.size() != 3)
{
osError << "Check " << sCurrentGrammarFile;
bOutput.addString(osError.str());
cout << osError.str() << endl;
return false;
}
if (bSpeechRecognized.get(0).toString() == "0")
{
osError << "Grammar not recognized";
bOutput.addString(osError.str());
cout << osError.str() << endl;
return false;
}
bAnswer = *bSpeechRecognized.get(1).asList();
if (bAnswer.toString() != "" && !bAnswer.isNull())
{
fGetaReply = true;
}
}
if (bAnswer.get(0).asString() == "stop the interaction")
{
iCurrentInstance = -1;
osError.str("");
osError << " | STOP called";
bOutput.addString(osError.str());
cout << osError.str() << endl;
return false;
}
bSemantic = *bAnswer.get(1).asList()->get(1).asList();
string sQuestionKind = bAnswer.get(1).asList()->get(0).toString();
cout << "I'm here ...................." << endl;
iCub->say("Say a sentence",false);
// Do you know any ...
if (sQuestionKind == "sentence")
{
/*
* bAnswer.get(0).asString() => the circle is to the left of of the cross
* bAnswer.toString() =>
* "the circle is to the left of of the cross" (sentence (sentence1 ((object "the circle") (relative_complete ((spatial_relative ((relative to) (spatial "the left"))) (object "the cross"))))))
*/
cout << "iCub says : 'I have understood '" << bAnswer.get(0).asString() << endl ;
iCub->say("I have understood ....", false);
sentence += bAnswer.get(0).asString() + " ";
iCub->say(bAnswer.get(0).asString(), false);
iCub->say("Is it ok ?", false);
cout << "iCub says : 'Is it ok ? ... 'No or Yes" << endl;
return nodeTestAP();
}
else if (sQuestionKind == "yesno")
{
string yesNo = bSemantic.check("agree", Value("none")).asString();
if (yesNo == "yes")
{
copyPastFile(fileXavierTrainAP.c_str(), fileAPimputS.c_str());
cout << fileXavierTrainAP << endl;
cout << fileAPimputS << endl;
createTestwithTrainData(fileAPimputS.c_str(), sentence);
callReservoir(fileAPimputS);
string result = openResult(fileAPoutputM.c_str());
iCub->say(result,false);
cout << result << endl;
int id = result.find(",");
int idf = result.size()-id;
cout << id << endl;
if (id != 0)
{
cout << result.substr(0, id) << endl;
cout << result.substr(id +1,idf) << endl;
string firstCommand = result.substr(0, id);
string secondCommand = result.substr(id +1,idf);
iCub->say("I will do the actions", false);
AREactions(extractVocabulary(firstCommand));
extractVocabulary(secondCommand);
AREactions(extractVocabulary(secondCommand));
return nodeTestAP();
}
else
{
iCub->say("I will do the actions", false);
AREactions(extractVocabulary(result));
sentence = " ";
return nodeTestAP();
}
cout << "iCub do the action..." << endl;
}
else if(yesNo == "no")
{
sentence = " ";
}
else if (sQuestionKind == "follow")
{
string continueExit = bSemantic.check("mode", Value("none")).asString();
if (continueExit == "continue the interaction")
{
cout << "Humain say a command ...." << endl;
sentence = " ";
return nodeTestAP();
}
else if (continueExit == "exit")
{
return nodeType();
}
}
}
return true;
}
