bool reservoirHandler::nodeTrainSD()
{
sCurrentNode = "nodeTrainSD";
sCurrentGrammarFile = nameGrammarNodeTrainSD;
ostringstream osError; // Error message
osError << "Error in reservoirHandler | "<< sCurrentNode << " :: ";
cout << endl << "In " << sCurrentNode << endl << endl;
Bottle bOutput;
bool fGetaReply = false;
Bottle bSpeechRecognized, //recceived FROM speech recog with transfer information (1/0 (bAnswer) ACK/NACK)
bMessenger, //to be send TO speech recog
bAnswer, //response from speech recog without transfer information, including raw sentence
bSemantic, // semantic information of the content of the recognition
bSendReasoning, // send the information of recall to the abmReasoning
bSpeak, // bottle for tts
bTemp;
bMessenger.addString("recog");
bMessenger.addString("grammarXML");
bMessenger.addString(grammarToString(sCurrentGrammarFile).c_str());
//to be replace by Say
while (!fGetaReply)
{
Port2SpeechRecog.write(bMessenger,bSpeechRecognized);
cout << "Reply from Speech Recog : " << bSpeechRecognized.toString() << endl;
if (bSpeechRecognized.toString() == "NACK" || bSpeechRecognized.size() != 3)
{
osError << "Check " << sCurrentGrammarFile;
bOutput.addString(osError.str());
cout << osError.str() << endl;
return false;
}
if (bSpeechRecognized.get(0).toString() == "0")
{
osError << "Grammar not recognized";
bOutput.addString(osError.str());
cout << osError.str() << endl;
return false;
}
bAnswer = *bSpeechRecognized.get(1).asList();
if (bAnswer.toString() != "" && !bAnswer.isNull())
{
fGetaReply = true;
}
}
bSemantic = *bAnswer.get(1).asList()->get(1).asList();
string sQuestionKind = bAnswer.get(1).asList()->get(0).toString();
if(sCurrentType == "train")
{
// Do you know any ...
if (sQuestionKind == "sentence")
{
cout << "Human : 'Let me see, I have finished...'" << endl;
cout << "iCub says : 'What is the corresponding sentence '" << bAnswer.get(0).asString() << " " << bAnswer.toString() << endl ;
/*
* bAnswer.get(0).asString() => the circle is to the left of of the cross
* bAnswer.toString() =>
* "the circle is to the left of of the cross" (sentence (sentence1 ((object "the circle") (relative_complete ((spatial_relative ((relative to) (spatial "the left"))) (object "the cross"))))))
*/
cout << "iCub says : 'I have recognized '" << bAnswer.get(0).asString() << endl ;
cout << "iCub says : 'Is it ok ? ... 'continue or exit" << endl;
string sSentence = bAnswer.get(0).asString();
nodeTrainSD();
}
else if (sQuestionKind == "follow")
{
string continueExit = bSemantic.check("mode", Value("none")).asString();
if (continueExit == "continue the interaction")
{
cout << "lMeaningsSentences " << endl;
lMeaningsSentences.push_back(sSentence+ sSentence_type);
nodeTrainSD();
}
else if (continueExit == "exit")
{
trainSaveMeaningSentence(fileSRinputM.c_str());
nodeType();
}
}
}
else if (sCurrentType == "test")
{
iCub->say("Ok, Set your initial situation, and show me your object of focus !", false);
while (!nodeYesNo())
{}
iCub->opc->update();
std::list<Entity*> PresentObjects = iCub->opc->EntitiesCache();
std::vector<RTObject> PresentRtoBefore;
for(std::list<Entity*>::iterator itE = PresentObjects.begin() ; itE != PresentObjects.end(); itE++)
{
if ((*itE)->isType(EFAA_OPC_ENTITY_RTOBJECT))
{
RTObject rto;
rto.fromBottle((*itE)->asBottle());
if (rto.m_present)PresentRtoBefore.push_back(rto) ;
}
}
if (PresentObjects.size() < 2 && PresentObjects.size() > 3)
{
iCub->say("Dude, I was expecting 2 or 3 objects... Star again !",false);
return nodeTrainSD();
}
//get the focus object
double maxSalience = 0;
string sObjectFocus = "none";
for (std::vector<RTObject>::iterator itRTO = PresentRtoBefore.begin() ; itRTO != PresentRtoBefore.end() ; itRTO++)
{
if (itRTO->m_saliency > maxSalience)
{
maxSalience = itRTO->m_saliency;
sObjectFocus = itRTO->name();
}
}
if (maxSalience == 0.)
{
iCub->say("I think I didn't get your focus object dude...",false);
return nodeTrainSD();
}
string sSentence = "Ok, so you decided to focus on " + sObjectFocus;
iCub->say(sSentence,false);
if (PresentRtoBefore.size()==2)
{
double deltaX = 0.0;
double deltaY = 0.0;
int iFactor;
(PresentRtoBefore[0].name() == sObjectFocus) ? iFactor = 1 : iFactor = -1;
deltaX = iFactor*(PresentRtoBefore[1].m_ego_position[0] - PresentRtoBefore[0].m_ego_position[0]);
deltaY = iFactor*(PresentRtoBefore[1].m_ego_position[1] - PresentRtoBefore[0].m_ego_position[1]);
string sLocation;
(deltaY>0)? sLocation = "right" : sLocation = "left";
string sRelative;
(iFactor==1)? sRelative = (PresentRtoBefore[1].name()) : sRelative =(PresentRtoBefore[0].name());
cout << "I understood :" << endl << sObjectFocus << "\t" << sLocation << "\t" << sRelative << endl ;
//TODO send to xavier get response
}
else // case of 3 objects
{
RTObject rtFocus,
rtRelative1,
rtRelative2;
bool bFirstRelative = true;
for (unsigned int i = 0 ; i < 3 ; i++)
{
if (PresentRtoBefore[i].name() != sObjectFocus )
{
bFirstRelative? rtRelative1 = PresentRtoBefore[i] : rtRelative2 =PresentRtoBefore[i];
bFirstRelative = false;
}
else
{
rtFocus = PresentRtoBefore[i];
}
}
iCub->say("Thinking of the situation",false);
double deltaX1 ; // difference btw focus and relative1
double deltaX2 ; // difference btw focus and relative2
deltaX1 = rtRelative1.m_ego_position[1] - rtFocus.m_ego_position[1];
deltaX2 = rtRelative2.m_ego_position[1] - rtFocus.m_ego_position[1];
string sLocation1;
string sLocation2;
string sRelative1 = rtRelative1.name();
string sRelative2 = rtRelative2.name();
(deltaX1>0)? sLocation1 = "right" : sLocation1 = "left";
(deltaX2>0)? sLocation2 = "right" : sLocation2 = "left";
cout << "I understood : " << sLocation1 << "\t" << sObjectFocus << "\t" << sRelative1 << endl;
cout << "and : " << sLocation2 << "\t" << sObjectFocus << "\t" << sRelative2 << endl;
if (sSentence_type.size()==0)
{
sSentence_type = " :C";
}
sdataTestSD = sLocation1 + " " + sObjectFocus + " " + sRelative1 + ", " + sLocation2 + " " + sObjectFocus + " " + sRelative2 + sSentence_type;
// TO SEND TO XAVIER
copyPastFile(fileXavierTrain.c_str(), fileSRinputM.c_str());
cout << fileXavierTrain << endl;
cout << fileSRinputM << endl;
createTestwithTrainData(fileSRinputM.c_str(), sdataTestSD);
callReservoir(pythonPath + fileSD);
string result = openResult(fileSRoutputS.c_str());
iCub->say(result,false);
cout << "iCub says : 'I have understood '" << result << endl ;
}
return nodeType();
}
if (bAnswer.get(0).asString() == "return the interaction")
{
nodeModality();
}
return true;
}
/*
* Node to ask Human to give feedback on quality of augmented kinematic structure image
*/
void abmInteraction::nodeFeedback(bool tryAgain, pair<string,int> & bestTimeAndRank)
{
ostringstream osError; // Error message
Bottle bOutput;
Bottle bRecognized, //recceived FROM speech recog with transfer information (1/0 (bAnswer))
bAnswer, //response from speech recog without transfer information, including raw sentence
bSemantic, // semantic information of the content of the recognition
bMessenger; //to be send TO speech recog
ostringstream osResponse;
/************************************ Usual feedback : show an image and ask for feedback ************************************/
if (tryAgain == false){
yInfo() << " Current time = " << *it_augmentedTime;
iCub->say("Note this kinematic structure between 1 and 10 please", false);
yInfo() << " iCub says : Note this kinematic structure between 1 and 10 please";
//feedback likert 1-5
/*iCub->say("Note this kinematic structure, Likert 1-5 quality");
yInfo() << " iCub says : Note this kinematic structure, Likert 1-5 quality" ;*/
//Preparing bottle to trigger the augmenting remembering
Bottle bRpc, bSubRealtime, bSubAugmentedTimes;
bRpc.addString("triggerStreaming");
bRpc.addInt(rememberedInstance);
bSubRealtime.addString("realtime");
bSubRealtime.addInt(1);
bSubAugmentedTimes.addString("augmentedTimes");
bSubAugmentedTimes.addString(*it_augmentedTime);
//If we have a previously best rank
if (bestTimeAndRank.second != 0) {
osResponse.str("");
osResponse << "The current best structure is shown at left. The rank is " << bestTimeAndRank.second; //<< " for time = " << bestAugmentedTime ;
iCub->say(osResponse.str().c_str(), false);
yInfo() << "iCub says : " << osResponse.str();
bSubAugmentedTimes.addString(bestTimeAndRank.first);
}
//Ask for showing the current testing augmented + the best one if relevant
bRpc.addList() = bSubRealtime;
bRpc.addList() = bSubAugmentedTimes;
iCub->getABMClient()->rpcCommand(bRpc);
}
else {
iCub->say("Can you repeat your feedback please?", false);
yInfo() << "iCub says : Can you repeat your feedback please?";
}
/************************************ Retrieve Human spoken feedback ************************************/
bRecognized = iCub->getRecogClient()->recogFromGrammarLoop(grammarToString(nameGrammarHumanFeedback));
if (bRecognized.get(0).asInt() == 0)
{
return;
}
bAnswer = *bRecognized.get(1).asList();
// bAnswer is the result of the regognition system (first element is the raw sentence, 2nd is the list of semantic element)
if (bAnswer.get(0).asString() == "stop")
{
osError.str("");
osError << " | STOP called";
bOutput.addString(osError.str());
cout << osError.str() << endl;
}
yInfo() << "bRecognized " << bRecognized.toString();
cout << bRecognized.get(1).toString() << endl;
string sQuestionKind = bAnswer.get(1).asList()->get(0).toString();
//feedback 1-10
if (sQuestionKind == "FEEDBACK") {
yInfo() << "FEEDBACK received from Human!";
}
else {
yError() << " The sentence type is not recognized, waiting for FEEDBACK";
tryAgain = true;
nodeFeedback(tryAgain, bestTimeAndRank);
return;
}
// semantic is the list of the semantic elements of the sentence except the type ef sentence
bSemantic = *bAnswer.get(1).asList()->get(1).asList();
//feedback number 1-10
string sFeedback10 = bSemantic.check("feedback10", Value("0")).asString();
int iFeedback10 = atoi(sFeedback10.c_str());
//feedback likert 1-5 quality
/*string sFeedback10 = bAnswer.get(1).asList()->get(0).asString() ;
int iFeedback10 = atoi(sFeedback10.c_str())*2 ;*/
/************************************ Ask confirmation for the feedback (i.e. if misrecognition) ************************************/
osResponse.str("");
osResponse << "So for you, this kinematic structure has a score of " << iFeedback10 << ", right?";
iCub->say(osResponse.str().c_str(), false);
yInfo() << "iCub says : " << osResponse.str();
if (!nodeYesNo())
{
osResponse.str("");
osResponse << "Oups, I am sorry";
tryAgain = true;
iCub->say(osResponse.str().c_str(), false);
yInfo() << "iCub says : " << osResponse.str();
nodeFeedback(tryAgain, bestTimeAndRank);
return;
}
else {
tryAgain = false;
}
/************************************ put the feedback into ABM, feedback table ************************************/
list<pair<string, string> > lArgument;
lArgument.push_back(pair<string, string>("Bob", "agent"));
lArgument.push_back(pair<string, string>("kinematic structure", "about"));
iCub->getABMClient()->sendActivity("action", "sentence", "feedback", lArgument, true);
Bottle bResult;
ostringstream osRequest;
//only augmented_time is needed but better clarity for the print
osRequest << "SELECT instance FROM main WHERE activitytype = 'feedback' ORDER BY \"time\" DESC LIMIT 1 ;";
bResult = iCub->getABMClient()->requestFromString(osRequest.str().c_str());
int feedbackInstance = -1;
feedbackInstance = atoi(bResult.get(0).asList()->get(0).toString().c_str());
yInfo() << "Feedback instance stored in main (from Bottle) : " << bResult.get(0).asList()->get(0).toString().c_str();
yInfo() << "Feedback instance stored in main (from feedbackInstance) : " << feedbackInstance;
//insert the feedback to the SQL database
insertFeedback(iFeedback10, feedbackInstance);
/************************************ Update best feedback ************************************/
if (iFeedback10 > bestTimeAndRank.second) {
bestTimeAndRank.first = *it_augmentedTime;
bestTimeAndRank.second = iFeedback10;
osResponse.str("");
osResponse << "Yes, I have improved my skills : best rank is now " << bestTimeAndRank.second;
iCub->say(osResponse.str().c_str(), false);
yInfo() << "iCub says : " << osResponse.str();
}
else {
osResponse.str("");
osResponse << "Erf, too bad";
iCub->say(osResponse.str().c_str(), false);
yInfo() << "iCub says : " << osResponse.str();
}
//Check that we still have augmented feedback to do
if (++it_augmentedTime == vAugmentedTime.end()){
osResponse.str("");
osResponse << "I have no more augmented to check, thank you for your feedback";
iCub->say(osResponse.str().c_str(), false);
yInfo() << "iCub says : " << osResponse.str();
return;
}
osResponse.str("");
osResponse << "Another one?";
iCub->say(osResponse.str().c_str(), false);
yInfo() << "iCub says : " << osResponse.str();
if (nodeYesNo())
{
nodeFeedback(tryAgain, bestTimeAndRank);
return;
}
else
{
osResponse.str("");
osResponse << "Ok, thanks anyway, bye";
iCub->say(osResponse.str().c_str(), false);
yInfo() << "iCub says : " << osResponse.str();
//set back default value
bestTimeAndRank.first = "";
bestTimeAndRank.second = 0;
return;
}
}
