bool reservoirHandler::nodeYesNo()
{
sCurrentNode = "nodeYesNo";
sCurrentGrammarFile = sGrammarYesNo;
ostringstream osError; // Error message
osError << "Error in reservoirHandler | "<< sCurrentNode << " :: ";
cout << endl << "In " << sCurrentNode << endl << endl;
Bottle bOutput;
bool fGetaReply = false;
Bottle bMessenger,
bSpeechRecognized,
bAnswer;
bMessenger.addString("recog");
bMessenger.addString("grammarXML");
bMessenger.addString(grammarToString(sCurrentGrammarFile).c_str());
//to be replace by Say
while (!fGetaReply)
{
Port2SpeechRecog.write(bMessenger,bSpeechRecognized);
cout << "Reply from Speech Recog : " << bSpeechRecognized.toString() << endl;
if (bSpeechRecognized.toString() == "NACK" || bSpeechRecognized.size() != 3)
{
osError << "Check " << sCurrentGrammarFile;
bOutput.addString(osError.str());
cout << osError.str() << endl;
return false;
}
if (bSpeechRecognized.get(0).toString() == "0")
{
osError << "Grammar not recognized";
bOutput.addString(osError.str());
cout << osError.str() << endl;
return false;
}
bAnswer = *bSpeechRecognized.get(1).asList();
if (bAnswer.toString() != "" && !bAnswer.isNull())
{
fGetaReply = true;
}
}
return (bAnswer.get(0).asString() == "yes");
}
bool speechInteraction::updateModule()
{
//bool fGetaReply = false;
Bottle bSpeechRecognized, //recceived FROM speech recog with transfer information (1/0 (bAnswer) ACK/NACK)
bMessenger, //to be send TO speech recog
bAnswer, //response from speech recog without transfer information, including raw sentence
bSemantic, // semantic information of the content of the recognition
bSendReasoning, // send the information of recall to the abmReasoning
bSpeak, // bottle for tts
bTemp;
Bottle bOutput;
ostringstream osError; // Error message
Bottle bRecognized;
string inputString;
if( speechType == 0 )
{
Bottle *inputBottle = inputPort.read();
inputString = inputBottle->toString();
}
else if( speechType == 1 )
{
bRecognized = iCub->getRecogClient()->recogFromGrammarLoop(grammarToString(GrammarAskNamePerson), 20);
if( bRecognized.get(0).asInt() != 0 )
{
bAnswer = *bRecognized.get(1).asList();
inputString = bAnswer.get(0).asString();
}
}
cout << "RECEIVED TEXT: " << inputString << endl;
inputString.erase(remove(inputString.begin(), inputString.end(), '\"'), inputString.end());
int index;
if( matchVocab(inputString, &index) )
{
if( index == 15 )
triggerBehaviour(index);
else if( index == 17 )
triggerBehaviour(index);
else if( index == 19 ) // ID in C++ (-1 from config file = 20 )
triggerBehaviour(index);
else
sendSpeech(index);
}
return true;
}
/*
* Yes/No asking for confirmation, another trial, ...
*/
bool abmInteraction::nodeYesNo()
{
//bool fGetaReply = false;
Bottle bRecognized, //recceived FROM speech recog with transfer information (1/0 (bAnswer))
bAnswer, //response from speech recog without transfer information, including raw sentence
bSemantic, // semantic information of the content of the recognition
bSendReasoning, // send the information of recall to the abmReasoning
bMessenger; //to be send TO speech recog
string sYesNo = "unknown";
while (!((sYesNo == "yes") || (sYesNo == "no"))){
bRecognized = iCub->getRecogClient()->recogFromGrammarLoop(grammarToString(nameGrammarYesNo));
if (bRecognized.get(0).asInt() == 0)
{
cout << bRecognized.get(1).toString() << endl;
return false;
}
bAnswer = *bRecognized.get(1).asList();
// bAnswer is the result of the regognition system (first element is the raw sentence, 2nd is the list of semantic element)
yInfo() << " bAnswer : " << bRecognized.get(1).toString();
yInfo() << " Yes/No : " << bAnswer.get(1).asList()->get(0).asString();
sYesNo = bAnswer.get(1).asList()->get(0).asString();
if (sYesNo == "yes") return true;
else if (sYesNo == "no") return false;
//if here, recognition of the default background grammar : should repeat (with sending again grammar?) TOCHECK
iCub->say("I am sorry, can you repeat please?", false);
}
return false;
}
int main(int argc, char *argv[])
{
pANTLR3_UINT8 filename;
pANTLR3_INPUT_STREAM input=NULL;
pProtoJSLexer lxr;
pANTLR3_COMMON_TOKEN_STREAM tstream;
pProtoJSParser psr;
ProtoJSParser_protocol_return pbjAST;
if (argc < 2 || argv[1] == NULL)
filename = (pANTLR3_UINT8)"./input";
else
filename = (pANTLR3_UINT8)argv[1];
const char * outputFilename="output";
if (argc>=3) {
outputFilename=argv[2];
}
char * csOut=NULL;
char * cppOut=NULL;
char * cppInclude=NULL;
int argindex;
const char *outputInternalNamespace="_ProtoJS_Internal";
const char *outputExternalNamespace="";
for (argindex=3;argindex<argc;++argindex) {
if (strncmp(argv[argindex],"--cpp=",6)==0) {
cppOut=argv[argindex]+6;
}
if (strncmp(argv[argindex],"--cs=",5)==0) {
csOut=argv[argindex]+5;
}
if (strncmp(argv[argindex],"--include=",10)==0) {
cppInclude=argv[argindex]+10;
}
if (strncmp(argv[argindex],"--inamespace=",13)==0) {
outputInternalNamespace=argv[argindex]+13;
}
if (strncmp(argv[argindex],"--namespace=",12)==0) {
outputExternalNamespace=argv[argindex]+12;
}
}
char*package=parsePackage((const char*)filename,outputFilename,outputInternalNamespace,outputExternalNamespace);
pANTLR3_HASH_TABLE qualifiedTypes=antlr3HashTableNew(11);
parseTypes((const char*)filename,outputFilename,outputInternalNamespace,outputExternalNamespace,package,qualifiedTypes);
if(generateASTProto((const char*)filename,outputFilename,outputInternalNamespace,outputExternalNamespace,package,qualifiedTypes,&pbjAST,&lxr,&psr,&tstream,&input)) {
pANTLR3_COMMON_TREE_NODE_STREAM nodes;
nodes = antlr3CommonTreeNodeStreamNewTree(pbjAST.tree, ANTLR3_SIZE_HINT); // sIZE HINT WILL SOON BE DEPRECATED!!
pANTLR3_STRING s = nodes->stringFactory->newRaw(nodes->stringFactory);
grammarToString(nodes->tnstream,nodes->root,NULL,s);
FILE*fp=fopen(outputFilename,"w");
if (!fp) {
perror("Unable to open output file!");
exit(2);
}
if (s->size>1)
fwrite(s->chars,s->size-1,1,fp);
fclose(fp);
stringFree(s);
nodes ->free (nodes); nodes = NULL;
}
psr->free(psr);
psr = NULL;
tstream->free(tstream);
tstream = NULL;
lxr->free(lxr);
lxr = NULL;
input->close(input);
input = NULL;
return 0;
}
/* Called periodically every getPeriod() seconds */
bool ears::updateModule() {
LockGuard lg(mutex);
if (bShouldListen)
{
yDebug() << "bListen";
Bottle bRecognized, //recceived FROM speech recog with transfer information (1/0 (bAnswer))
bAnswer, //response from speech recog without transfer information, including raw sentence
bSemantic; // semantic information of the content of the recognition
bRecognized = iCub->getRecogClient()->recogFromGrammarLoop(grammarToString(MainGrammar), 1, true);
//bShouldListen=true;
if (bRecognized.get(0).asInt() == 0)
{
yWarning() << " error in ears::updateModule | Error in speechRecog";
return true;
}
bAnswer = *bRecognized.get(1).asList();
if (bAnswer.get(0).asString() == "stop")
{
yInfo() << " in abmHandler::node1 | stop called";
return true;
}
// bAnswer is the result of the regognition system (first element is the raw sentence, 2nd is the list of semantic element)
bSemantic = *(*bAnswer.get(1).asList()).get(1).asList();
cout << bSemantic.toString() << endl;
string sObject, sAction;
string sQuestionKind = bAnswer.get(1).asList()->get(0).toString();
//string sPredicate = bSemantic.check("predicate", Value("none")).asString();
string sObjectType, sCommand;
if(sQuestionKind == "SENTENCEOBJECT") {
sObject = bSemantic.check("object", Value("none")).asString();
sAction = bSemantic.check("predicateObject", Value("none")).asString();
sCommand = "followingOrder";
sObjectType = "object";
} else if(sQuestionKind == "SENTENCEBODYPART") {
sObject = bSemantic.check("bodypart", Value("none")).asString();
sCommand = "touchingOrder";
sObjectType = "bodypart";
} else if(sQuestionKind == "SENTENCENARRATIVE") {
sCommand = "followingOrder";
sAction = "narrate";
sObjectType = "";
sObject = "";
}else{
yError() << "[ears] Unknown predicate";
}
Bottle &bToTarget = portTarget.prepare();
bToTarget.clear();
bToTarget.addString(sAction);
bToTarget.addString(sObjectType);
bToTarget.addString(sObject);
portTarget.write();
Bottle bCondition;
bCondition.addString(sCommand);
//bCondition.addString(sAction);
bCondition.addString(sObjectType);
bCondition.addString(sObject);
portToBehavior.write(bCondition);
yDebug() << "Sending " + bCondition.toString();
} else {
yDebug() << "Not bListen";
}
return true;
}
int LanguageActionAnalysis::languageNode()
{
//iquestion = 3;
//return grammarNode();
sCurrentNode = "nodeQuestion";
sCurrentGrammarFile = nameGrammarNode;
ostringstream osError; // Error message
osError << "Error in LanguageActionAnalysis | "<< sCurrentNode << " :: ";
cout << endl << "In " << sCurrentNode << endl << endl;
Bottle bOutput;
bool fGetaReply = false;
Bottle bSpeechRecognized, //recceived FROM speech recog with transfer information (1/0 (bAnswer) ACK/NACK)
bMessenger, //to be send TO speech recog
bAnswer, //response from speech recog without transfer information, including raw sentence
bSemantic; // semantic information of the content of the recognition
bMessenger.addString("recogBottle");
bMessenger.addString("grammarXML");
bMessenger.addString(grammarToString(sCurrentGrammarFile).c_str());
while (!fGetaReply)
{
bSpeechRecognized.clear();
Port2SpeechRecog.write(bMessenger,bSpeechRecognized);
//e.g. : Reply from Speech Recog : 1 ("I want you to produce language" (INFORMATION (type produce))) ACK
cout << "Reply from Speech Recog : " << bSpeechRecognized.toString() << endl;
if (bSpeechRecognized.toString() == "NACK" || bSpeechRecognized.size() != 3)
{
osError << "Check " << sCurrentGrammarFile;
bOutput.addString(osError.str());
cout << osError.str() << endl;
//return bOutput;
}
if (bSpeechRecognized.get(0).toString() == "0")
{
osError << "Grammar not recognized";
bOutput.addString(osError.str());
cout << osError.str() << endl;
//return bOutput;
}
// bAnswer is the result of the regognition system (first element is the raw sentence, 2nd is the list of semantic element)
bAnswer = *bSpeechRecognized.get(1).asList();
if (bAnswer.toString() != "" && !bAnswer.isNull())
{
fGetaReply = true;
}
}
if (bAnswer.get(0).asString() == "stop")
{
iCurrentInstance = -1;
osError.str("");
osError << " | STOP called";
bOutput.addString(osError.str());
cout << osError.str() << endl;
//return bOutput;
}
bAnswer = *bSpeechRecognized.get(1).asList();
if (bAnswer.get(0).asString() == "next")
{
return mainNode();
}
if (bAnswer.get(0).asString() == "what happened")
{
iquestion = 1;
return grammarNode();
}
else if (bAnswer.get(1).asList()->get(0).asString() == "whoagent")
{
cout << "bAnswer.get(0).asString() : " << bAnswer.get(1).asList()->get(0).asString() << endl;
cout << "here " << bAnswer.get(1).asString() << endl;
iquestion = 2;
bSemantic = *bAnswer.get(1).asList()->get(1).asList();
sagent = bSemantic.check("agent", Value("none")).asString();
return grammarNode();
}
else if (bAnswer.get(0).asString() == "how did that happen")
{
iquestion = 3;
return grammarNode();
}
else if (bAnswer.get(0).asString() == "no")
{
iCub->say("go ahead");
}
return languageNode();
}
bool reservoirHandler::nodeTrainSD()
{
sCurrentNode = "nodeTrainSD";
sCurrentGrammarFile = nameGrammarNodeTrainSD;
ostringstream osError; // Error message
osError << "Error in reservoirHandler | "<< sCurrentNode << " :: ";
cout << endl << "In " << sCurrentNode << endl << endl;
Bottle bOutput;
bool fGetaReply = false;
Bottle bSpeechRecognized, //recceived FROM speech recog with transfer information (1/0 (bAnswer) ACK/NACK)
bMessenger, //to be send TO speech recog
bAnswer, //response from speech recog without transfer information, including raw sentence
bSemantic, // semantic information of the content of the recognition
bSendReasoning, // send the information of recall to the abmReasoning
bSpeak, // bottle for tts
bTemp;
bMessenger.addString("recog");
bMessenger.addString("grammarXML");
bMessenger.addString(grammarToString(sCurrentGrammarFile).c_str());
//to be replace by Say
while (!fGetaReply)
{
Port2SpeechRecog.write(bMessenger,bSpeechRecognized);
cout << "Reply from Speech Recog : " << bSpeechRecognized.toString() << endl;
if (bSpeechRecognized.toString() == "NACK" || bSpeechRecognized.size() != 3)
{
osError << "Check " << sCurrentGrammarFile;
bOutput.addString(osError.str());
cout << osError.str() << endl;
return false;
}
if (bSpeechRecognized.get(0).toString() == "0")
{
osError << "Grammar not recognized";
bOutput.addString(osError.str());
cout << osError.str() << endl;
return false;
}
bAnswer = *bSpeechRecognized.get(1).asList();
if (bAnswer.toString() != "" && !bAnswer.isNull())
{
fGetaReply = true;
}
}
bSemantic = *bAnswer.get(1).asList()->get(1).asList();
string sQuestionKind = bAnswer.get(1).asList()->get(0).toString();
if(sCurrentType == "train")
{
// Do you know any ...
if (sQuestionKind == "sentence")
{
cout << "Human : 'Let me see, I have finished...'" << endl;
cout << "iCub says : 'What is the corresponding sentence '" << bAnswer.get(0).asString() << " " << bAnswer.toString() << endl ;
/*
* bAnswer.get(0).asString() => the circle is to the left of of the cross
* bAnswer.toString() =>
* "the circle is to the left of of the cross" (sentence (sentence1 ((object "the circle") (relative_complete ((spatial_relative ((relative to) (spatial "the left"))) (object "the cross"))))))
*/
cout << "iCub says : 'I have recognized '" << bAnswer.get(0).asString() << endl ;
cout << "iCub says : 'Is it ok ? ... 'continue or exit" << endl;
string sSentence = bAnswer.get(0).asString();
nodeTrainSD();
}
else if (sQuestionKind == "follow")
{
string continueExit = bSemantic.check("mode", Value("none")).asString();
if (continueExit == "continue the interaction")
{
cout << "lMeaningsSentences " << endl;
lMeaningsSentences.push_back(sSentence+ sSentence_type);
nodeTrainSD();
}
else if (continueExit == "exit")
{
trainSaveMeaningSentence(fileSRinputM.c_str());
nodeType();
}
}
}
else if (sCurrentType == "test")
{
iCub->say("Ok, Set your initial situation, and show me your object of focus !", false);
while (!nodeYesNo())
{}
iCub->opc->update();
std::list<Entity*> PresentObjects = iCub->opc->EntitiesCache();
std::vector<RTObject> PresentRtoBefore;
for(std::list<Entity*>::iterator itE = PresentObjects.begin() ; itE != PresentObjects.end(); itE++)
{
if ((*itE)->isType(EFAA_OPC_ENTITY_RTOBJECT))
{
RTObject rto;
rto.fromBottle((*itE)->asBottle());
if (rto.m_present)PresentRtoBefore.push_back(rto) ;
}
}
if (PresentObjects.size() < 2 && PresentObjects.size() > 3)
{
iCub->say("Dude, I was expecting 2 or 3 objects... Star again !",false);
return nodeTrainSD();
}
//get the focus object
double maxSalience = 0;
string sObjectFocus = "none";
for (std::vector<RTObject>::iterator itRTO = PresentRtoBefore.begin() ; itRTO != PresentRtoBefore.end() ; itRTO++)
{
if (itRTO->m_saliency > maxSalience)
{
maxSalience = itRTO->m_saliency;
sObjectFocus = itRTO->name();
}
}
if (maxSalience == 0.)
{
iCub->say("I think I didn't get your focus object dude...",false);
return nodeTrainSD();
}
string sSentence = "Ok, so you decided to focus on " + sObjectFocus;
iCub->say(sSentence,false);
if (PresentRtoBefore.size()==2)
{
double deltaX = 0.0;
double deltaY = 0.0;
int iFactor;
(PresentRtoBefore[0].name() == sObjectFocus) ? iFactor = 1 : iFactor = -1;
deltaX = iFactor*(PresentRtoBefore[1].m_ego_position[0] - PresentRtoBefore[0].m_ego_position[0]);
deltaY = iFactor*(PresentRtoBefore[1].m_ego_position[1] - PresentRtoBefore[0].m_ego_position[1]);
string sLocation;
(deltaY>0)? sLocation = "right" : sLocation = "left";
string sRelative;
(iFactor==1)? sRelative = (PresentRtoBefore[1].name()) : sRelative =(PresentRtoBefore[0].name());
cout << "I understood :" << endl << sObjectFocus << "\t" << sLocation << "\t" << sRelative << endl ;
//TODO send to xavier get response
}
else // case of 3 objects
{
RTObject rtFocus,
rtRelative1,
rtRelative2;
bool bFirstRelative = true;
for (unsigned int i = 0 ; i < 3 ; i++)
{
if (PresentRtoBefore[i].name() != sObjectFocus )
{
bFirstRelative? rtRelative1 = PresentRtoBefore[i] : rtRelative2 =PresentRtoBefore[i];
bFirstRelative = false;
}
else
{
rtFocus = PresentRtoBefore[i];
}
}
iCub->say("Thinking of the situation",false);
double deltaX1 ; // difference btw focus and relative1
double deltaX2 ; // difference btw focus and relative2
deltaX1 = rtRelative1.m_ego_position[1] - rtFocus.m_ego_position[1];
deltaX2 = rtRelative2.m_ego_position[1] - rtFocus.m_ego_position[1];
string sLocation1;
string sLocation2;
string sRelative1 = rtRelative1.name();
string sRelative2 = rtRelative2.name();
(deltaX1>0)? sLocation1 = "right" : sLocation1 = "left";
(deltaX2>0)? sLocation2 = "right" : sLocation2 = "left";
cout << "I understood : " << sLocation1 << "\t" << sObjectFocus << "\t" << sRelative1 << endl;
cout << "and : " << sLocation2 << "\t" << sObjectFocus << "\t" << sRelative2 << endl;
if (sSentence_type.size()==0)
{
sSentence_type = " :C";
}
sdataTestSD = sLocation1 + " " + sObjectFocus + " " + sRelative1 + ", " + sLocation2 + " " + sObjectFocus + " " + sRelative2 + sSentence_type;
// TO SEND TO XAVIER
copyPastFile(fileXavierTrain.c_str(), fileSRinputM.c_str());
cout << fileXavierTrain << endl;
cout << fileSRinputM << endl;
createTestwithTrainData(fileSRinputM.c_str(), sdataTestSD);
callReservoir(pythonPath + fileSD);
string result = openResult(fileSRoutputS.c_str());
iCub->say(result,false);
cout << "iCub says : 'I have understood '" << result << endl ;
}
return nodeType();
}
if (bAnswer.get(0).asString() == "return the interaction")
{
nodeModality();
}
return true;
}
bool reservoirHandler::nodeTestAP()
{
sCurrentNode = "nodeTestAP";
sCurrentGrammarFile = nameGrammarNodeTestAP;
ostringstream osError; // Error message
osError << "Error in reservoirHandler | "<< sCurrentNode << " :: ";
Bottle bOutput;
bool fGetaReply = false;
Bottle bSpeechRecognized, //recceived FROM speech recog with transfer information (1/0 (bAnswer) ACK/NACK)
bMessenger, //to be send TO speech recog
bAnswer, //response from speech recog without transfer information, including raw sentence
bSemantic, // semantic information of the content of the recognition
bSendReasoning, // send the information of recall to the abmReasoning
bSpeak, // bottle for tts
bTemp;
bMessenger.addString("recog");
bMessenger.addString("grammarXML");
bMessenger.addString(grammarToString(sCurrentGrammarFile).c_str());
//to be replace by Say
while (!fGetaReply)
{
Port2SpeechRecog.write(bMessenger,bSpeechRecognized);
cout << "Reply from Speech Recog : " << bSpeechRecognized.toString() << endl;
if (bSpeechRecognized.toString() == "NACK" || bSpeechRecognized.size() != 3)
{
osError << "Check " << sCurrentGrammarFile;
bOutput.addString(osError.str());
cout << osError.str() << endl;
return false;
}
if (bSpeechRecognized.get(0).toString() == "0")
{
osError << "Grammar not recognized";
bOutput.addString(osError.str());
cout << osError.str() << endl;
return false;
}
bAnswer = *bSpeechRecognized.get(1).asList();
if (bAnswer.toString() != "" && !bAnswer.isNull())
{
fGetaReply = true;
}
}
if (bAnswer.get(0).asString() == "stop the interaction")
{
iCurrentInstance = -1;
osError.str("");
osError << " | STOP called";
bOutput.addString(osError.str());
cout << osError.str() << endl;
return false;
}
bSemantic = *bAnswer.get(1).asList()->get(1).asList();
string sQuestionKind = bAnswer.get(1).asList()->get(0).toString();
cout << "I'm here ...................." << endl;
iCub->say("Say a sentence",false);
// Do you know any ...
if (sQuestionKind == "sentence")
{
/*
* bAnswer.get(0).asString() => the circle is to the left of of the cross
* bAnswer.toString() =>
* "the circle is to the left of of the cross" (sentence (sentence1 ((object "the circle") (relative_complete ((spatial_relative ((relative to) (spatial "the left"))) (object "the cross"))))))
*/
cout << "iCub says : 'I have understood '" << bAnswer.get(0).asString() << endl ;
iCub->say("I have understood ....", false);
sentence += bAnswer.get(0).asString() + " ";
iCub->say(bAnswer.get(0).asString(), false);
iCub->say("Is it ok ?", false);
cout << "iCub says : 'Is it ok ? ... 'No or Yes" << endl;
return nodeTestAP();
}
else if (sQuestionKind == "yesno")
{
string yesNo = bSemantic.check("agree", Value("none")).asString();
if (yesNo == "yes")
{
copyPastFile(fileXavierTrainAP.c_str(), fileAPimputS.c_str());
cout << fileXavierTrainAP << endl;
cout << fileAPimputS << endl;
createTestwithTrainData(fileAPimputS.c_str(), sentence);
callReservoir(fileAPimputS);
string result = openResult(fileAPoutputM.c_str());
iCub->say(result,false);
cout << result << endl;
int id = result.find(",");
int idf = result.size()-id;
cout << id << endl;
if (id != 0)
{
cout << result.substr(0, id) << endl;
cout << result.substr(id +1,idf) << endl;
string firstCommand = result.substr(0, id);
string secondCommand = result.substr(id +1,idf);
iCub->say("I will do the actions", false);
AREactions(extractVocabulary(firstCommand));
extractVocabulary(secondCommand);
AREactions(extractVocabulary(secondCommand));
return nodeTestAP();
}
else
{
iCub->say("I will do the actions", false);
AREactions(extractVocabulary(result));
sentence = " ";
return nodeTestAP();
}
cout << "iCub do the action..." << endl;
}
else if(yesNo == "no")
{
sentence = " ";
}
else if (sQuestionKind == "follow")
{
string continueExit = bSemantic.check("mode", Value("none")).asString();
if (continueExit == "continue the interaction")
{
cout << "Humain say a command ...." << endl;
sentence = " ";
return nodeTestAP();
}
else if (continueExit == "exit")
{
return nodeType();
}
}
}
return true;
}
bool reservoirHandler::nodeTrainAP()
{
sCurrentNode = "nodeModality";
sCurrentGrammarFile = nameGrammarNodeModality;
ostringstream osError; // Error message
osError << "Error in reservoirHandler | "<< sCurrentNode << " :: ";
Bottle bOutput;
bool fGetaReply = false;
Bottle bSpeechRecognized, //recceived FROM speech recog with transfer information (1/0 (bAnswer) ACK/NACK)
bMessenger, //to be send TO speech recog
bAnswer, //response from speech recog without transfer information, including raw sentence
bSemantic, // semantic information of the content of the recognition
bSendReasoning, // send the information of recall to the abmReasoning
bSpeak, // bottle for tts
bTemp;
bMessenger.addString("recog");
bMessenger.addString("grammarXML");
bMessenger.addString(grammarToString(sCurrentGrammarFile).c_str());
//to be replace by Say
string sSentence = "You want to know more about it ?";
iCub->say(sSentence,false);
while (!fGetaReply)
{
Port2SpeechRecog.write(bMessenger,bSpeechRecognized);
cout << "Reply from Speech Recog : " << bSpeechRecognized.toString() << endl;
if (bSpeechRecognized.toString() == "NACK" || bSpeechRecognized.size() != 3)
{
osError << "Check " << sCurrentGrammarFile;
bOutput.addString(osError.str());
cout << osError.str() << endl;
return false;
}
if (bSpeechRecognized.get(0).toString() == "0")
{
osError << "Grammar not recognized";
bOutput.addString(osError.str());
cout << osError.str() << endl;
return false;
}
bAnswer = *bSpeechRecognized.get(1).asList();
if (bAnswer.toString() != "" && !bAnswer.isNull())
{
fGetaReply = true;
}
}
if (bAnswer.get(0).asString() == "stop the interaction")
{
iCurrentInstance = -1;
osError.str("");
osError << " | STOP called";
bOutput.addString(osError.str());
cout << osError.str() << endl;
return false;
}
bSemantic = *bAnswer.get(1).asList()->get(1).asList();
string sQuestionKind = bAnswer.get(1).asList()->get(0).toString();
cout << "iCub execute an action...'" << endl;
cout << "iCub says : 'What is this action ? '" << endl;
// Do you know any ...
if (sQuestionKind == "sentence")
{
/*
* bAnswer.get(0).asString() => the circle is to the left of of the cross
* bAnswer.toString() =>
* "the circle is to the left of of the cross" (sentence (sentence1 ((object "the circle") (relative_complete ((spatial_relative ((relative to) (spatial "the left"))) (object "the cross"))))))
*/
cout << "Human says : " << bAnswer.get(0).asString() << endl ;
cout << "Human says : 'Do you want continue or exit'" << endl ;
iCub->say("Do you want continue or exit",false);
sSentence = bAnswer.get(0).asString();
cout << "sSentence" << sSentence << endl;
return nodeTrainAP();
}
else if (sQuestionKind == "follow")
{
string continueExit = bSemantic.check("mode", Value("none")).asString();
if (continueExit == "continue the interaction")
{
cout << "lMeaningsSentences " << endl;
lMeaningsSentences.push_back(sSentence);
return nodeTrainAP();
}
else if (continueExit == "exit")
{
trainSaveMeaningSentence(fileAPimputS.c_str());
return nodeType();
}
}
return true;
}
bool reservoirHandler::nodeModality()
{
sCurrentNode = "nodeModality";
sCurrentGrammarFile = nameGrammarNodeModality;
ostringstream osError; // Error message
osError << "Error in reservoirHandler | "<< sCurrentNode << " :: ";
Bottle bOutput;
bool fGetaReply = false;
Bottle bSpeechRecognized, //recceived FROM speech recog with transfer information (1/0 (bAnswer) ACK/NACK)
bMessenger, //to be send TO speech recog
bAnswer, //response from speech recog without transfer information, including raw sentence
bSemantic, // semantic information of the content of the recognition
bSendReasoning, // send the information of recall to the abmReasoning
bSpeak, // bottle for tts
bTemp;
bMessenger.addString("recog");
bMessenger.addString("grammarXML");
bMessenger.addString(grammarToString(sCurrentGrammarFile).c_str());
//to be replace by Say
while (!fGetaReply)
{
Port2SpeechRecog.write(bMessenger,bSpeechRecognized);
cout << "Reply from Speech Recog : " << bSpeechRecognized.toString() << endl;
if (bSpeechRecognized.toString() == "NACK" || bSpeechRecognized.size() != 3)
{
osError << "Check " << sCurrentGrammarFile;
bOutput.addString(osError.str());
cout << osError.str() << endl;
return false;
}
if (bSpeechRecognized.get(0).toString() == "0")
{
osError << "Grammar not recognized";
bOutput.addString(osError.str());
cout << osError.str() << endl;
return false;
}
bAnswer = *bSpeechRecognized.get(1).asList();
if (bAnswer.toString() != "" && !bAnswer.isNull())
{
fGetaReply = true;
}
}
if (bAnswer.get(0).asString() == "stop the interaction")
{
iCurrentInstance = -1;
osError.str("");
osError << " | STOP called";
bOutput.addString(osError.str());
cout << osError.str() << endl;
return false;
}
bSemantic = *bAnswer.get(1).asList()->get(1).asList();
string sQuestionKind = bAnswer.get(1).asList()->get(0).toString();
if (sQuestionKind == "CANONICAL")
{
sCurrentCanonical = bSemantic.check("focus", Value("none")).asString();
if (sCurrentCanonical == "object")
{
cout << "iCub says : 'Fine. Let's focus about object'" << endl ;
sSentence_type = " :C";
}
else if(sCurrentCanonical == "location")
{
cout << "iCub says : 'Oh, tricky! Let's go with locations'" << endl ;
sSentence_type = " :N";
}
//copyTrainData(fileXavierTrain.c_str(),fileSRinputM.c_str());
cout << "iCub says : 'I have understood'" << endl ;
nodeType();
}
if (sQuestionKind == "INFORMATION")
{
sCurrentType = bSemantic.check("modality", Value("none")).asString();
if (sCurrentActivity == "understand")
{
/* Mode Action Performer => Meaning */
// Module test_mode
iCub->say("Let me see",false);
cout << "iCub says : 'Let me see...'" << endl ;
iCub->say("I see all the objects",false);
cout << "iCub says : 'I see all the objects'" << endl ;
if (sCurrentType == "train")
{
/*
* Training
* 1. Robot generates random actions [meaning]
* 2. Human says a corresponding command [sentence]
*/
if (lMeaningsSentences.size()!=0)
{
lMeaningsSentences.clear();
}
cout << "iCub says : 'What is this action ?'" << endl;
if (sCurrentCanonical.length() != 0)
{
return nodeTrainAP();
}
else
{
return nodeModality();
}
}
else if(sCurrentType == "test")
{
cout << "I am here" << endl;
/*
* Testing
* 1. Human says a command [sentence]
* 2. Robot performs corresponding actions [meaning]
*/
iCub->say("Go in test mode", false);
cout << "go on test mode " << endl;
nodeTestAP();
}
}
else if (sCurrentActivity == "produce")
{
/* Mode Scene Describer => Produce sentence*/
if (sCurrentType == "test")
{
/*
* Testing
* 1. Human arranges objects on the table [meaning]
* 2. Robot describes the scene [sentence]
*/
iCub->say("Do you want me to focus the description about object or location ?", false);
inbsentence=2;
return nodeTrainSD();
}
else if (sCurrentType == "train")
{
/*
* Training
* 1. Human arranges objects on the table [meaning]
* 2. Human describes the scene [sentence]
*/
if (lMeaningsSentences.size()!=0)
{
lMeaningsSentences.clear();
}
inbsentence=1;
return nodeTrainSD();
}
cout << "iCub says : 'The focus object is $OBJ_FOCUS'" << endl ;
}
}
// bAnswer is the result of the regognition system (first element is the raw sentence, 2nd is the list of semantic element)
if (bAnswer.get(0).asString() == "stop the interaction")
{
iCurrentInstance = -1;
osError.str("");
osError << " | STOP called";
bOutput.addString(osError.str());
cout << osError.str() << endl;
return false;
}
if (bAnswer.get(0).asString() == "return the interaction")
{
return nodeModality();
}
return true;
}
/* Node 1: general question
* produce or understand
*/
bool reservoirHandler::nodeType()
{
sCurrentNode = "nodeType";
sCurrentGrammarFile = nameGrammarNodeType;
ostringstream osError; // Error message
osError << "Error in reservoirHandler | "<< sCurrentNode << " :: ";
cout << endl << "In " << sCurrentNode << endl << endl;
Bottle bOutput;
bool fGetaReply = false;
Bottle bSpeechRecognized, //recceived FROM speech recog with transfer information (1/0 (bAnswer) ACK/NACK)
bMessenger, //to be send TO speech recog
bAnswer, //response from speech recog without transfer information, including raw sentence
bSemantic, // semantic information of the content of the recognition
bSendReasoning, // send the information of recall to the abmReasoning
bSpeak, // bottle for tts
bTemp;
bMessenger.addString("recog");
bMessenger.addString("grammarXML");
bMessenger.addString(grammarToString(sCurrentGrammarFile).c_str());
//to be replace by Say
cout << "ICUB SAY : Do you want me to understand or produce language? " << endl;
while (!fGetaReply)
{
bSpeechRecognized.clear();
Port2SpeechRecog.write(bMessenger,bSpeechRecognized);
//e.g. : Reply from Speech Recog : 1 ("I want you to produce language" (INFORMATION (type produce))) ACK
cout << "Reply from Speech Recog : " << bSpeechRecognized.toString() << endl;
if (bSpeechRecognized.toString() == "NACK" || bSpeechRecognized.size() != 3)
{
osError << "Check " << sCurrentGrammarFile;
bOutput.addString(osError.str());
cout << osError.str() << endl;
return false;
}
if (bSpeechRecognized.get(0).toString() == "0")
{
osError << "Grammar not recognized";
bOutput.addString(osError.str());
cout << osError.str() << endl;
return false;
}
bAnswer = *bSpeechRecognized.get(1).asList();
if (bAnswer.toString() != "" && !bAnswer.isNull())
{
fGetaReply = true;
}
}
// bAnswer is the result of the regognition system (first element is the raw sentence, 2nd is the list of semantic element)
if (bAnswer.get(0).asString() == "stop the interaction")
{
iCurrentInstance = -1;
osError.str("");
osError << " | STOP called";
bOutput.addString(osError.str());
cout << osError.str() << endl;
return false;
}
bSemantic = *bAnswer.get(1).asList()->get(1).asList();
string sQuestionKind = bAnswer.get(1).asList()->get(0).toString();
if (sQuestionKind == "INFORMATION")
{
sCurrentActivity = bSemantic.check("type", Value("none")).asString();
if (sCurrentActivity == "understand")
{
/*
* Mode Action Performer => Meaning
* Training
* 1. Robot generates random actions [meaning]
* 2. Human says a corresponding command [sentence]
* Testing
* 1. Human says a command [sentence]
* 2. Robot performs corresponding actions [meaning]
*/
cout << "iCub says : 'Set the objects'" << endl ;
return nodeModality();
}
else if (sCurrentActivity == "produce")
{
/*
* Mode Scene Describer => Produce
* Training
* 1. Human arranges objects on the table [meaning]
* 2. Human describes the scene [sentence]
* Testing
* 1. Human arranges objects on the table [meaning]
* 2. Robot describes the scene [sentence]
*/
nodeModality();
}
else
return nodeType();
}
return true;
}
/*
* Node to ask Human to give feedback on quality of augmented kinematic structure image
*/
void abmInteraction::nodeFeedback(bool tryAgain, pair<string,int> & bestTimeAndRank)
{
ostringstream osError; // Error message
Bottle bOutput;
Bottle bRecognized, //recceived FROM speech recog with transfer information (1/0 (bAnswer))
bAnswer, //response from speech recog without transfer information, including raw sentence
bSemantic, // semantic information of the content of the recognition
bMessenger; //to be send TO speech recog
ostringstream osResponse;
/************************************ Usual feedback : show an image and ask for feedback ************************************/
if (tryAgain == false){
yInfo() << " Current time = " << *it_augmentedTime;
iCub->say("Note this kinematic structure between 1 and 10 please", false);
yInfo() << " iCub says : Note this kinematic structure between 1 and 10 please";
//feedback likert 1-5
/*iCub->say("Note this kinematic structure, Likert 1-5 quality");
yInfo() << " iCub says : Note this kinematic structure, Likert 1-5 quality" ;*/
//Preparing bottle to trigger the augmenting remembering
Bottle bRpc, bSubRealtime, bSubAugmentedTimes;
bRpc.addString("triggerStreaming");
bRpc.addInt(rememberedInstance);
bSubRealtime.addString("realtime");
bSubRealtime.addInt(1);
bSubAugmentedTimes.addString("augmentedTimes");
bSubAugmentedTimes.addString(*it_augmentedTime);
//If we have a previously best rank
if (bestTimeAndRank.second != 0) {
osResponse.str("");
osResponse << "The current best structure is shown at left. The rank is " << bestTimeAndRank.second; //<< " for time = " << bestAugmentedTime ;
iCub->say(osResponse.str().c_str(), false);
yInfo() << "iCub says : " << osResponse.str();
bSubAugmentedTimes.addString(bestTimeAndRank.first);
}
//Ask for showing the current testing augmented + the best one if relevant
bRpc.addList() = bSubRealtime;
bRpc.addList() = bSubAugmentedTimes;
iCub->getABMClient()->rpcCommand(bRpc);
}
else {
iCub->say("Can you repeat your feedback please?", false);
yInfo() << "iCub says : Can you repeat your feedback please?";
}
/************************************ Retrieve Human spoken feedback ************************************/
bRecognized = iCub->getRecogClient()->recogFromGrammarLoop(grammarToString(nameGrammarHumanFeedback));
if (bRecognized.get(0).asInt() == 0)
{
return;
}
bAnswer = *bRecognized.get(1).asList();
// bAnswer is the result of the regognition system (first element is the raw sentence, 2nd is the list of semantic element)
if (bAnswer.get(0).asString() == "stop")
{
osError.str("");
osError << " | STOP called";
bOutput.addString(osError.str());
cout << osError.str() << endl;
}
yInfo() << "bRecognized " << bRecognized.toString();
cout << bRecognized.get(1).toString() << endl;
string sQuestionKind = bAnswer.get(1).asList()->get(0).toString();
//feedback 1-10
if (sQuestionKind == "FEEDBACK") {
yInfo() << "FEEDBACK received from Human!";
}
else {
yError() << " The sentence type is not recognized, waiting for FEEDBACK";
tryAgain = true;
nodeFeedback(tryAgain, bestTimeAndRank);
return;
}
// semantic is the list of the semantic elements of the sentence except the type ef sentence
bSemantic = *bAnswer.get(1).asList()->get(1).asList();
//feedback number 1-10
string sFeedback10 = bSemantic.check("feedback10", Value("0")).asString();
int iFeedback10 = atoi(sFeedback10.c_str());
//feedback likert 1-5 quality
/*string sFeedback10 = bAnswer.get(1).asList()->get(0).asString() ;
int iFeedback10 = atoi(sFeedback10.c_str())*2 ;*/
/************************************ Ask confirmation for the feedback (i.e. if misrecognition) ************************************/
osResponse.str("");
osResponse << "So for you, this kinematic structure has a score of " << iFeedback10 << ", right?";
iCub->say(osResponse.str().c_str(), false);
yInfo() << "iCub says : " << osResponse.str();
if (!nodeYesNo())
{
osResponse.str("");
osResponse << "Oups, I am sorry";
tryAgain = true;
iCub->say(osResponse.str().c_str(), false);
yInfo() << "iCub says : " << osResponse.str();
nodeFeedback(tryAgain, bestTimeAndRank);
return;
}
else {
tryAgain = false;
}
/************************************ put the feedback into ABM, feedback table ************************************/
list<pair<string, string> > lArgument;
lArgument.push_back(pair<string, string>("Bob", "agent"));
lArgument.push_back(pair<string, string>("kinematic structure", "about"));
iCub->getABMClient()->sendActivity("action", "sentence", "feedback", lArgument, true);
Bottle bResult;
ostringstream osRequest;
//only augmented_time is needed but better clarity for the print
osRequest << "SELECT instance FROM main WHERE activitytype = 'feedback' ORDER BY \"time\" DESC LIMIT 1 ;";
bResult = iCub->getABMClient()->requestFromString(osRequest.str().c_str());
int feedbackInstance = -1;
feedbackInstance = atoi(bResult.get(0).asList()->get(0).toString().c_str());
yInfo() << "Feedback instance stored in main (from Bottle) : " << bResult.get(0).asList()->get(0).toString().c_str();
yInfo() << "Feedback instance stored in main (from feedbackInstance) : " << feedbackInstance;
//insert the feedback to the SQL database
insertFeedback(iFeedback10, feedbackInstance);
/************************************ Update best feedback ************************************/
if (iFeedback10 > bestTimeAndRank.second) {
bestTimeAndRank.first = *it_augmentedTime;
bestTimeAndRank.second = iFeedback10;
osResponse.str("");
osResponse << "Yes, I have improved my skills : best rank is now " << bestTimeAndRank.second;
iCub->say(osResponse.str().c_str(), false);
yInfo() << "iCub says : " << osResponse.str();
}
else {
osResponse.str("");
osResponse << "Erf, too bad";
iCub->say(osResponse.str().c_str(), false);
yInfo() << "iCub says : " << osResponse.str();
}
//Check that we still have augmented feedback to do
if (++it_augmentedTime == vAugmentedTime.end()){
osResponse.str("");
osResponse << "I have no more augmented to check, thank you for your feedback";
iCub->say(osResponse.str().c_str(), false);
yInfo() << "iCub says : " << osResponse.str();
return;
}
osResponse.str("");
osResponse << "Another one?";
iCub->say(osResponse.str().c_str(), false);
yInfo() << "iCub says : " << osResponse.str();
if (nodeYesNo())
{
nodeFeedback(tryAgain, bestTimeAndRank);
return;
}
else
{
osResponse.str("");
osResponse << "Ok, thanks anyway, bye";
iCub->say(osResponse.str().c_str(), false);
yInfo() << "iCub says : " << osResponse.str();
//set back default value
bestTimeAndRank.first = "";
bestTimeAndRank.second = 0;
return;
}
}
