bool IndexData<DataSamples, LabelVector>::LabelCopyAtIndex(LabelVector &target,
size_t topos,
size_t frompos) {
if ((frompos < labels_->rows()) && (topos < target.rows())) {
target.row(topos) = labels_->row(frompos);
return true;
} else {
return false;
}
}
int Assembler::AddLabel(string& name, LabelVector& labels)
{
int index = 0;
LabelVector::iterator i;
for (i = labels.begin(); i != labels.end(); i ++, index ++) {
if(i->label == name)
return index;
}
labels.push_back(Label(name));
return index;
}
bool Assembler::SetLabels(Assembly& assembly, LabelVector& labels)
{
dword *bytecode = assembly.bytecode;
dword curpos = assembly.curpos;
size_t size = labels.size();
for(dword *cp = bytecode; cp < bytecode + curpos; cp += 2) {
if(IsJumpOp(cp[0])) {
if(cp[1] < size) {
dword pos = labels.at(cp[1]).pos;
if(pos != -1)
cp[1] = pos;
else {
// this is a common error
printf("Label not found: %s\n", labels.at(cp[1]).label.c_str());
labels.clear();
return false;
}
} else {
// if we get here, there's a bug in the assembler ;(
printf("Label index not found: %d\n", cp[1]);
labels.clear();
return false;
}
}
}
labels.clear();
return true;
}
bool Assembler::ParseLine(string& line, int linecount, Assembly& assembly, LabelVector& labels)
{
string op;
int i = NextWord(line, op);
// skip commented and empty lines
if(i == -1 || op == "//" || op == ";") return true;
// this is a a very slow way of doing things, but then this code will only
// be used during development and for debugging
OpDesc *opcode;
int codecount = sizeof(opcodes)/sizeof(OpDesc);
int j = 0;
for(; j < codecount; j ++) {
if(stricmp(opcodes[j].name, op.c_str()) == 0) {
opcode = &opcodes[j];
break;
}
}
// is this a label?
if(j == codecount) {
size_t k = op.length() - 1;
if(op.find_first_of(":") == k) {
string name = op.substr(0, k);
labels.at(AddLabel(name, labels)).pos = assembly.curpos;
return true;
} else
return false;
} else {
assembly.WriteDword(opcode->code);
string param;
i = NextWord(line, param, i);
if(IsJumpOp(opcode->code)) {
// write the label index, we'll later replace this with the code position
assembly.WriteDword(AddLabel(param, labels));
} else {
if(i == -1) {
assembly.WriteDword(0);
if(opcode->paramcount > 0) return false; // if we need more params, bail
} else
assembly.WriteDword(StringToOperand(param, opcode));
}
}
return true;
}
void IndexData<DataSamples, LabelVector>::ResizeLabel(LabelVector &target,
size_t samplecnt) {
target.resize(samplecnt);
}
static void expandLabels(Bundle& bundle, size_t beamSize)
{
auto& outputActivationsVector = bundle["outputActivations"].get<MatrixVector>();
auto& outputActivations = outputActivationsVector.back();
auto& labels = bundle["referenceLabels"].get<LabelVector>();
auto& inputTimesteps = bundle["inputTimesteps"].get<IndexVector>();
size_t characters = outputActivations.size().front();
size_t miniBatchSize = outputActivations.size()[outputActivations.size().size() - 2];
size_t maxTimesteps = outputActivations.size()[outputActivations.size().size() - 1];
size_t originalMiniBatchSize = miniBatchSize / beamSize;
LabelVector expandedLabels;
IndexVector expandedTimesteps;
for(size_t miniBatch = 0; miniBatch < originalMiniBatchSize; ++miniBatch)
{
for(size_t beam = 0; beam < beamSize; ++beam)
{
expandedLabels.push_back(labels[miniBatch]);
expandedTimesteps.push_back(inputTimesteps[miniBatch]);
}
}
labels = std::move(expandedLabels);
inputTimesteps = std::move(expandedTimesteps);
auto referenceActivations = matrix::zeros({characters, miniBatchSize, maxTimesteps},
outputActivations.precision());
for(size_t miniBatch = 0; miniBatch < miniBatchSize; ++miniBatch)
{
for(size_t timestep = 0; timestep < maxTimesteps; ++timestep)
{
if(timestep < labels[miniBatch].size())
{
size_t character = labels[miniBatch][timestep];
assert(character < characters);
referenceActivations(character, miniBatch, timestep) = 1.0;
}
else
{
referenceActivations(characters - 1, miniBatch, timestep) = 1.0;
}
}
}
if(util::isLogEnabled("CTCDecoderLayer::Detail"))
{
util::log("CTCDecoderLayer::Detail") << " reference labels: \n";
for(auto& label : labels)
{
util::log("CTCDecoderLayer::Detail") << " reference label: "
<< util::toString(label) << "\n";
}
util::log("CTCDecoderLayer::Detail") << " reference activations: "
<< referenceActivations.debugString();
}
else
{
util::log("CTCDecoderLayer::Detail") << " reference labels size: "
<< labels.size() << "\n";
util::log("CTCDecoderLayer") << " reference activations size: "
<< outputActivations.shapeString() << "\n";
}
bundle["referenceActivations"] = MatrixVector({referenceActivations});
}
