void process(void) {
uint64_t num_errors = 0, num_valid_ops = 0, num_valid_bytes = 0;
entry entry;
int dump_version = processHeader();
/* Exclude the final checksum for RDB >= 5. Will be checked at the end. */
if (dump_version >= 5) {
if (positions[0].size < 8) {
printf("RDB version >= 5 but no room for checksum.\n");
exit(1);
}
positions[0].size -= 8;
}
level = 1;
while(positions[0].offset < positions[0].size) {
positions[1] = positions[0];
entry = loadEntry();
if (!entry.success) {
printValid(num_valid_ops, num_valid_bytes);
printErrorStack(&entry);
num_errors++;
num_valid_ops = 0;
num_valid_bytes = 0;
/* search for next valid entry */
uint64_t offset = positions[0].offset + 1;
int i = 0;
while (!entry.success && offset < positions[0].size) {
positions[1].offset = (size_t)offset; WIN_PORT_FIX /* cast (size_t) */
/* find 3 consecutive valid entries */
for (i = 0; i < 3; i++) {
entry = loadEntry();
if (!entry.success) break;
}
/* check if we found 3 consecutive valid entries */
if (i < 3) {
offset++;
}
}
/* print how many bytes we have skipped to find a new valid opcode */
if (offset < positions[0].size) {
printSkipped(offset - positions[0].offset, offset);
}
positions[0].offset = (size_t)offset; WIN_PORT_FIX /* cast (size_t) */
} else {
void process() {
uint64_t num_errors = 0, num_valid_ops = 0, num_valid_bytes = 0;
entry entry;
int dump_version = processHeader();
/* Exclude the final checksum for RDB >= 5. Will be checked at the end. */
if (dump_version >= 5) {
if (positions[0].size < 8) {
printf("RDB version >= 5 but no room for checksum.\n");
exit(1);
}
positions[0].size -= 8;;
}
level = 1;
while(positions[0].offset < positions[0].size) {
positions[1] = positions[0];
entry = loadEntry();
if (!entry.success) {
uint64_t offset;
int i;
printValid(num_valid_ops, num_valid_bytes);
printErrorStack(&entry);
num_errors++;
num_valid_ops = 0;
num_valid_bytes = 0;
/* search for next valid entry */
offset = positions[0].offset + 1;
i = 0;
while (!entry.success && offset < positions[0].size) {
positions[1].offset = (size_t)offset;
/* find 3 consecutive valid entries */
for (i = 0; i < 3; i++) {
entry = loadEntry();
if (!entry.success) break;
}
/* check if we found 3 consecutive valid entries */
if (i < 3) {
offset++;
}
}
/* print how many bytes we have skipped to find a new valid opcode */
if (offset < positions[0].size) {
printSkipped(offset - positions[0].offset, offset);
}
positions[0].offset = (size_t)offset;
} else {
num_valid_ops++;
num_valid_bytes += positions[1].offset - positions[0].offset;
/* advance position */
positions[0] = positions[1];
}
free(entry.key);
}
/* because there is another potential error,
* print how many valid ops we have processed */
printValid(num_valid_ops, num_valid_bytes);
/* expect an eof */
if (entry.type != REDIS_EOF) {
/* last byte should be EOF, add error */
errors.level = 0;
SHIFT_ERROR(positions[0].offset, "Expected EOF, got %s", types[entry.type]);
/* this is an EOF error so reset type */
entry.type = -1;
printErrorStack(&entry);
num_errors++;
}
/* Verify checksum */
if (dump_version >= 5) {
uint64_t crc = crc64(0,positions[0].data,positions[0].size);
uint64_t crc2;
unsigned char *p = (unsigned char*)positions[0].data+positions[0].size;
crc2 = ((uint64_t)p[0] << 0) |
((uint64_t)p[1] << 8) |
((uint64_t)p[2] << 16) |
((uint64_t)p[3] << 24) |
((uint64_t)p[4] << 32) |
((uint64_t)p[5] << 40) |
((uint64_t)p[6] << 48) |
((uint64_t)p[7] << 56);
if (crc != crc2) {
SHIFT_ERROR(positions[0].offset, "RDB CRC64 does not match.");
} else {
printf("CRC64 checksum is OK\n");
}
}
/* print summary on errors */
if (num_errors) {
printf("\n");
printf("Total unprocessable opcodes: %llu\n",
(unsigned long long) num_errors);
}
}
static void*
mainLoop(void* arg)
{
int *id = reinterpret_cast<int *>(arg);
PrintStack printStack;
for( ; ; )
{
SentRep* srp = new SentRep(params.maxSentLen);
pthread_mutex_lock(&readlock);
if(Bchart::tokenize)
*tokStream >> *srp;
else
*nontokStream >> *srp;
int locCount = sentenceCount++;
ExtPos extPos;
if(params.extPosIfstream)
extPos.read(params.extPosIfstream,*srp);
pthread_mutex_unlock(&readlock);
if( !params.field().in(sentenceCount) ) continue;
printStruct printS;
printS.name = srp->getName();
printS.sentenceCount = locCount;
printS.numDiff = 0;
int len = srp->length();
if (len == 0) {
break;
}
if (len > params.maxSentLen)
{
ECString msg("skipping sentence longer than specified limit of ");
msg += intToString(params.maxSentLen);
WARN( msg.c_str() );
printSkipped(srp,NULL,printStack,printS);
continue;
}
// handle input containing reserved word Bchart::HEADWORD_S1; could probably do
// better (like undo replacement before printing) but this seems sufficient.
int i;
for (i = 0; i < len; ++i)
{
ECString& w = ((*srp)[i]).lexeme();
if (w == Bchart::HEADWORD_S1)
{
ECString msg = ECString("Replacing reserved token \"") + Bchart::HEADWORD_S1;
msg += "\" at index " + intToString(i) + " of input with token \"^^^\"";
WARN( msg.c_str() );
w = "^^^";
}
}
MeChart* chart = new MeChart( *srp,extPos,*id );
chart->parse( );
Item* topS = chart->topS();
if(!topS)
{
if (extPos.hasExtPos()) {
WARN("Parse failed: !topS -- reparsing without POS constraints");
chart = new MeChart(*srp, *id);
chart->parse();
topS = chart->topS();
if (!topS) {
WARN("Reparsing without POS constraints failed too: !topS");
printSkipped(srp, chart, printStack, printS);
continue;
}
} else {
WARN( "Parse failed: !topS" );
printSkipped(srp,chart,printStack,printS);
continue;
}
}
bool failed = decodeParses(len, locCount, srp, chart, printS, printStack);
if (failed) {
continue;
}
if( printS.numDiff == 0)
{
if (extPos.hasExtPos()) {
WARN("Parse failed from 0, inf or nan probabililty -- reparsing without POS constraints");
chart = new MeChart(*srp, *id);
chart->parse();
bool failed = decodeParses(len, locCount, srp, chart, printS, printStack);
if (failed || printS.numDiff == 0) {
WARN("Parse failed from 0, inf or nan probabililty -- failed even without POS constraints");
printSkipped(srp,chart,printStack,printS);
continue;
}
} else {
WARN("Parse failed from 0, inf or nan probabililty");
printSkipped(srp,chart,printStack,printS);
continue;
}
}
/* put the sentence with which we just finished at the end of the printStack*/
printStack.push_back(printS);
workOnPrintStack(&printStack);
delete chart;
delete srp;
}
void process() {
uint64_t num_errors = 0, num_valid_ops = 0, num_valid_bytes = 0;
entry entry;
processHeader();
level = 1;
while(positions[0].offset < positions[0].size) {
positions[1] = positions[0];
entry = loadEntry();
if (!entry.success) {
printValid(num_valid_ops, num_valid_bytes);
printErrorStack(&entry);
num_errors++;
num_valid_ops = 0;
num_valid_bytes = 0;
/* search for next valid entry */
uint64_t offset = positions[0].offset + 1;
int i = 0;
while (!entry.success && offset < positions[0].size) {
positions[1].offset = offset;
/* find 3 consecutive valid entries */
for (i = 0; i < 3; i++) {
entry = loadEntry();
if (!entry.success) break;
}
/* check if we found 3 consecutive valid entries */
if (i < 3) {
offset++;
}
}
/* print how many bytes we have skipped to find a new valid opcode */
if (offset < positions[0].size) {
printSkipped(offset - positions[0].offset, offset);
}
positions[0].offset = offset;
} else {
num_valid_ops++;
num_valid_bytes += positions[1].offset - positions[0].offset;
/* advance position */
positions[0] = positions[1];
}
}
/* because there is another potential error,
* print how many valid ops we have processed */
//printValid(num_valid_ops, num_valid_bytes);
db_stats.num_valid_ops = num_valid_ops;
db_stats.num_valid_bytes = num_valid_bytes;
/* expect an eof */
if (entry.type != REDIS_EOF) {
/* last byte should be EOF, add error */
errors.level = 0;
SHIFT_ERROR(positions[0].offset,
"Expected EOF, got %s", types[entry.type]);
/* this is an EOF error so reset type */
entry.type = -1;
printErrorStack(&entry);
num_errors++;
}
/* print summary on errors */
if (num_errors) {
printf("\n");
printf("Total unprocessable opcodes: %llu\n",
(unsigned long long) num_errors);
}
}
