/**
only reads initial header information
*/
void readHeader(ByteFile* bf)
{
seekPos(bf, ALN_HEAD);
READ_VAR(bf->fh, bf->numTax);
READ_VAR(bf->fh, bf->numPattern);
READ_VAR(bf->fh, bf->numPartitions);
READ_VAR(bf->fh, bf->gappyness) ;
bf->hasRead |= ALN_HEAD;
}
static guint32
lookup_var (test_entry_t *entry, const char *name)
{
if (!strcmp ("file-size", name))
return entry->data_size;
if (!strcmp ("pe-signature", name))
return get_pe_header (entry) - 4;
if (!strcmp ("pe-header", name))
return get_pe_header (entry);
if (!strcmp ("pe-optional-header", name))
return get_pe_header (entry) + 20;
if (!strcmp ("section-table", name))
return get_pe_header (entry) + 244;
if (!strcmp ("cli-header", name))
return get_cli_header (entry);
if (!strcmp ("cli-metadata", name))
return get_cli_metadata_root (entry);
if (!strcmp ("tables-header", name)) {
guint32 metadata_root = get_cli_metadata_root (entry);
guint32 tilde_stream = get_metadata_stream_header (entry, 0);
guint32 offset = READ_VAR (guint32, entry->data + tilde_stream);
return metadata_root + offset;
}
printf ("Unknown variable in expression %s\n", name);
exit (INVALID_VARIABLE_NAME);
}
static guint32
get_cli_metadata_root (test_entry_t *entry)
{
guint32 offset = get_cli_header (entry);
offset += 8; /*metadata rva offset*/
return translate_rva (entry, READ_VAR (guint32, entry->data + offset));
}
static guint32
get_cli_header (test_entry_t *entry)
{
guint32 offset = get_pe_header (entry) + 20; /*pe-optional-header*/
offset += 208; /*cli header entry offset in the pe-optional-header*/
return translate_rva (entry, READ_VAR (guint32, entry->data + offset));
}
static guint32
translate_rva (test_entry_t *entry, guint32 rva)
{
guint32 pe_header = get_pe_header (entry);
guint32 sectionCount = READ_VAR (guint16, entry->data + pe_header + 2);
guint32 idx = pe_header + 244;
while (sectionCount-- > 0) {
guint32 size = READ_VAR (guint32, entry->data + idx + 8);
guint32 base = READ_VAR (guint32, entry->data + idx + 12);
guint32 offset = READ_VAR (guint32, entry->data + idx + 20);
if (rva >= base && rva <= base + size)
return (rva - base) + offset;
idx += 40;
}
printf ("Could not translate RVA %x\n", rva);
exit (INVALID_RVA);
}
static void
apply_effect (patch_effect_t *effect, test_entry_t *entry, guint32 offset)
{
gint32 value = 0;
char *ptr = entry->data + offset;
if (effect->expression)
value = expression_eval (effect->expression, entry);
switch (effect->type) {
case EFFECT_SET_BYTE:
DEBUG_PARSER (printf("\tset-byte effect old value [%x] new value [%x]\n", READ_VAR (guint8, ptr), value));
SET_VAR (guint8, ptr, value);
break;
case EFFECT_SET_USHORT:
DEBUG_PARSER (printf("\tset-ushort effect old value [%x] new value [%x]\n", READ_VAR (guint16, ptr), value));
SET_VAR (guint16, ptr, value);
break;
case EFFECT_SET_UINT:
DEBUG_PARSER (printf("\tset-uint effect old value [%x] new value [%x]\n", READ_VAR (guint32, ptr), value));
SET_VAR (guint32, ptr, value);
break;
case EFFECT_SET_TRUNC:
DEBUG_PARSER (printf("\ttrunc effect [%d]\n", offset));
entry->data_size = offset;
break;
case EFFECT_SET_BIT:
DEBUG_PARSER (printf("\tset-bit effect bit %d old value [%x]\n", value, READ_BIT (ptr, value)));
SET_BIT (ptr, value);
break;
case EFFECT_OR_BYTE:
DEBUG_PARSER (printf("\tor-byte effect old value [%x] new value [%x]\n", READ_VAR (guint8, ptr), value));
SET_VAR (guint8, ptr, READ_VAR (guint8, ptr) | value);
break;
case EFFECT_OR_USHORT:
DEBUG_PARSER (printf("\tor-ushort effect old value [%x] new value [%x]\n", READ_VAR (guint16, ptr), value));
SET_VAR (guint16, ptr, READ_VAR (guint16, ptr) | value);
break;
case EFFECT_OR_UINT:
DEBUG_PARSER (printf("\tor-uint effect old value [%x] new value [%x]\n", READ_VAR (guint32, ptr), value));
SET_VAR (guint32, ptr, READ_VAR (guint32, ptr) | value);
break;
default:
printf ("Invalid effect type %d\n", effect->type);
exit (INVALID_EFFECT);
}
}
static guint32
get_metadata_stream_header (test_entry_t *entry, guint32 idx)
{
guint32 offset;
offset = get_cli_metadata_root (entry);
offset = pad4 (offset + 16 + READ_VAR (guint32, entry->data + offset + 12));
offset += 4;
while (idx--) {
int i;
offset += 8;
for (i = 0; i < 32; ++i) {
if (!READ_VAR (guint8, entry->data + offset++))
break;
}
offset = pad4 (offset);
}
return offset;
}
/**
reads the taxon names from the byte file
*/
void readTaxa(ByteFile *bf)
{
int i;
assert(bf->taxaNames == (char **)NULL);
seekPos(bf, ALN_TAXA);
bf->taxaNames = (char **)calloc(bf->numTax, sizeof(char*));
for(i = 0; i < bf->numTax; ++i)
{
int len = 0;
READ_VAR(bf->fh, len );
bf->taxaNames[i] = (char*)calloc(len, sizeof(char));
READ_ARRAY(bf->fh, bf->taxaNames[i], len, sizeof(char));
}
bf->hasRead |= ALN_TAXA;
}
/**
initializes ByteFile **bf
*/
void initializeByteFile(ByteFile **bf, char *name)
{
*bf = (ByteFile *)calloc(1,sizeof(ByteFile));
ByteFile *result = *bf;
result->fh = myfopen(name, "rb");
int
sizeOfSizeT = 0,
version = 0,
magicNumber = 0;
READ_VAR(result->fh, sizeOfSizeT);
if(sizeOfSizeT != sizeof(size_t))
{
if(processID == 0)
{
printf("\nError: the address data type has a size of %d bits on the current system while on the system on which you created the binary alignment file using the parser the address size is %d bits!\n",
8 * (int)sizeof(size_t), 8 * sizeOfSizeT);
printf("Usually this indicates that the parser was executed on a 32-bit system while you are trying to run ExaML on a 64-bit system.\n");
printf("Please parse the binary alignment file on the same hardware on which you intend to run ExaML.\n\n\n");
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_Finalize();
exit(-1);
}
//check that version numbers of parser and ExaML match
READ_VAR(result->fh, version);
if(version != (int)programVersionInt)
{
if(processID == 0)
{
printf("\nError: Version number %d of ExaML parser and version number %d of ExaML don't match.\n", version, (int)programVersionInt);
printf("You are either using an outdated version of the parser or of ExaML.\n");
printf("Hasta siempre comandante.\n\n\n");
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_Finalize();
exit(-1);
}
READ_VAR(result->fh, magicNumber);
if(magicNumber != 6517718)
{
if(processID == 0)
{
printf("\nError: The magic number %d of ExaML parser and magic number %d of ExaML don't match.\n", magicNumber, 6517718);
printf("Something went terribly wrong here.\n");
printf("Hasta la victoria siempre.\n\n\n");
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_Finalize();
exit(-1);
}
}
/**
reads partition information from the byte file.
*/
void readPartitions(ByteFile *bf)
{
int i ;
seekPos(bf, ALN_PARTITIONS);
assert(bf->partitions == (pInfo **)NULL);
bf->partitions = (pInfo **)calloc((size_t)bf->numPartitions, sizeof(pInfo*) );
for(i = 0; i < bf->numPartitions; ++i)
{
bf->partitions[i] = (pInfo*)calloc(1,sizeof(pInfo));
pInfo* p = bf->partitions[i];
p->frequencies = (double*)NULL;
p->partitionName = (char *)NULL;
READ_VAR(bf->fh, p->states);
READ_VAR(bf->fh, p->maxTipStates);
READ_VAR(bf->fh, p->lower);
READ_VAR(bf->fh, p->upper);
/* DONT use this value! */
READ_VAR(bf->fh, p->width);
p->width = 0;
READ_VAR(bf->fh, p->dataType);
READ_VAR(bf->fh, p->protModels);
//READ_VAR(bf->fh, p->autoProtModels);
READ_VAR(bf->fh, p->protFreqs);
READ_VAR(bf->fh, p->nonGTR);
READ_VAR(bf->fh, p->optimizeBaseFrequencies);
// READ_VAR(bf->fh, p->numberOfCategories);
/* read string */
unsigned int len = 0;
READ_VAR(bf->fh, len);
p->partitionName = (char*)calloc(len,sizeof(char));
READ_ARRAY(bf->fh, p->partitionName, len, sizeof(char));
p->frequencies = (double*)calloc((size_t)p->states, sizeof(double));
READ_ARRAY(bf->fh, p->frequencies, (size_t)p->states , sizeof(double));
}
bf->hasRead |= ALN_PARTITIONS;
}
see_vm_prog_t
see_vm_prog_parse_bin(see_heap_t heap, see_io_char_stream_in_f in, void *data)
{
see_vm_uword_t constant_count;
see_vm_uword_t opcode_count;
if (!READ_VAR(constant_count, in, data)) return NULL;
if (!READ_VAR(opcode_count, in, data)) return NULL;
see_vm_prog_t prog = SEE_OBJECT_NEW(heap, see_vm_prog_s);
if (prog == NULL) return NULL;
if ((prog->constant_slot =
(see_vm_slot_s *)SEE_MALLOC(constant_count * sizeof(see_vm_slot_s)))
== NULL)
{
if (heap == NULL)
see_object_host_free(prog);
else see_object_unprotect(prog);
return NULL;
}
if ((prog->opcode_entry =
(see_vm_opcode_t *)SEE_MALLOC(opcode_count * sizeof(see_vm_opcode_t)))
== NULL)
{
SEE_FREE(prog->constant_slot);
if (heap == NULL)
see_object_host_free(prog);
else see_object_unprotect(prog);
return NULL;
}
prog->constant_count = constant_count;
prog->opcode_count = opcode_count;
see_vm_uword_t i;
/* XXX read object constant */
for (i = 0; i < constant_count; ++ i)
{
see_vm_word_t word;
if (READ_VAR(word, in, data))
{
prog->constant_slot[i].type = SEE_VM_SLOT_TYPE_WORD;
prog->constant_slot[i]._word = word;
}
else
{
SEE_FREE(prog->constant_slot);
SEE_FREE(prog->opcode_entry);
if (heap == NULL)
see_object_host_free(prog);
else see_object_unprotect(prog);
return NULL;
}
}
for (i = 0; i < opcode_count; ++ i)
{
if (!READ_VAR(prog->opcode_entry[i], in, data))
{
SEE_FREE(prog->constant_slot);
SEE_FREE(prog->opcode_entry);
if (heap == NULL)
see_object_host_free(prog);
else see_object_unprotect(prog);
return NULL;
}
}
SEE_OBJECT_TYPE_INIT(prog, SEE_TYPE_VM_PROG);
return prog;
}
Function* bas_load_compiled_function(Basic* kernel, DataReader* reader)
{
Function* function = NULL;
uint16_t function_size;
uint32_t n;
uint16_t* global_table = NULL;
uint16_t function_globals;
uint8_t name_size;
char* name = NULL;
int success = 0;
READ_VAR(function_size);
//printf( "Function size %d\n", function_size );
function = (Function*)mem_malloc(sizeof(Function)+function_size);
if (function == 0)
return NULL;
function->header.ref_count = 1;
function->header.type = kTypeFunction;
READ_VAR(function->locals);
READ_VAR(function->arguments);
READ_VAR(function->stack_usage);
READ_ARRAY( (char*)(function+1), function_size );
READ_VAR(function_globals);
global_table = (uint16_t*)mem_malloc(sizeof(uint16_t)*function_globals);
if (global_table == NULL)
goto end;
for(n=0; n<function_globals; ++n)
{
READ_VAR(name_size);
name = mem_malloc(name_size+1);
if (name == NULL)
goto end;
READ_ARRAY(name, name_size);
name[name_size] = '\0';
global_table[n] = bas_find_or_create_global(kernel, name);
if (global_table[n] == 0xFFFF)
goto end;
mem_free(name);
name = NULL;
}
// patch the bytecode to fix the global indices
uint8_t* bc_ptr = (uint8_t*)(function+1);
uint8_t* bc_end = bc_ptr + function_size;
while (bc_ptr < bc_end)
{
switch (*bc_ptr)
{
case BC_GET_GLOBAL:
case BC_SET_GLOBAL:
{
bc_ptr ++;
uint16_t index;
memcpy( &index, bc_ptr, 2 );
index = global_table[index];
memcpy( bc_ptr, &index, 2 );
bc_ptr += 2;
break;
}
case BC_ADD:
case BC_SUBTRACT:
case BC_MULTIPLY:
case BC_DIVIDE:
case BC_RETURN_ZERO:
case BC_POP:
case BC_COMPARE_LT:
case BC_COMPARE_LE:
case BC_COMPARE_GT:
case BC_COMPARE_GE:
case BC_COMPARE_EQ:
case BC_COMPARE_NE:
case BC_NOT:
case BC_NEGATE:
bc_ptr++;
break; // do nothing
case BC_CALL:
bc_ptr += 2; // skip number of arguments
break;
case BC_PUSH_INT32:
bc_ptr += 5; // skip literal
break;
case BC_PUSH_FLOAT:
bc_ptr += sizeof(float)+1; // skip literal
break;
case BC_PUSH_STRING:
{
// skip string size and characters
bc_ptr ++;
uint16_t stringSize;
memcpy( &stringSize, bc_ptr, 2 ); bc_ptr += 2;
bc_ptr += stringSize;
}
break;
case BC_BRANCH_TRUE:
case BC_BRANCH_FALSE:
{
bc_ptr += 3; // skip branch offset
}
break;
default:
goto end;
}
}
success = 1;
end:
if (!success)
{
if (function != NULL)
mem_free(function);
function = NULL;
}
if (name != NULL) mem_free(name);
if (global_table != NULL) mem_free(global_table);
return function;
}
static guint32
call_func (test_entry_t *entry, const char *name, GSList *args)
{
if (!strcmp ("read.byte", name)) {
guint32 offset;
if (g_slist_length (args) != 1) {
printf ("Invalid number of args to read.ushort %d\n", g_slist_length (args));
exit (INVALID_ARG_COUNT);
}
offset = expression_eval (args->data, entry);
return READ_VAR (guint8, entry->data + offset);
}
if (!strcmp ("read.ushort", name)) {
guint32 offset;
if (g_slist_length (args) != 1) {
printf ("Invalid number of args to read.ushort %d\n", g_slist_length (args));
exit (INVALID_ARG_COUNT);
}
offset = expression_eval (args->data, entry);
return READ_VAR (guint16, entry->data + offset);
}
if (!strcmp ("read.uint", name)) {
guint32 offset;
if (g_slist_length (args) != 1) {
printf ("Invalid number of args to read.uint %d\n", g_slist_length (args));
exit (INVALID_ARG_COUNT);
}
offset = expression_eval (args->data, entry);
return READ_VAR (guint32, entry->data + offset);
}
if (!strcmp ("translate.rva", name)) {
guint32 rva;
if (g_slist_length (args) != 1) {
printf ("Invalid number of args to translate.rva %d\n", g_slist_length (args));
exit (INVALID_ARG_COUNT);
}
rva = expression_eval (args->data, entry);
return translate_rva (entry, rva);
}
if (!strcmp ("translate.rva.ind", name)) {
guint32 rva;
if (g_slist_length (args) != 1) {
printf ("Invalid number of args to translate.rva.ind %d\n", g_slist_length (args));
exit (INVALID_ARG_COUNT);
}
rva = expression_eval (args->data, entry);
rva = READ_VAR (guint32, entry->data + rva);
return translate_rva (entry, rva);
}
if (!strcmp ("stream-header", name)) {
guint32 idx;
if (g_slist_length (args) != 1) {
printf ("Invalid number of args to stream-header %d\n", g_slist_length (args));
exit (INVALID_ARG_COUNT);
}
idx = expression_eval (args->data, entry);
return get_metadata_stream_header (entry, idx);
}
if (!strcmp ("table-row", name)) {
const char *data;
guint32 table, row;
const MonoTableInfo *info;
if (g_slist_length (args) != 2) {
printf ("Invalid number of args to table-row %d\n", g_slist_length (args));
exit (INVALID_ARG_COUNT);
}
table = expression_eval (args->data, entry);
row = expression_eval (args->next->data, entry);
info = mono_image_get_table_info (entry->test_set->image, table);
data = info->base + row * info->row_size;
return data - entry->test_set->assembly_data;
}
if (!strcmp ("blob.i", name)) {
guint32 offset, base;
MonoStreamHeader blob = entry->test_set->image->heap_blob;
if (g_slist_length (args) != 1) {
printf ("Invalid number of args to blob %d\n", g_slist_length (args));
exit (INVALID_ARG_COUNT);
}
base = blob.data - entry->test_set->image->raw_data;
offset = expression_eval (args->data, entry);
offset = READ_VAR (guint16, entry->data + offset);
return base + offset;
}
printf ("Unknown function %s\n", name);
exit (INVALID_FUNCTION_NAME);
}
static guint32
get_pe_header (test_entry_t *entry)
{
return READ_VAR (guint32, entry->data + 0x3c) + 4;
}
static int ReadImageDescription( Image *imPtr, stBuf *sPtr, char *line )
{
Image im;
stBuf sBuf;
char *ch = line;
char buf[256];
int i;
memcpy( &im, imPtr, sizeof(Image) );
memcpy( &sBuf, sPtr, sizeof(stBuf ));
im.cP.radial = FALSE;
while( *ch != 0)
{
switch(*ch)
{
case 'v': READ_VAR( "%lf", &im.hfov );
break;
case 'a': READ_VAR( "%lf", &(im.cP.radial_params[0][3]));
im.cP.radial = TRUE;
break;
case 'b': READ_VAR("%lf", &(im.cP.radial_params[0][2]));
im.cP.radial = TRUE;
break;
case 'c': READ_VAR("%lf", &(im.cP.radial_params[0][1]));
im.cP.radial = TRUE;
break;
case 'f': READ_VAR( "%ld", &im.format );
if( im.format == _panorama || im.format == _equirectangular )
im.cP.correction_mode |= correction_mode_vertical;
break;
case 'y': READ_VAR( "%lf", &im.yaw);
break;
case 'p': READ_VAR( "%lf", &im.pitch);
break;
case 'r': READ_VAR( "%lf", &im.roll);
break;
case 'd': READ_VAR("%lf", &(im.cP.horizontal_params[0]));
im.cP.horizontal = TRUE;
break;
case 'e': READ_VAR("%lf", &(im.cP.vertical_params[0]));
im.cP.vertical = TRUE;
break;
case 'g': READ_VAR("%lf", &(im.cP.shear_x));
im.cP.shear = TRUE;
break;
case 't': READ_VAR("%lf", &(im.cP.shear_y));
im.cP.shear = TRUE;
break;
case '+': nextWord2( buf, &ch );
sprintf( sBuf.srcName, "%s", buf);
break;
case '-': nextWord2( buf, &ch );
sprintf( sBuf.destName, "%s", buf );
break;
case 'u': READ_VAR( "%d", &(sBuf.feather) );
break;
case 's': READ_VAR( "%d", &sBuf.seam );
{
if(sBuf.seam != _dest)
sBuf.seam = _middle;
}
break;
case 'w': READ_VAR( "%ld", &im.width );
break;
case 'h': READ_VAR("%ld", &im.height);
break;
case 'o': ch++;
im.cP.correction_mode |= correction_mode_morph;
break;
case 'k': // Colour correction
READ_VAR( "%d", &i );
// i &= 3;
sBuf.colcorrect |= i&3;
sBuf.colcorrect += (i+1)*4;
break;
case 'm': // Frame
ch++;
switch( *ch )
{
case 'x':
READ_VAR( "%d", &im.cP.fwidth );
im.cP.cutFrame = TRUE;
break;
case 'y':
READ_VAR( "%d", &im.cP.fheight );
im.cP.cutFrame = TRUE;
break;
default:
ch--;
READ_VAR( "%d", &(im.cP.frame) );
im.cP.cutFrame = TRUE;
break;
}
break;
case 'n': // Name string (used for panorama format)
nextWord2( buf, &ch );
strcpy( im.name, buf );
break;
case 'S': nextWord2( buf, &ch );
sscanf( buf, "%ld,%ld,%ld,%ld", &im.selection.left, &im.selection.right, &im.selection.top, &im.selection.bottom );
break;
case 'C': nextWord2( buf, &ch );
sscanf( buf, "%ld,%ld,%ld,%ld", &im.selection.left, &im.selection.right, &im.selection.top, &im.selection.bottom );
im.cP.cutFrame = TRUE;
break;
default: ch++;
break;
}
}
// Set 4th polynomial parameter
im.cP.radial_params[0][0] = 1.0 - ( im.cP.radial_params[0][3] + im.cP.radial_params[0][2]
+ im.cP.radial_params[0][1] ) ;
SetEquColor( &im.cP );
SetCorrectionRadius( &im.cP );
// Do checks
// appears ok
memcpy( imPtr, &im, sizeof(Image) );
memcpy( sPtr, &sBuf, sizeof(stBuf ) );
return 0;
}
Quantizer* LshKBM::load(FILE *f)
{
cout<<"LshKBM::load: loading"<<endl;
unsigned int signature=0;
unsigned char ver=99;
READ_VAR(signature); assert(signature==0xFAADBEEB);
READ_VAR(ver); assert(ver<=mVer);
mVer=ver;
READ_VAR(type); assert(type==CV_8UC1);
READ_VAR(dims); assert(dims==BriefDesc::BRIEF_K);
READ_VAR(num_descriptors);
READ_VAR(num_images);
unsigned int word_count;
READ_VAR(word_count);
// Create forest of kd-trees for searching
switch(mVer) {
case 0:
mJfinal=256.0;
mDtKBinMeans=0.0;
for(uint32_t i=0;i<guidLen+1;++i)
m_guid[i]=0;
break;
case 1:
READ_VAR(mJfinal);
READ_VAR(mDtKBinMeans);
for(uint32_t i=0;i<guidLen+1;++i)
m_guid[i]=0;
break;
case 2:
READ_VAR(mJfinal);
READ_VAR(mDtKBinMeans);
READ_VAR(m_guid);
break;
case 3:
READ_VAR(mJfinal);
READ_VAR(mDtKBinMeans);
READ_VAR(m_guid);
uint32_t m,l;
READ_VAR(m);
READ_VAR(l);
mLshKbmP=LshKBMParams(word_count,l,m,mLshKbmP.getThreadMax());
break;
default:
Log::error("Invalid file version: %d!\n", ver);
return NULL;
}
// m_num_clusters = word_count;
cout<<"sig="<<signature<<" ver="<<ver<<" type="<<type<<" dims="<<dims<<" #descriptors="<<num_descriptors<<" #images="<<num_images<<" #words="<<word_count<<endl;
cout<<" mJfinal="<<mJfinal<<" mDtKBinMeans="<<mDtKBinMeans<<endl;
mLshKbmP=LshKBMParams(word_count,mLshKbmP.getL(),mLshKbmP.getM(),mLshKbmP.getThreadMax());
mCentroids.resize(mLshKbmP.getK(), NULL);
for(uint32_t i=0; i<mLshKbmP.getK(); ++i)
{ // create all centroids
mCentroids[i]=new BriefDescS();
READ_PTR(&(mCentroids[i]->bd[0]), dims*sizeof(mCentroids[i]->bd[0]));
}
// Create forest of kd-trees for searching
switch(ver) {
case 0:
case 1:
case 2:
case 3:
Log::info("Quantizer file had NO prebuilt LSH\n");
// build hash tables - the class will realize,
// that the centroids have been loaded already!
prepareFromCentroids(mCentroids);
cout<<"LshKBM::load: loaded successfully"<<endl;
break;
default:
Log::error("Invalid file version: %d!\n", ver);
return NULL;
}
return this;
}
