int bin64_t::peaks (uint64_t length, bin64_t* peaks) {
int pp=0;
uint8_t layer = 0;
while (length) {
if (length&1)
peaks[pp++] = bin64_t(layer,length^1);
length>>=1;
layer++;
}
for(int i=0; i<(pp>>1); i++) {
uint64_t memo = peaks[pp-1-i];
peaks[pp-1-i] = peaks[i];
peaks[i] = memo;
}
peaks[pp] = NONE;
return pp;
}
void readConfig( const char* filename ) {
/* States:
* 0 : reading new entities ( to: 1, 3, 4, 5 )
* 1 : reading machine
* 2 : reading role
* 3 : reading core
* 4 : reading file
* 5 : reading test ( to: 2 )
*/
int state = 0;
struct machine m;
struct role r;
struct file f;
struct core c;
struct test t;
xmlTextReaderPtr xmlRead = xmlReaderForFile( filename, NULL, XML_PARSE_NONET | XML_PARSE_NOENT | XML_PARSE_NOCDATA | XML_PARSE_NOXINCNODE | XML_PARSE_COMPACT );
if( !xmlRead )
quit( "Could not open %s for reading as XML config\n", filename );
xmlTextReaderSetErrorHandler( xmlRead, xmlErrorHandler, 0 );
int nodeType, ret;
xmlChar* nodeName;
while( 1 ) {
if( ( ret = xmlTextReaderRead( xmlRead ) ) != 1 ) {
if( ret < 0 )
quit( "Error occurred\n" );
if( state )
quit( "No more nodes to read, but state is not 0. Incomplete elements.\n" );
xmlTextReaderClose( xmlRead );
return;
}
nodeType = xmlTextReaderNodeType( xmlRead );
nodeName = NULL;
switch( nodeType ) {
case XmlNodeType.Element :
nodeName = xmlTextReaderLocalName( xmlRead );
switch( state ) {
case 0 :
if( !strcmp( nodeName, "machine" ) ) {
state = 1;
bzero( m, sizeof( struct machine ) );
readValidRequiredAttribute( m.name, "machine", "name" );
readValidRequiredAttribute( m.address, "machine", "address" );
if( !strcmp( m.address, "DAS4" ) ) {
readValidRequiredAttribute( m.count, "machine", "DAS4 node count" );
char* end = NULL;
m.icount = strtol( m.count, &end, 10 );
if( end == m.count || *end || m.icount < 1 )
quit( "Invalid DAS4 node count for machine %s\n", m.name );
}
break;
}
if( !strcmp( nodeName, "core" ) ) {
state = 3;
bzero( c, sizeof( struct core ) );
readValidRequiredAttribute( c.name, "core", "name" );
break;
}
if( !strcmp( nodeName, "file" ) ) {
state = 4;
bzero( f, sizeof( struct file ) );
readValidRequiredAttribute( f.name, "file", "name" );
readValidRequiredAttribute( f.size, "file", "size" );
char* end = NULL;
f.isize = strtol( f.size, &end, 10 );
if( end == f.size || f.isize < 1 )
quit( "Invalid size specifier for file %s\n", f.name );
if( *end && *(end+1) )
quit( "Invalid size specifier for file %s\n", f.name );
if( *end ) {
switch( *end ) {
case 'b' :
case 'B' :
break;
case 'k' :
case 'K' :
f.isize *= 1024;
break;
case 'm' :
case 'M' :
f.isize *= 1024 * 1024;
break;
case 'g' :
case 'G' :
f.isize *= 1024 * 1024 * 1024;
break;
case 't' :
case 'T' :
f.isize *= 1024 * 1024 * 1024 * 1024;
break;
default :
quit( "Invalid size specifier for file %s\n", f.name );
}
}
if( f.isize > 512 * 1024 * 1024 ) {
int p = 512*1024*1024;
while( p < f.isize )
p <<= 1;
if( p != f.isize )
quit( "Invalid size specifier for file %s: sizes above 512M should be powers of 2\n", f.name );
}
if( f.isize & 0x3 )
quit( "Invalid size specifier for file %s: sizes should be a multiple of 4\n", f.name );
break;
}
if( !strcmp( nodeName, "test" ) ) {
state = 5;
bzero( t, sizeof( struct test ) );
readValidRequiredAttribute( t.name, "test", "name" );
break;
}
break;
case 1 :
if( !strcmp( nodeName, "tmpDir" ) ) {
onlyOne( m.tmpdir, "machine", "tmpDir" );
readValidString( m.tmpdir, "temporary directory location" );
skipToEndElement( "tmpDir" );
break;
}
if( !strcmp( nodeName, "params" ) ) {
onlyOne( m.params, "machine", "params" );
readValidString( m.params, "params" );
skipToEndElement( "params" );
break;
}
printf( "Unexpected element %s in machine %s\n", nodeName, m.name );
break;
case 2 :
if( !strcmp( nodeName, "machine" ) ) {
if( r.machineCount > 255 )
quit( "Maximum of 256 machines per role passed on line %d\n", xmlTextReaderGetParserLineNumber( xmlRead ) );
readValid( r.machine[r.machineCount], "machine name" );
r.machineCount++;
skipToEndElement( "machine" );
break;
}
if( !strcmp( nodeName, "user" ) ) {
onlyOne( r.user, "role", "user" );
readValidString( r.user, "user" );
skipToEndElement( "user" );
break;
}
if( !strcmp( nodeName, "core" ) ) {
onlyOne( r.core, "role", "core" );
readValidString( r.core, "core name" );
skipToEndElement( "core" );
break;
}
if( !strcmp( nodeName, "file" ) ) {
onlyOne( r.file, "role", "file" );
readValidString( r.file, "file name" );
skipToEndElement( "file" );
break;
}
if( !strcmp( nodeName, "params" ) ) {
onlyOne( r.params, "role", "params" );
readValidString( r.params, "params" );
skipToEndElement( "params" );
break;
}
printf( "Unexpected element %s in role on line %d\n", nodeName, xmlTextReaderGetParserLineNumber( xmlRead ) );
break;
case 3 :
if( !strcmp( nodeName, "params" ) ) {
onlyOne( c.params, "core", "params" );
readValidString( c.params, "params" );
skipToEndElement( "params" );
break;
}
if( !strcmp( nodeName, "localDir" ) ) {
onlyOne( c.localdir, "core", "localDir" );
readValidString( c.localdir, "local core directory" );
skipToEndElement( "localDir" );
break;
}
if( !strcmp( nodeName, "compilationDir" ) ) {
onlyOne( c.compdir, "core", "compilationDir" );
readValidString( c.compdir, "relative compilation directory" );
skipToEndElement( "compilationDir" );
break;
}
if( !strcmp( nodeName, "program" ) ) {
onlyOne( c.program, "core", "program" );
readValidString( c.program, "program name" );
skipToEndElement( "program" );
break;
}
printf( "Unexpected element %s in core %s\n", nodeName, c.name );
skipToEndElement( nodeName );
break;
case 4 :
if( !strcmp( nodeName, "offset" ) ) {
onlyOne( c.offset, "file", "offset" );
readValidString( c.offset, "offset" );
char* end = NULL;
f.ioffset = strtol( f.offset, &end, 10 );
if( end == f.offset || *end || f.ioffset < 0 || f >= f.isize )
quit( "Invalid file offset for file %s\n", f.name );
skipToEndElement( "offset" );
break;
}
printf( "Unexpected element %s in file %s\n", nodeName, f.name );
skipToEndElement( nodeName );
break;
case 5 :
if( !strcmp( nodeName, "role" ) ) {
state = 2;
bzero( r, sizeof( struct role ) );
readValidRequiredAttribute( r.type, "role", "type" );
if( strcmp( r.type, "seed" ) && strcmp( r.type, "leech" ) )
quit( "Invalid value for attribute 'type' for role on line %d, expected 'seed' or 'leech'\n", xmlTextReaderGetParserLineNumber( xmlRead ) );
break;
}
quit( "Unexpected element %s in test on line %d\n", nodeName, xmlTextReaderGetParserLineNumber( xmlRead ) );
default :
quit( "State sanity\n", nodeName );
}
break;
case XmlNodeType.EndElement :
nodeName = xmlTextReaderLocalName( xmlRead );
switch( state ) {
case 0 :
quit( "End element %s found while not in entity\n", nodeName );
case 1 :
if( !strcmp( nodeName, "machine" ) ) {
memcpy( machines + machineCount, &m, sizeof( struct machine ) );
machineCount++;
state = 0;
}
break;
case 2 :
if( !strcmp( nodeName, "role" ) ) {
mempcy( t.roles + r.roleCount, &r, sizeof( struct role ) );
t.roleCount++;
state = 5;
}
break;
case 3 :
if( !strcmp( nodeName, "core" ) ) {
memcpy( cores + coreCount, &c, sizeof( struct core ) );
coreCount++;
state = 0;
}
break;
case 4 :
if( !strcmp( nodeName, "file" ) ) {
memcpy( files + fileCount, &f, sizeof( struct file ) );
fileCount++;
state = 0;
}
break;
case 5 :
if( !strcmp( nodeName, "test" ) ) {
memcpy( tests + testCount, &t, sizeof( struct test ) );
testCount++;
state = 0;
}
break;
default :
quit( "State sanity\n" );
}
default :
}
if( nodeName )
free( nodeName );
}
}
void validateConfigAndGenerateScripts( ) {
int i, j, k, l;
bool found;
// == Validate test-role references to machines, cores and files
// == Also register which machine uses which cores and which files
for( i = 0; i < testCount; i++ ) {
for( j = 0; j < tests[i].roleCount; j++ ) {
// Check for validity of each role's core
found = false;
for( k = 0; k < coreCount; k++ ) {
if( !strcmp( cores[k].name, tests[i].roles[j].core ) ) {
found = true;
break;
}
}
if( !found )
quit( "Test %s role %i refers to core %s which does not exist\n", tests[i].name, j, tests[i].roles[j].core );
// Check for validity of each role's file
found = false;
for( k = 0; k < fileCount; k++ ) {
if( !strcmp( files[k].name, tests[i].roles[j].file ) ) {
found = true;
break;
}
}
if( !found )
quit( "Test %s role %i refers to file %s which does not exist\n", tests[i].name, j, tests[i].roles[j].file );
// Check for validity of each role's machines
for( l = 0; l < tests[i].roles[j].machineCount; l++ ) {
found = false;
for( k = 0; k < machineCount; k++ ) {
if( !strcmp( machines[k].name, tests[i].roles[j].machines[l] ) ) {
// Machine found: register used core with the machine if needed
for( m = 0; m < machines[k].coreCount; m++ ) {
if( !strcmp( machines[k].cores[m], tests[i].roles[j].core ) ) {
found = true;
break;
}
}
if( !found )
machines[k].cores[machines[k].coreCount++] = tests[i].roles[j].core;
// Machine found: register used file with the machine if needed, only if seeding
if( !strcmp( tests[i].roles[j].type, "seed" ) ) {
found = false;
for( m = 0; m < machines[k].fileCount; m++ ) {
if( !strcmp( machines[k].files[m], tests[i].roles[j].file ) ) {
found = true;
break;
}
}
if( !found )
machines[k].files[machines[k].fileCount++] = tests[i].roles[j].file;
}
// Machine found
found = true;
break;
}
}
if( !found )
quit( "Test %s role %i refers to machine %s which does not exist\n", tests[i].name, j, tests[i].roles[j].machines[l] );
}
}
}
// Create temporary script
FILE* script = tmpfile();
if( !script )
quit( "Can't create temporary script\n" );
fprintf( script, "#!/bin/bash\n" );
// == Check validity of machines and users: can each machine be accessed?
for( i = 0; i < machineCount; i++ ) {
// Creates a function in the script for sending command to the machine using ssh
// Call using
// ssh_machine_%i "commands" || cleanup -1
// where %i is the index of the machine in machines. Also be sure to return non-zero from your commands on error.
fprintf( script, "function ssh_machine_%i {\n", i );
if( strcmp( machines[i].address, "DAS4" ) )
fprintf( script, " ssh -T -n -o BatchMode=yes -h \"%s\"", machines[i].address );
else
fprintf( script, " ssh -T -n -o BatchMode=yes -h fs3.das4.tudelft.nl" );
if( machines[i].user )
fprintf( script, " -l \"%s\"", machines[i].user );
if( machines[i].params )
fprintf( script, " %s", machines[i].params );
fprintf( script, " $1 || return -1;\n" );
fprintf( script, "}\n" );
// Creates a function in the script for sending files to the machine using scp
// Call using
// scp_to_machine_%i localfile remotefile
fprintf( script, "function scp_to_machine_%i {\n", i );
fprintf( script, "scp -o BatchMode=yes " );
if( machines[i].params )
fprintf( script, "%s ", machines[i].params );
fprintf( script, "$1 ", l );
if( machines[i].user )
fprintf( script, "\"%s\"@", machines[i].user );
if( strcmp( machines[i].address, "DAS4" ) )
fprintf( script, "\"%s\"", machines[i].address );
else
fprintf( script, "fs3.das4.tudelft.nl" );
fprintf( script, ":$2 || cleanup -1\n", i, l )
fprintf( script, "}\n" );
// Creates a function in the script for retrieving files from the machine using scp
// Call using
// scp_from_machine_%i remotefile localfile
fprintf( script, "function scp_from_machine_%i {\n", i );
fprintf( script, "scp -o BatchMode=yes " );
if( machines[i].params )
fprintf( script, "%s ", machines[i].params );
if( machines[i].user )
fprintf( script, "\"%s\"@", machines[i].user );
if( strcmp( machines[i].address, "DAS4" ) )
fprintf( script, "\"%s\"", machines[i].address );
else
fprintf( script, "fs3.das4.tudelft.nl" );
fprintf( script, ":$1 $2 || cleanup -1\n", i, l )
fprintf( script, "}\n" );
// Checks reachability of machine
fprintf( script, "ssh_machine_%i || exit -1\n", i );
}
// == Check validity of each core: can each core be packaged? Can it be compiled locally and does the program then exist?
// Create a cleanup file. This file should have code appended to cleanup things when errors occur or testing has finished. See existing code for examples of concatenating to it.
// The cleanup function is available after this as well. Call it with an exit argument to end the script cleanly.
fprintf( script, "CLEANUPFILE=`mktemp`\n" );
fprintf( script, "chmod +x CLEANUPFILE\n" );
fprintf( script, "[ -x CLEANUPFILE ] || exit -1\n" );
fprintf( script, "function cleanup {\n" );
fprintf( script, " (\ncat <<EOL\nrm $CLEANUPFILE\nEOL\n) >> $CLEANUPFILE\n" );
fprintf( script, " . $CLEANUPFILE\n" );
fprintf( script " exit $1\n" );
fprintf( script, "}\n" );
// Create a local temporary directory for storage
fprintf( script, "TARDIR=`mktemp -d`\n[ \"X${TARDIR}X\" == \"XX\" ] && exit -1\n" );
fprintf( script, "(\ncat <<EOL\n#!/bin/bash\nrm -rf $TARDIR\nEOL\n) >> $CLEANUPFILE\n" );
fprintf( script, "CURDIR=`pwd`\n" );
// Create a tarball for the testenvironment in the temporary local storage
fprintf( script, "make clean || cleanup -1\n" );
fprintf( script, "tar cf ${TARDIR}/testenvironment.tar . || cleanup -1\n" );
fprintf( script, "bzip2 ${TARDIR}/testenvironment.tar || cleanup -1\n" );
// Check whether the needed tools of the testenvironment compile locally
fprintf( script, "make genfakedata || cleanup -1\n" );
for( i = 0; i < coreCount; i++ ) {
if( !cores[i].localdir )
cores[i].localdir = strdup( "../" );
// Create a tarball for the core in the temporary local storage
fprintf( script, "cd %s || cleanup -1\n", cores[i].localdir );
fprintf( script, "make clean\n" ); // Not checked: SHOULD be available, but...
fprintf( script, "tar cf ${TARDIR}/core_%i.tar . || cleanup -1\n", i );
fprintf( script, "bzip2 ${TARDIR}/core_%i.tar || cleanup -1\n", i );
if( !cores[i].compdir )
cores[i].compdir = strdup( "testenvironment/" );
// Check whether the core compiles locally and the program exists after
fprintf( script, "cd %s || cleanup -1\n", cores[i].compdir );
fprintf( script, "make || cleanup -1\n" );
if( !cores[i].program )
cores[i].program = strdup( "swift" );
fprintf( script, "[ -x %s ] || cleanup -1\n", cores[i].program );
fprintf( script, "cd ${CURDIR}\n" );
}
// For each file, precalculate the hash
for( i = 0; i < fileCount; i++ ) {
// Create some temporary file to write fake data to. These fake data files will be regenerated at each machine since generating is faster than copying.
size_t size = files[i].isize;
FILE* data = tmpfile();
if( !data )
quit( "can't create temporary data file\n" );
int datan = fileno( data );
int filesize;
if( size > 512*1024*1024 )
filesize = 512*1024*1024;
else
filesize = size;
if( generateFakeData( datan, filesize ) )
quit( "could not write fake data\n" );
MemoryHashStorage mhs;
FileOffsetDataStorage fods( datan, files[i].ioffset );
if( !fods.valid() )
quit( "can't read back from temporary data file\n" );
HashTree ht( fods, Sha1Hash::ZERO, mhs );
Sha1Hash hash = ht.root_hash();
if( size > filesize ) {
MemoryHashStorage mhs2;
int max = size/filesize;
for( j = 0; j < max; j++ )
mhs2.setHash( bin64(0,j), hash );
int lvl = 0;
do {
max >>= 1;
lvl++;
for( j = 0; j < max; j++ )
mhs2.hashLeftRight( bin64_t(lvl, j) );
} while( max > 1 );
files[i].hash = mhs.getHash( bin64_t(lvl, 0) );
}
else