int bin64_t::peaks (uint64_t length, bin64_t* peaks) { int pp=0; uint8_t layer = 0; while (length) { if (length&1) peaks[pp++] = bin64_t(layer,length^1); length>>=1; layer++; } for(int i=0; i<(pp>>1); i++) { uint64_t memo = peaks[pp-1-i]; peaks[pp-1-i] = peaks[i]; peaks[i] = memo; } peaks[pp] = NONE; return pp; }
void readConfig( const char* filename ) { /* States: * 0 : reading new entities ( to: 1, 3, 4, 5 ) * 1 : reading machine * 2 : reading role * 3 : reading core * 4 : reading file * 5 : reading test ( to: 2 ) */ int state = 0; struct machine m; struct role r; struct file f; struct core c; struct test t; xmlTextReaderPtr xmlRead = xmlReaderForFile( filename, NULL, XML_PARSE_NONET | XML_PARSE_NOENT | XML_PARSE_NOCDATA | XML_PARSE_NOXINCNODE | XML_PARSE_COMPACT ); if( !xmlRead ) quit( "Could not open %s for reading as XML config\n", filename ); xmlTextReaderSetErrorHandler( xmlRead, xmlErrorHandler, 0 ); int nodeType, ret; xmlChar* nodeName; while( 1 ) { if( ( ret = xmlTextReaderRead( xmlRead ) ) != 1 ) { if( ret < 0 ) quit( "Error occurred\n" ); if( state ) quit( "No more nodes to read, but state is not 0. Incomplete elements.\n" ); xmlTextReaderClose( xmlRead ); return; } nodeType = xmlTextReaderNodeType( xmlRead ); nodeName = NULL; switch( nodeType ) { case XmlNodeType.Element : nodeName = xmlTextReaderLocalName( xmlRead ); switch( state ) { case 0 : if( !strcmp( nodeName, "machine" ) ) { state = 1; bzero( m, sizeof( struct machine ) ); readValidRequiredAttribute( m.name, "machine", "name" ); readValidRequiredAttribute( m.address, "machine", "address" ); if( !strcmp( m.address, "DAS4" ) ) { readValidRequiredAttribute( m.count, "machine", "DAS4 node count" ); char* end = NULL; m.icount = strtol( m.count, &end, 10 ); if( end == m.count || *end || m.icount < 1 ) quit( "Invalid DAS4 node count for machine %s\n", m.name ); } break; } if( !strcmp( nodeName, "core" ) ) { state = 3; bzero( c, sizeof( struct core ) ); readValidRequiredAttribute( c.name, "core", "name" ); break; } if( !strcmp( nodeName, "file" ) ) { state = 4; bzero( f, sizeof( struct file ) ); readValidRequiredAttribute( f.name, "file", "name" ); readValidRequiredAttribute( f.size, "file", "size" ); char* end = NULL; f.isize = strtol( f.size, &end, 10 ); if( end == f.size || f.isize < 1 ) quit( "Invalid size specifier for file %s\n", f.name ); if( *end && *(end+1) ) quit( "Invalid size specifier for file %s\n", f.name ); if( *end ) { switch( *end ) { case 'b' : case 'B' : break; case 'k' : case 'K' : f.isize *= 1024; break; case 'm' : case 'M' : f.isize *= 1024 * 1024; break; case 'g' : case 'G' : f.isize *= 1024 * 1024 * 1024; break; case 't' : case 'T' : f.isize *= 1024 * 1024 * 1024 * 1024; break; default : quit( "Invalid size specifier for file %s\n", f.name ); } } if( f.isize > 512 * 1024 * 1024 ) { int p = 512*1024*1024; while( p < f.isize ) p <<= 1; if( p != f.isize ) quit( "Invalid size specifier for file %s: sizes above 512M should be powers of 2\n", f.name ); } if( f.isize & 0x3 ) quit( "Invalid size specifier for file %s: sizes should be a multiple of 4\n", f.name ); break; } if( !strcmp( nodeName, "test" ) ) { state = 5; bzero( t, sizeof( struct test ) ); readValidRequiredAttribute( t.name, "test", "name" ); break; } break; case 1 : if( !strcmp( nodeName, "tmpDir" ) ) { onlyOne( m.tmpdir, "machine", "tmpDir" ); readValidString( m.tmpdir, "temporary directory location" ); skipToEndElement( "tmpDir" ); break; } if( !strcmp( nodeName, "params" ) ) { onlyOne( m.params, "machine", "params" ); readValidString( m.params, "params" ); skipToEndElement( "params" ); break; } printf( "Unexpected element %s in machine %s\n", nodeName, m.name ); break; case 2 : if( !strcmp( nodeName, "machine" ) ) { if( r.machineCount > 255 ) quit( "Maximum of 256 machines per role passed on line %d\n", xmlTextReaderGetParserLineNumber( xmlRead ) ); readValid( r.machine[r.machineCount], "machine name" ); r.machineCount++; skipToEndElement( "machine" ); break; } if( !strcmp( nodeName, "user" ) ) { onlyOne( r.user, "role", "user" ); readValidString( r.user, "user" ); skipToEndElement( "user" ); break; } if( !strcmp( nodeName, "core" ) ) { onlyOne( r.core, "role", "core" ); readValidString( r.core, "core name" ); skipToEndElement( "core" ); break; } if( !strcmp( nodeName, "file" ) ) { onlyOne( r.file, "role", "file" ); readValidString( r.file, "file name" ); skipToEndElement( "file" ); break; } if( !strcmp( nodeName, "params" ) ) { onlyOne( r.params, "role", "params" ); readValidString( r.params, "params" ); skipToEndElement( "params" ); break; } printf( "Unexpected element %s in role on line %d\n", nodeName, xmlTextReaderGetParserLineNumber( xmlRead ) ); break; case 3 : if( !strcmp( nodeName, "params" ) ) { onlyOne( c.params, "core", "params" ); readValidString( c.params, "params" ); skipToEndElement( "params" ); break; } if( !strcmp( nodeName, "localDir" ) ) { onlyOne( c.localdir, "core", "localDir" ); readValidString( c.localdir, "local core directory" ); skipToEndElement( "localDir" ); break; } if( !strcmp( nodeName, "compilationDir" ) ) { onlyOne( c.compdir, "core", "compilationDir" ); readValidString( c.compdir, "relative compilation directory" ); skipToEndElement( "compilationDir" ); break; } if( !strcmp( nodeName, "program" ) ) { onlyOne( c.program, "core", "program" ); readValidString( c.program, "program name" ); skipToEndElement( "program" ); break; } printf( "Unexpected element %s in core %s\n", nodeName, c.name ); skipToEndElement( nodeName ); break; case 4 : if( !strcmp( nodeName, "offset" ) ) { onlyOne( c.offset, "file", "offset" ); readValidString( c.offset, "offset" ); char* end = NULL; f.ioffset = strtol( f.offset, &end, 10 ); if( end == f.offset || *end || f.ioffset < 0 || f >= f.isize ) quit( "Invalid file offset for file %s\n", f.name ); skipToEndElement( "offset" ); break; } printf( "Unexpected element %s in file %s\n", nodeName, f.name ); skipToEndElement( nodeName ); break; case 5 : if( !strcmp( nodeName, "role" ) ) { state = 2; bzero( r, sizeof( struct role ) ); readValidRequiredAttribute( r.type, "role", "type" ); if( strcmp( r.type, "seed" ) && strcmp( r.type, "leech" ) ) quit( "Invalid value for attribute 'type' for role on line %d, expected 'seed' or 'leech'\n", xmlTextReaderGetParserLineNumber( xmlRead ) ); break; } quit( "Unexpected element %s in test on line %d\n", nodeName, xmlTextReaderGetParserLineNumber( xmlRead ) ); default : quit( "State sanity\n", nodeName ); } break; case XmlNodeType.EndElement : nodeName = xmlTextReaderLocalName( xmlRead ); switch( state ) { case 0 : quit( "End element %s found while not in entity\n", nodeName ); case 1 : if( !strcmp( nodeName, "machine" ) ) { memcpy( machines + machineCount, &m, sizeof( struct machine ) ); machineCount++; state = 0; } break; case 2 : if( !strcmp( nodeName, "role" ) ) { mempcy( t.roles + r.roleCount, &r, sizeof( struct role ) ); t.roleCount++; state = 5; } break; case 3 : if( !strcmp( nodeName, "core" ) ) { memcpy( cores + coreCount, &c, sizeof( struct core ) ); coreCount++; state = 0; } break; case 4 : if( !strcmp( nodeName, "file" ) ) { memcpy( files + fileCount, &f, sizeof( struct file ) ); fileCount++; state = 0; } break; case 5 : if( !strcmp( nodeName, "test" ) ) { memcpy( tests + testCount, &t, sizeof( struct test ) ); testCount++; state = 0; } break; default : quit( "State sanity\n" ); } default : } if( nodeName ) free( nodeName ); } } void validateConfigAndGenerateScripts( ) { int i, j, k, l; bool found; // == Validate test-role references to machines, cores and files // == Also register which machine uses which cores and which files for( i = 0; i < testCount; i++ ) { for( j = 0; j < tests[i].roleCount; j++ ) { // Check for validity of each role's core found = false; for( k = 0; k < coreCount; k++ ) { if( !strcmp( cores[k].name, tests[i].roles[j].core ) ) { found = true; break; } } if( !found ) quit( "Test %s role %i refers to core %s which does not exist\n", tests[i].name, j, tests[i].roles[j].core ); // Check for validity of each role's file found = false; for( k = 0; k < fileCount; k++ ) { if( !strcmp( files[k].name, tests[i].roles[j].file ) ) { found = true; break; } } if( !found ) quit( "Test %s role %i refers to file %s which does not exist\n", tests[i].name, j, tests[i].roles[j].file ); // Check for validity of each role's machines for( l = 0; l < tests[i].roles[j].machineCount; l++ ) { found = false; for( k = 0; k < machineCount; k++ ) { if( !strcmp( machines[k].name, tests[i].roles[j].machines[l] ) ) { // Machine found: register used core with the machine if needed for( m = 0; m < machines[k].coreCount; m++ ) { if( !strcmp( machines[k].cores[m], tests[i].roles[j].core ) ) { found = true; break; } } if( !found ) machines[k].cores[machines[k].coreCount++] = tests[i].roles[j].core; // Machine found: register used file with the machine if needed, only if seeding if( !strcmp( tests[i].roles[j].type, "seed" ) ) { found = false; for( m = 0; m < machines[k].fileCount; m++ ) { if( !strcmp( machines[k].files[m], tests[i].roles[j].file ) ) { found = true; break; } } if( !found ) machines[k].files[machines[k].fileCount++] = tests[i].roles[j].file; } // Machine found found = true; break; } } if( !found ) quit( "Test %s role %i refers to machine %s which does not exist\n", tests[i].name, j, tests[i].roles[j].machines[l] ); } } } // Create temporary script FILE* script = tmpfile(); if( !script ) quit( "Can't create temporary script\n" ); fprintf( script, "#!/bin/bash\n" ); // == Check validity of machines and users: can each machine be accessed? for( i = 0; i < machineCount; i++ ) { // Creates a function in the script for sending command to the machine using ssh // Call using // ssh_machine_%i "commands" || cleanup -1 // where %i is the index of the machine in machines. Also be sure to return non-zero from your commands on error. fprintf( script, "function ssh_machine_%i {\n", i ); if( strcmp( machines[i].address, "DAS4" ) ) fprintf( script, " ssh -T -n -o BatchMode=yes -h \"%s\"", machines[i].address ); else fprintf( script, " ssh -T -n -o BatchMode=yes -h fs3.das4.tudelft.nl" ); if( machines[i].user ) fprintf( script, " -l \"%s\"", machines[i].user ); if( machines[i].params ) fprintf( script, " %s", machines[i].params ); fprintf( script, " $1 || return -1;\n" ); fprintf( script, "}\n" ); // Creates a function in the script for sending files to the machine using scp // Call using // scp_to_machine_%i localfile remotefile fprintf( script, "function scp_to_machine_%i {\n", i ); fprintf( script, "scp -o BatchMode=yes " ); if( machines[i].params ) fprintf( script, "%s ", machines[i].params ); fprintf( script, "$1 ", l ); if( machines[i].user ) fprintf( script, "\"%s\"@", machines[i].user ); if( strcmp( machines[i].address, "DAS4" ) ) fprintf( script, "\"%s\"", machines[i].address ); else fprintf( script, "fs3.das4.tudelft.nl" ); fprintf( script, ":$2 || cleanup -1\n", i, l ) fprintf( script, "}\n" ); // Creates a function in the script for retrieving files from the machine using scp // Call using // scp_from_machine_%i remotefile localfile fprintf( script, "function scp_from_machine_%i {\n", i ); fprintf( script, "scp -o BatchMode=yes " ); if( machines[i].params ) fprintf( script, "%s ", machines[i].params ); if( machines[i].user ) fprintf( script, "\"%s\"@", machines[i].user ); if( strcmp( machines[i].address, "DAS4" ) ) fprintf( script, "\"%s\"", machines[i].address ); else fprintf( script, "fs3.das4.tudelft.nl" ); fprintf( script, ":$1 $2 || cleanup -1\n", i, l ) fprintf( script, "}\n" ); // Checks reachability of machine fprintf( script, "ssh_machine_%i || exit -1\n", i ); } // == Check validity of each core: can each core be packaged? Can it be compiled locally and does the program then exist? // Create a cleanup file. This file should have code appended to cleanup things when errors occur or testing has finished. See existing code for examples of concatenating to it. // The cleanup function is available after this as well. Call it with an exit argument to end the script cleanly. fprintf( script, "CLEANUPFILE=`mktemp`\n" ); fprintf( script, "chmod +x CLEANUPFILE\n" ); fprintf( script, "[ -x CLEANUPFILE ] || exit -1\n" ); fprintf( script, "function cleanup {\n" ); fprintf( script, " (\ncat <<EOL\nrm $CLEANUPFILE\nEOL\n) >> $CLEANUPFILE\n" ); fprintf( script, " . $CLEANUPFILE\n" ); fprintf( script " exit $1\n" ); fprintf( script, "}\n" ); // Create a local temporary directory for storage fprintf( script, "TARDIR=`mktemp -d`\n[ \"X${TARDIR}X\" == \"XX\" ] && exit -1\n" ); fprintf( script, "(\ncat <<EOL\n#!/bin/bash\nrm -rf $TARDIR\nEOL\n) >> $CLEANUPFILE\n" ); fprintf( script, "CURDIR=`pwd`\n" ); // Create a tarball for the testenvironment in the temporary local storage fprintf( script, "make clean || cleanup -1\n" ); fprintf( script, "tar cf ${TARDIR}/testenvironment.tar . || cleanup -1\n" ); fprintf( script, "bzip2 ${TARDIR}/testenvironment.tar || cleanup -1\n" ); // Check whether the needed tools of the testenvironment compile locally fprintf( script, "make genfakedata || cleanup -1\n" ); for( i = 0; i < coreCount; i++ ) { if( !cores[i].localdir ) cores[i].localdir = strdup( "../" ); // Create a tarball for the core in the temporary local storage fprintf( script, "cd %s || cleanup -1\n", cores[i].localdir ); fprintf( script, "make clean\n" ); // Not checked: SHOULD be available, but... fprintf( script, "tar cf ${TARDIR}/core_%i.tar . || cleanup -1\n", i ); fprintf( script, "bzip2 ${TARDIR}/core_%i.tar || cleanup -1\n", i ); if( !cores[i].compdir ) cores[i].compdir = strdup( "testenvironment/" ); // Check whether the core compiles locally and the program exists after fprintf( script, "cd %s || cleanup -1\n", cores[i].compdir ); fprintf( script, "make || cleanup -1\n" ); if( !cores[i].program ) cores[i].program = strdup( "swift" ); fprintf( script, "[ -x %s ] || cleanup -1\n", cores[i].program ); fprintf( script, "cd ${CURDIR}\n" ); } // For each file, precalculate the hash for( i = 0; i < fileCount; i++ ) { // Create some temporary file to write fake data to. These fake data files will be regenerated at each machine since generating is faster than copying. size_t size = files[i].isize; FILE* data = tmpfile(); if( !data ) quit( "can't create temporary data file\n" ); int datan = fileno( data ); int filesize; if( size > 512*1024*1024 ) filesize = 512*1024*1024; else filesize = size; if( generateFakeData( datan, filesize ) ) quit( "could not write fake data\n" ); MemoryHashStorage mhs; FileOffsetDataStorage fods( datan, files[i].ioffset ); if( !fods.valid() ) quit( "can't read back from temporary data file\n" ); HashTree ht( fods, Sha1Hash::ZERO, mhs ); Sha1Hash hash = ht.root_hash(); if( size > filesize ) { MemoryHashStorage mhs2; int max = size/filesize; for( j = 0; j < max; j++ ) mhs2.setHash( bin64(0,j), hash ); int lvl = 0; do { max >>= 1; lvl++; for( j = 0; j < max; j++ ) mhs2.hashLeftRight( bin64_t(lvl, j) ); } while( max > 1 ); files[i].hash = mhs.getHash( bin64_t(lvl, 0) ); } else