// Generate //------------------------------------------------------------------------------ void Report::Generate( const FBuildStats & stats ) { Timer t; // pre-allocate a large string for output m_Output.SetReserved( MEGABYTE ); m_Output.SetLength( 0 ); // generate some common data used in reporting GetLibraryStats( stats ); // build the report CreateHeader(); CreateTitle(); CreateOverview( stats ); DoCPUTimeByType( stats ); DoCacheStats( stats ); DoCPUTimeByLibrary(); DoCPUTimeByItem( stats ); DoIncludes(); CreateFooter(); // patch in time take const float time = t.GetElapsed(); AStackString<> timeTakenBuffer; stats.FormatTime( time, timeTakenBuffer ); char * placeholder = m_Output.Find( "^^^^ " ); memcpy( placeholder, timeTakenBuffer.Get(), timeTakenBuffer.GetLength() ); }
// AStackStringOverflow //------------------------------------------------------------------------------ void TestAString::AStackStringOverflow() const { { // constructor with string longer than buffer AStackString< 8 > string( "01234567890123456789" ); TEST_ASSERT( string.GetLength() == 20 ); TEST_ASSERT( string.GetLength() == AString::StrLen( string.Get() ) ); } { // assigned of string longer than buffer AStackString< 8 > string; string = "01234567890123456789"; TEST_ASSERT( string.GetLength() == 20 ); TEST_ASSERT( string.GetLength() == AString::StrLen( string.Get() ) ); } { // concetentation of string longer than buffer AStackString< 8 > string; string += "01234567890123456789"; TEST_ASSERT( string.GetLength() == 20 ); TEST_ASSERT( string.GetLength() == AString::StrLen( string.Get() ) ); } }
// Format //------------------------------------------------------------------------------ void TestAString::Format() const { // Create a really long input string AStackString<> longInput; const size_t longStringLen( 1024 * 1024 ); for ( size_t i=0; i<longStringLen; ++i ) { longInput += 'A'; } // Make sure we correctly handle formatting large strings AStackString<> buffer; buffer.Format( "%s", longInput.Get() ); TEST_ASSERT( buffer.GetLength() == longStringLen ); TEST_ASSERT( AString::StrLen( buffer.Get() ) == longStringLen ); }
// WriteNestedProjects //------------------------------------------------------------------------------ void SLNGenerator::WriteNestedProjects( const Array< AString > & solutionProjectsToFolder, const Array< AString > & solutionFolderPaths ) { if ( solutionProjectsToFolder.GetSize() == 0 && solutionFolderPaths.GetSize() == 0 ) { return; // skip global section } Write( "\tGlobalSection(NestedProjects) = preSolution\r\n" ); // Write every project to solution folder relationships const AString * const solutionProjectsToFolderEnd = solutionProjectsToFolder.End(); for( const AString * it = solutionProjectsToFolder.Begin() ; it != solutionProjectsToFolderEnd ; ++it ) { Write( it->Get() ); } // Write every intermediate path const AString * const solutionFolderPathsEnd = solutionFolderPaths.End(); for( const AString * it = solutionFolderPaths.Begin() ; it != solutionFolderPathsEnd ; ++it ) { // parse solution folder parent path AStackString<> solutionFolderParentGuid; const char * lastSlash = it->FindLast( NATIVE_SLASH ); if ( lastSlash ) { AStackString<> solutionFolderParentPath( it->Get(), lastSlash ); VSProjectGenerator::FormatDeterministicProjectGUID( solutionFolderParentGuid, solutionFolderParentPath ); } if ( solutionFolderParentGuid.GetLength() > 0 ) { // generate a guid for the solution folder AStackString<> solutionFolderGuid; VSProjectGenerator::FormatDeterministicProjectGUID( solutionFolderGuid, *it ); solutionFolderGuid.ToUpper(); solutionFolderParentGuid.ToUpper(); // write parent solution folder relationship Write( "\t\t%s = %s\r\n", solutionFolderGuid.Get(), solutionFolderParentGuid.Get() ); } } Write( "\tEndGlobalSection\r\n" ); }
// Write //------------------------------------------------------------------------------ void VSProjectGenerator::Write( const char * fmtString, ... ) { AStackString< 1024 > tmp; va_list args; va_start(args, fmtString); tmp.VFormat( fmtString, args ); va_end( args ); // resize output buffer in large chunks to prevent re-sizing if ( m_Tmp.GetLength() + tmp.GetLength() > m_Tmp.GetReserved() ) { m_Tmp.SetReserved( m_Tmp.GetReserved() + MEGABYTE ); } m_Tmp += tmp; }
// BigString //------------------------------------------------------------------------------ void TestAString::BigString() const { // create a massive string AutoPtr< char > mem( (char *)ALLOC( ( 10 * MEGABYTE ) + 1 ) ); memset( mem.Get(), 'a', 10 * MEGABYTE ); mem.Get()[ 10 * MEGABYTE ] = '\000'; // create a stack string AStackString< 2048 > string; // now overflow massively so the buffer resizes // (string will now be 10MB, which is well over the 64K limit // previously imposed on strings) string += mem.Get(); TEST_ASSERT( string.GetLength() == 10 * MEGABYTE ); TEST_ASSERT( string.GetLength() == AString::StrLen( string.Get() ) ); }
// AStackStringConstructors //------------------------------------------------------------------------------ void TestAString::AStackStringConstructors() const { { // AStackString<> with no arguments AStackString<> empty; TEST_ASSERT( empty.GetLength() == 0 ); TEST_ASSERT( empty.GetReserved() > 0 ); TEST_ASSERT( empty.IsEmpty() == true ); TEST_ASSERT( empty.MemoryMustBeFreed() == false ); } { // AStackString<> from char * AStackString<> fromCharStar( "hello" ); TEST_ASSERT( fromCharStar.GetLength() == 5 ); TEST_ASSERT( fromCharStar.GetReserved() >= 5 ); TEST_ASSERT( fromCharStar.IsEmpty() == false ); TEST_ASSERT( fromCharStar.MemoryMustBeFreed() == false ); // AStackString<> from AStackString AStackString<> fromAString( fromCharStar ); TEST_ASSERT( fromAString.GetLength() == 5 ); TEST_ASSERT( fromAString.GetReserved() >= 5 ); TEST_ASSERT( fromAString.IsEmpty() == false ); TEST_ASSERT( fromAString.MemoryMustBeFreed() == false ); } { // AStackString<> from AString AString astring( "hello" ); AStackString<> fromAString( astring ); TEST_ASSERT( fromAString.GetLength() == 5 ); TEST_ASSERT( fromAString.GetReserved() >= 5 ); TEST_ASSERT( fromAString.IsEmpty() == false ); TEST_ASSERT( fromAString.MemoryMustBeFreed() == false ); } { const char * hello = "hellohellohello"; AStackString<> fromCharStarPair( hello, hello + 5 ); TEST_ASSERT( fromCharStarPair.GetLength() == 5 ); TEST_ASSERT( fromCharStarPair.GetReserved() >= 5 ); TEST_ASSERT( fromCharStarPair.IsEmpty() == false ); TEST_ASSERT( fromCharStarPair.MemoryMustBeFreed() == false ); } }
// TracingOutputCallback //------------------------------------------------------------------------------ /*static*/ bool FLog::TracingOutputCallback( const char * message ) { uint32_t threadIndex = WorkerThread::GetThreadIndex(); AStackString< 2048 > tmp; if ( s_ShowProgress ) { // clear previous progress message tmp += g_ClearLineString; } // print output and then progress if ( threadIndex > 0 ) { char buffer[ 8 ]; _itoa_s( threadIndex, buffer, 8, 10 ); tmp += buffer; tmp += '>'; if ( threadIndex < 10 ) { tmp += ' '; // keep output aligned when there are > 9 threads } } tmp += message; // output to debugger if present #ifdef DEBUG #ifdef __WINDOWS__ OutputDebugStringA( message ); #endif #endif tmp += m_ProgressText; fwrite( tmp.Get(), 1, tmp.GetLength(), stdout ); return false; // tell tracing not to output it again }
// Monitor //------------------------------------------------------------------------------ /*static*/ void FLog::Monitor( const char * formatString, ... ) { // Is monitoring enabled? if ( g_MonitorFileStream == nullptr ) { return; // No - nothing to do } PROFILE_SECTION( "FLog::Monitor" ) AStackString< 1024 > buffer; va_list args; va_start( args, formatString ); buffer.VFormat( formatString, args ); va_end( args ); AStackString< 1024 > finalBuffer; finalBuffer.Format( "%llu %s", Time::GetCurrentFileTime(), buffer.Get() ); MutexHolder lock( g_MonitorMutex ); g_MonitorFileStream->WriteBuffer( finalBuffer.Get(), finalBuffer.GetLength() ); }
// CompareHashTimes_Small //------------------------------------------------------------------------------ void TestHash::CompareHashTimes_Small() const { // some different strings to hash Array< AString > strings( 32, true ); strings.Append( AString( " " ) ); strings.Append( AString( "short" ) ); strings.Append( AString( "mediumstringmediumstring123456789" ) ); strings.Append( AString( "longstring_98274ncoif834jodhiorhmwe8r8wy48on87h8mhwejrijrdierwurd9j,8chm8hiuorciwriowjri" ) ); strings.Append( AString( "c:\\files\\subdir\\project\\thing\\stuff.cpp" ) ); const size_t numStrings = strings.GetSize(); const size_t numIterations = 102400; // calc datasize size_t dataSize( 0 ); for ( size_t i=0; i<numStrings; ++i ) { dataSize += strings[ i ].GetLength(); } dataSize *= numIterations; // xxHash - 32 { Timer t; uint32_t crc( 0 ); for ( size_t j=0; j<numIterations; ++j ) { for ( size_t i=0; i<numStrings; ++i ) { crc += xxHash::Calc32( strings[ i ].Get(), strings[ i ].GetLength() ); } } float time = t.GetElapsed(); float speed = ( (float)dataSize / (float)( 1024 * 1024 * 1024 ) ) / time; OUTPUT( "xxHash-32 : %2.3fs @ %6.3f GiB/s (hash: 0x%x)\n", time, speed, crc ); } // xxHash - 64 { Timer t; uint64_t crc( 0 ); for ( size_t j=0; j<numIterations; ++j ) { for ( size_t i=0; i<numStrings; ++i ) { crc += xxHash::Calc64( strings[ i ].Get(), strings[ i ].GetLength() ); } } float time = t.GetElapsed(); float speed = ( (float)dataSize / (float)( 1024 * 1024 * 1024 ) ) / time; OUTPUT( "xxHash-64 : %2.3fs @ %6.3f GiB/s (hash: %016llx)\n", time, speed, crc ); } // Murmur3 - 32 { Timer t; uint32_t crc( 0 ); for ( size_t j=0; j<numIterations; ++j ) { for ( size_t i=0; i<numStrings; ++i ) { crc += Murmur3::Calc32( strings[ i ].Get(), strings[ i ].GetLength() ); } } float time = t.GetElapsed(); float speed = ( (float)dataSize / (float)( 1024 * 1024 * 1024 ) ) / time; OUTPUT( "Murmur3-32 : %2.3fs @ %6.3f GiB/s (hash: 0x%x)\n", time, speed, crc ); } // Murmur3 - 128 { Timer t; uint64_t hashB( 0 ); uint64_t hashA( 0 ); for ( size_t j=0; j<numIterations; ++j ) { for ( size_t i=0; i<numStrings; ++i ) { hashA += Murmur3::Calc128( strings[ i ].Get(), strings[ i ].GetLength(), hashB ); } } float time = t.GetElapsed(); float speed = ( (float)dataSize / (float)( 1024 * 1024 * 1024 ) ) / time; OUTPUT( "Murmur3-128 : %2.3fs @ %6.3f GiB/s (%016llx, %016llx)\n", time, speed, hashA, hashB ); } // CRC32 - 8x8 slicing { Timer t; uint32_t crc( 0 ); for ( size_t j=0; j<numIterations; ++j ) { for ( size_t i=0; i<numStrings; ++i ) { crc += CRC32::Calc( strings[ i ].Get(), strings[ i ].GetLength() ); } } float time = t.GetElapsed(); float speed = ( (float)dataSize / (float)( 1024 * 1024 * 1024 ) ) / time; OUTPUT( "CRC32 8x8 : %2.3fs @ %6.3f GiB/s (hash: 0x%x)\n", time, speed, crc ); } // CRC32 - "standard" algorithm { Timer t; uint32_t crc( 0 ); for ( size_t j=0; j<numIterations; ++j ) { for ( size_t i=0; i<numStrings; ++i ) { crc += CRC32::Start(); crc += CRC32::Update( crc, strings[ i ].Get(), strings[ i ].GetLength() ); crc += CRC32::Stop( crc ); } } float time = t.GetElapsed(); float speed = ( (float)dataSize / (float)( 1024 * 1024 * 1024 ) ) / time; OUTPUT( "CRC32 : %2.3fs @ %6.3f GiB/s (hash: 0x%x)\n", time, speed, crc ); } // CRC32Lower { Timer t; uint32_t crc( 0 ); for ( size_t j=0; j<numIterations; ++j ) { for ( size_t i=0; i<numStrings; ++i ) { crc += CRC32::CalcLower( strings[ i ].Get(), strings[ i ].GetLength() ); } } float time = t.GetElapsed(); float speed = ( (float)dataSize / (float)( 1024 * 1024 * 1024 ) ) / time; OUTPUT( "CRC32Lower : %2.3fs @ %6.3f GiB/s (hash: 0x%x)\n", time, speed, crc ); } // Murmur3 - 32 Lower { Timer t; // lower-case and hash it uint32_t crc( 0 ); for ( size_t j=0; j<numIterations; ++j ) { for ( size_t i=0; i<numStrings; ++i ) { const AString & str( strings[ i ] ); AStackString<> tmp; const size_t strLen( str.GetLength() ); tmp.SetLength( (uint32_t)strLen ); for ( size_t k=0; k<strLen; ++k ) { char c = str[ (uint32_t)k ]; tmp[ (uint32_t)k ] = ( ( c >= 'A' ) && ( c <= 'Z' ) ) ? 'a' + ( c - 'A' ) : c; } crc += Murmur3::Calc32( tmp.Get(), tmp.GetLength() ); } } float time = t.GetElapsed(); float speed = ( (float)dataSize / (float)( 1024 * 1024 * 1024 ) ) / time; OUTPUT( "Murmur3-32-Lower: %2.3fs @ %6.3f GiB/s (hash: 0x%x)\n", time, speed, crc ); } }
// Deserialize //------------------------------------------------------------------------------ void ToolManifest::Deserialize( IOStream & ms ) { ms.Read( m_ToolId ); ASSERT( m_Files.IsEmpty() ); uint32_t numFiles( 0 ); ms.Read( numFiles ); m_Files.SetCapacity( numFiles ); for ( size_t i=0; i<(size_t)numFiles; ++i ) { AStackString<> name; uint64_t timeStamp( 0 ); uint32_t hash( 0 ); uint32_t contentSize( 0 ); ms.Read( name ); ms.Read( timeStamp ); ms.Read( hash ); ms.Read( contentSize ); m_Files.Append( File( name, timeStamp, hash, nullptr, contentSize ) ); } // determine if any files are remaining from a previous run size_t numFilesAlreadySynchronized = 0; for ( size_t i=0; i<(size_t)numFiles; ++i ) { AStackString<> localFile; GetRemoteFilePath( (uint32_t)i, localFile ); // is this file already present? AutoPtr< FileStream > fileStream( FNEW( FileStream ) ); FileStream & f = *( fileStream.Get() ); if ( f.Open( localFile.Get() ) == false ) { continue; // file not found } if ( f.GetFileSize() != m_Files[ i ].m_ContentSize ) { continue; // file is not complete } AutoPtr< char > mem( (char *)ALLOC( (size_t)f.GetFileSize() ) ); if ( f.Read( mem.Get(), (size_t)f.GetFileSize() ) != f.GetFileSize() ) { continue; // problem reading file } if( Murmur3::Calc32( mem.Get(), (size_t)f.GetFileSize() ) != m_Files[ i ].m_Hash ) { continue; // file contents unexpected } // file present and ok m_Files[ i ].m_FileLock = fileStream.Release(); // NOTE: keep file open to prevent deletions m_Files[ i ].m_SyncState = File::SYNCHRONIZED; numFilesAlreadySynchronized++; } // Generate Environment ASSERT( m_RemoteEnvironmentString == nullptr ); // PATH= AStackString<> basePath; GetRemotePath( basePath ); AStackString<> paths; paths.Format( "PATH=%s", basePath.Get() ); // TMP= AStackString<> normalTmp; Env::GetEnvVariable( "TMP", normalTmp ); AStackString<> tmp; tmp.Format( "TMP=%s", normalTmp.Get() ); // SystemRoot= AStackString<> sysRoot( "SystemRoot=C:\\Windows" ); char * mem = (char *)ALLOC( paths.GetLength() + 1 + tmp.GetLength() + 1 + sysRoot.GetLength() + 1 + 1 ); m_RemoteEnvironmentString = mem; AString::Copy( paths.Get(), mem, paths.GetLength() + 1 ); // including null mem += ( paths.GetLength() + 1 ); // including null AString::Copy( tmp.Get(), mem, tmp.GetLength() + 1 ); // including null mem += ( tmp.GetLength() + 1 ); // including null AString::Copy( sysRoot.Get(), mem, sysRoot.GetLength() + 1 ); // including null mem += ( sysRoot.GetLength() + 1 ); // including null *mem = 0; ++mem; // double null // are all files already present? if ( numFilesAlreadySynchronized == m_Files.GetSize() ) { m_Synchronized = true; } }
// ParseNamedVariableName //------------------------------------------------------------------------------ /*static*/ bool BFFParser::ParseVariableName( BFFIterator & iter, AString & name, bool & parentScope ) { // skip over the declaration symbol ASSERT( *iter == BFF_DECLARE_VAR_INTERNAL || *iter == BFF_DECLARE_VAR_PARENT ); parentScope = ( *iter == BFF_DECLARE_VAR_PARENT ); const BFFIterator varNameStart = iter; // include type token in var name iter++; // make sure we haven't hit the end of the file if ( iter.IsAtEnd() ) { Error::Error_1012_UnexpectedEndOfFile( iter ); return false; } if ( *iter == '\'' || *iter == '"' ) { // parse the string const BFFIterator openToken = iter; iter.SkipString( *openToken ); if ( *iter != *openToken ) { Error::Error_1002_MatchingClosingTokenNotFound( openToken, nullptr, *openToken ); return false; } BFFIterator stringStart = openToken; stringStart++; // unescape and subsitute embedded variables AStackString< 256 > value; if ( PerformVariableSubstitutions( stringStart, iter, value ) == false ) { return false; } iter++; // skip close token BFFIterator varNameIter( value.Get(), value.GetLength(), iter.GetFileName().Get(), iter.GetFileTimeStamp() ); // sanity check it is a sensible length if ( value.GetLength() + 1/* '.' will be added */ > MAX_VARIABLE_NAME_LENGTH ) { Error::Error_1014_VariableNameIsTooLong( varNameIter, (uint32_t)value.GetLength(), (uint32_t)MAX_VARIABLE_NAME_LENGTH ); return false; } // sanity check it is a valid variable name while ( varNameIter.IsAtEnd() == false ) { if ( varNameIter.IsAtValidVariableNameCharacter() == false ) { Error::Error_1013_UnexpectedCharInVariableName( varNameIter, nullptr ); return false; } varNameIter++; } // append '.' to variable name name = "."; name.Append( value ); } else { // make sure immediately after the symbol starts a variable name if ( iter.IsAtValidVariableNameCharacter() == false ) { Error::Error_1013_UnexpectedCharInVariableName( iter, nullptr ); return false; } // find the end of the variable name iter.SkipVariableName(); const BFFIterator varNameEnd = iter; // sanity check it is a sensible length size_t varNameLen = varNameStart.GetDistTo( varNameEnd ); if ( varNameLen > MAX_VARIABLE_NAME_LENGTH ) { Error::Error_1014_VariableNameIsTooLong( iter, (uint32_t)varNameLen, (uint32_t)MAX_VARIABLE_NAME_LENGTH ); return false; } // store variable name name.Assign( varNameStart.GetCurrent(), varNameEnd.GetCurrent() ); } ASSERT( name.GetLength() > 0 ); if ( parentScope ) { // exchange '^' with '.' ASSERT( BFF_DECLARE_VAR_PARENT == name[0] ); name[0] = BFF_DECLARE_VAR_INTERNAL; } return true; }
// Commit //------------------------------------------------------------------------------ /*virtual*/ bool FunctionCopy::Commit( NodeGraph & nodeGraph, const BFFIterator & funcStartIter ) const { // make sure all required variables are defined Array< AString > sources( 16, true ); const BFFVariable * dstFileV; if ( !GetStrings( funcStartIter, sources, ".Source", true ) || !GetString( funcStartIter, dstFileV, ".Dest", true ) ) { return false; // GetString will have emitted errors } // Optional AStackString<> sourceBasePath; if ( !GetString( funcStartIter, sourceBasePath, ".SourceBasePath", false ) ) { return false; // GetString will have emitted errors } // Canonicalize the SourceBasePath if ( !sourceBasePath.IsEmpty() ) { AStackString<> cleanValue; NodeGraph::CleanPath( sourceBasePath, cleanValue ); PathUtils::EnsureTrailingSlash( cleanValue ); sourceBasePath = cleanValue; } // check sources are not paths { const AString * const end = sources.End(); for ( const AString * it = sources.Begin(); it != end; ++it ) { const AString & srcFile( *it ); // source must be a file, not a path if ( PathUtils::IsFolderPath( srcFile ) ) { Error::Error_1105_PathNotAllowed( funcStartIter, this, ".Source", srcFile ); return false; } } } // Pre-build dependencies Dependencies preBuildDependencies; if ( !GetNodeList( nodeGraph, funcStartIter, ".PreBuildDependencies", preBuildDependencies, false ) ) { return false; // GetNodeList will have emitted an error } Array< AString > preBuildDependencyNames( preBuildDependencies.GetSize(), false ); for ( const auto & dep : preBuildDependencies ) { preBuildDependencyNames.Append( dep.GetNode()->GetName() ); } // get source node Array< Node * > srcNodes; { const AString * const end = sources.End(); for ( const AString * it = sources.Begin(); it != end; ++it ) { Node * srcNode = nodeGraph.FindNode( *it ); if ( srcNode ) { if ( GetSourceNodes( funcStartIter, srcNode, srcNodes ) == false ) { return false; } } else { // source file not defined by use - assume an external file srcNodes.Append( nodeGraph.CreateFileNode( *it ) ); } } } AStackString<> dstFile; NodeGraph::CleanPath( dstFileV->GetString(), dstFile ); const bool dstIsFolderPath = PathUtils::IsFolderPath( dstFile ); // make all the nodes for copies Dependencies copyNodes( srcNodes.GetSize(), false ); for ( const Node * srcNode : srcNodes ) { AStackString<> dst( dstFile ); // dest can be a file OR a path. If it's a path, use the source filename part if ( dstIsFolderPath ) { // find filename part of source const AString & srcName = srcNode->GetName(); // If the sourceBasePath is specified (and valid) use the name relative to that if ( !sourceBasePath.IsEmpty() && PathUtils::PathBeginsWith( srcName, sourceBasePath ) ) { // Use everything relative to the SourceBasePath dst += srcName.Get() + sourceBasePath.GetLength(); } else { // Use just the file name const char * lastSlash = srcName.FindLast( NATIVE_SLASH ); dst += lastSlash ? ( lastSlash + 1 ) // append filename part if found : srcName.Get(); // otherwise append whole thing } } // check node doesn't already exist if ( nodeGraph.FindNode( dst ) ) { // TODO:C could have a specific error for multiple sources with only 1 output // to differentiate from two rules creating the same dst target Error::Error_1100_AlreadyDefined( funcStartIter, this, dst ); return false; } // create our node CopyFileNode * copyFileNode = nodeGraph.CreateCopyFileNode( dst ); copyFileNode->m_Source = srcNode->GetName(); copyFileNode->m_PreBuildDependencyNames = preBuildDependencyNames; if ( !copyFileNode->Initialize( nodeGraph, funcStartIter, this ) ) { return false; // Initialize will have emitted an error } copyNodes.Append( Dependency( copyFileNode ) ); } // handle alias creation return ProcessAlias( nodeGraph, funcStartIter, copyNodes ); }