BOOL
FGetLex(LEX * plex,
BOOL fInComment)
{
int ch;
LEX lex;
char *pchStore;
lex.lt = plex->lt = ltNil;
if (!FSkipWhite())
return fFalse;
if (fInComment)
{
if (!FSkipToEndOfCComment())
lex.lt = ltCComment; /* keep going with it */
else
lex.lt = ltEndCComment; /* ok, the comment is over */
}
else
{
pchStore = lex.szId;
ch = *pchStore++ = *pchLex++;
*pchStore = 0;
switch (ch)
{
/*
* BUG! could use a lookup table...
*/
/*
* TODO logical operators
*/
SLT(0xA0, ltConst) /* RMa add to support TRAP special case */
SLT('.', ltPoint) SLT('+', ltPlus) SLT('-', ltMinus) SLT('*', ltMult) SLT('%', ltMod) SLT('(', ltLParen) SLT(')', ltRParen) SLT('[', ltLBracket) SLT(']', ltRBracket) SLT('{', ltLBrace) SLT('}', ltRBrace) SLT(',', ltComma) SLT('?', ltQuestion) SLT(':', ltColon) SLT('^', ltCaret) SLT('\\', ltBSlash) SLT('#', ltPound) SLT('@', ltAt) SLT(';', ltSemi) SLT('|', ltPipe) case '/':
if (*pchLex == '/')
{
*pchStore++ = *pchLex++;
*pchStore = 0;
lex.lt = ltDoubleSlash;
}
else if (*pchLex == '*')
{
commentDepth = 1;
pchLex++;
if (!FSkipToEndOfCComment())
lex.lt = ltCComment;
else
lex.lt = ltEndCComment; /* return place holder token */
}
else
lex.lt = ltDiv;
break;
case '<':
if (*pchLex == '=')
{
*pchStore++ = *pchLex++;
*pchStore = 0;
lex.lt = ltLTE;
}
else if (*pchLex == '>')
{
*pchStore++ = *pchLex++;
*pchStore = 0;
lex.lt = ltNE;
}
else
lex.lt = ltLT;
break;
case '>':
if (*pchLex == '=')
{
*pchStore++ = *pchLex++;
*pchStore = 0;
lex.lt = ltGTE;
}
else
lex.lt = ltGT;
break;
case '=':
if (*pchLex == '=')
{
*pchStore++ = *pchLex++;
*pchStore = 0;
lex.lt = ltEQ;
}
else
lex.lt = ltAssign;
break;
case '"':
lex.lt = ltStr;
pchStore = lex.szId;
while (*pchLex != '"')
{
int n, tmp;
n = (*pfnChkCode) ((const unsigned char *)pchLex, &tmp);
if (n >= 1)
{
while (n-- > 0)
*pchStore++ = *pchLex++;
}
else if (*pchLex == '\\')
{
int ch;
pchLex++;
ch = *pchLex++;
switch (ch)
{
case 'a':
ch = '\a';
break;
case 'b':
ch = '\b';
break;
case 'f':
ch = '\f';
break;
case 'n':
ch = '\n';
break;
case 'r':
ch = '\r';
break;
case 't':
ch = '\t';
break;
case 'v':
ch = '\v';
break;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
ch = ChParseOctal(ch);
break;
case 'x':
case 'X':
ch = ChParseHex(ch);
break;
case '\0': /* handle slash at the end of the line */
ch = '\\';
break;
case 'e': /* ESC, thank you Richard Brinkerhoff */
ch = '\033';
break;
case 'z': /* special control z'er */
ch = 'z' & 0x1f;
break;
case '_': /* special ignore - turns to nothing */
ch = '\0';
break;
}
/*** This program does not handle nulls in strings ***/
if (ch)
*pchStore++ = (char)ch; /* gratuitous cast - Unicode alert!!!! */
}
else
*pchStore++ = *pchLex++;
if (pchStore - lex.szId == cchIdMax - 1)
{
ErrorLine("String too long");
break;
}
if (*pchLex == 0)
{
ErrorLine("Unterminated string");
break;
}
}
pchLex++;
*pchStore = 0;
break;
default:
if (FParseConst(&lex, ch) || FParseId(&lex, ch))
{
/*
* do nuthin...code is easier to read this way
*/
}
else
{
ErrorLine("Unknown character: '%c'", ch);
}
pchLex--;
break;
}
}
*plex = lex;
return lex.lt != ltNil;
}
void FStatsMemoryDumpCommand::InternalRun()
{
FParse::Value( FCommandLine::Get(), TEXT( "-INFILE=" ), SourceFilepath );
const int64 Size = IFileManager::Get().FileSize( *SourceFilepath );
if( Size < 4 )
{
UE_LOG( LogStats, Error, TEXT( "Could not open: %s" ), *SourceFilepath );
return;
}
TAutoPtr<FArchive> FileReader( IFileManager::Get().CreateFileReader( *SourceFilepath ) );
if( !FileReader )
{
UE_LOG( LogStats, Error, TEXT( "Could not open: %s" ), *SourceFilepath );
return;
}
if( !Stream.ReadHeader( *FileReader ) )
{
UE_LOG( LogStats, Error, TEXT( "Could not open, bad magic: %s" ), *SourceFilepath );
return;
}
UE_LOG( LogStats, Warning, TEXT( "Reading a raw stats file for memory profiling: %s" ), *SourceFilepath );
const bool bIsFinalized = Stream.Header.IsFinalized();
check( bIsFinalized );
check( Stream.Header.Version == EStatMagicWithHeader::VERSION_5 );
StatsThreadStats.MarkAsLoaded();
TArray<FStatMessage> Messages;
if( Stream.Header.bRawStatsFile )
{
FScopeLogTime SLT( TEXT( "FStatsMemoryDumpCommand::InternalRun" ), nullptr, FScopeLogTime::ScopeLog_Seconds );
// Read metadata.
TArray<FStatMessage> MetadataMessages;
Stream.ReadFNamesAndMetadataMessages( *FileReader, MetadataMessages );
StatsThreadStats.ProcessMetaDataOnly( MetadataMessages );
// Find all UObject metadata messages.
for( const auto& Meta : MetadataMessages )
{
FName LongName = Meta.NameAndInfo.GetRawName();
const FString Desc = FStatNameAndInfo::GetShortNameFrom( LongName ).GetPlainNameString();
const bool bContainsUObject = Desc.Contains( TEXT( "//" ) );
if( bContainsUObject )
{
UObjectNames.Add( LongName );
}
}
const int64 CurrentFilePos = FileReader->Tell();
// Update profiler's metadata.
CreateThreadsMapping();
// Read frames offsets.
Stream.ReadFramesOffsets( *FileReader );
// Buffer used to store the compressed and decompressed data.
TArray<uint8> SrcArray;
TArray<uint8> DestArray;
const bool bHasCompressedData = Stream.Header.HasCompressedData();
check( bHasCompressedData );
TMap<int64, FStatPacketArray> CombinedHistory;
int64 TotalDataSize = 0;
int64 TotalStatMessagesNum = 0;
int64 MaximumPacketSize = 0;
int64 TotalPacketsNum = 0;
// Read all packets sequentially, force by the memory profiler which is now a part of the raw stats.
// !!CAUTION!! Frame number in the raw stats is pointless, because it is time based, not frame based.
// Background threads usually execute time consuming operations, so the frame number won't be valid.
// Needs to be combined by the thread and the time, not by the frame number.
{
// Display log information once per 5 seconds to avoid spamming.
double PreviousSeconds = FPlatformTime::Seconds();
const int64 FrameOffset0 = Stream.FramesInfo[0].FrameFileOffset;
FileReader->Seek( FrameOffset0 );
const int64 FileSize = FileReader->TotalSize();
while( FileReader->Tell() < FileSize )
{
// Read the compressed data.
FCompressedStatsData UncompressedData( SrcArray, DestArray );
*FileReader << UncompressedData;
if( UncompressedData.HasReachedEndOfCompressedData() )
{
break;
}
FMemoryReader MemoryReader( DestArray, true );
FStatPacket* StatPacket = new FStatPacket();
Stream.ReadStatPacket( MemoryReader, *StatPacket );
const int64 StatPacketFrameNum = StatPacket->Frame;
FStatPacketArray& Frame = CombinedHistory.FindOrAdd( StatPacketFrameNum );
// Check if we need to combine packets from the same thread.
FStatPacket** CombinedPacket = Frame.Packets.FindByPredicate( [&]( FStatPacket* Item ) -> bool
{
return Item->ThreadId == StatPacket->ThreadId;
} );
const int64 PacketSize = StatPacket->StatMessages.GetAllocatedSize();
TotalStatMessagesNum += StatPacket->StatMessages.Num();
if( CombinedPacket )
{
TotalDataSize -= (*CombinedPacket)->StatMessages.GetAllocatedSize();
(*CombinedPacket)->StatMessages += StatPacket->StatMessages;
TotalDataSize += (*CombinedPacket)->StatMessages.GetAllocatedSize();
delete StatPacket;
}
else
{
Frame.Packets.Add( StatPacket );
TotalDataSize += PacketSize;
}
const double CurrentSeconds = FPlatformTime::Seconds();
if( CurrentSeconds > PreviousSeconds + NumSecondsBetweenLogs )
{
const int32 PctPos = int32( 100.0*FileReader->Tell() / FileSize );
UE_LOG( LogStats, Log, TEXT( "%3i%% %10llu (%.1f MB) read messages, last read frame %4i" ), PctPos, TotalStatMessagesNum, TotalDataSize / 1024.0f / 1024.0f, StatPacketFrameNum );
PreviousSeconds = CurrentSeconds;
}
MaximumPacketSize = FMath::Max( MaximumPacketSize, PacketSize );
TotalPacketsNum++;
}
}
// Dump frame stats
for( const auto& It : CombinedHistory )
{
const int64 FrameNum = It.Key;
int64 FramePacketsSize = 0;
int64 FrameStatMessages = 0;
int64 FramePackets = It.Value.Packets.Num(); // Threads
for( const auto& It2 : It.Value.Packets )
{
FramePacketsSize += It2->StatMessages.GetAllocatedSize();
FrameStatMessages += It2->StatMessages.Num();
}
UE_LOG( LogStats, Warning, TEXT( "Frame: %10llu/%3lli Size: %.1f MB / %10lli" ),
FrameNum,
FramePackets,
FramePacketsSize / 1024.0f / 1024.0f,
FrameStatMessages );
}
UE_LOG( LogStats, Warning, TEXT( "TotalPacketSize: %.1f MB, Max: %1f MB" ),
TotalDataSize / 1024.0f / 1024.0f,
MaximumPacketSize / 1024.0f / 1024.0f );
TArray<int64> Frames;
CombinedHistory.GenerateKeyArray( Frames );
Frames.Sort();
const int64 MiddleFrame = Frames[Frames.Num() / 2];
ProcessMemoryOperations( CombinedHistory );
}
}
void FStatsMemoryDumpCommand::ProcessingUObjectAllocations( const TMap<uint64, FAllocationInfo>& AllocationMap )
{
// This code is not optimized.
FScopeLogTime SLT( TEXT( "ProcessingUObjectAllocations" ), nullptr, FScopeLogTime::ScopeLog_Seconds );
UE_LOG( LogStats, Warning, TEXT( "Processing UObject allocations" ) );
FDiagnosticTableViewer MemoryReport( *FDiagnosticTableViewer::GetUniqueTemporaryFilePath( TEXT( "MemoryReport-UObject" ) ) );
// Write a row of headings for the table's columns.
MemoryReport.AddColumn( TEXT( "Size (bytes)" ) );
MemoryReport.AddColumn( TEXT( "Size (MB)" ) );
MemoryReport.AddColumn( TEXT( "Count" ) );
MemoryReport.AddColumn( TEXT( "UObject class" ) );
MemoryReport.CycleRow();
TMap<FName, FSizeAndCount> UObjectAllocations;
// To minimize number of calls to expensive DecodeCallstack.
TMap<FName,FName> UObjectCallstackToClassMapping;
uint64 NumAllocations = 0;
uint64 TotalAllocatedMemory = 0;
for( const auto& It : AllocationMap )
{
const FAllocationInfo& Alloc = It.Value;
FName UObjectClass = UObjectCallstackToClassMapping.FindRef( Alloc.EncodedCallstack );
if( UObjectClass == NAME_None )
{
TArray<FString> DecodedCallstack;
DecodeCallstack( Alloc.EncodedCallstack, DecodedCallstack );
for( int32 Index = DecodedCallstack.Num() - 1; Index >= 0; --Index )
{
NAME_INDEX NameIndex = 0;
TTypeFromString<NAME_INDEX>::FromString( NameIndex, *DecodedCallstack[Index] );
const FName LongName = FName( NameIndex, NameIndex, 0 );
const bool bValid = UObjectNames.Contains( LongName );
if( bValid )
{
const FString ObjectName = FStatNameAndInfo::GetShortNameFrom( LongName ).GetPlainNameString();
UObjectClass = *ObjectName.Left( ObjectName.Find( TEXT( "//" ) ) );;
UObjectCallstackToClassMapping.Add( Alloc.EncodedCallstack, UObjectClass );
break;
}
}
}
if( UObjectClass != NAME_None )
{
FSizeAndCount& SizeAndCount = UObjectAllocations.FindOrAdd( UObjectClass );
SizeAndCount.Size += Alloc.Size;
SizeAndCount.Count += 1;
TotalAllocatedMemory += Alloc.Size;
NumAllocations++;
}
}
// Dump memory to the log.
UObjectAllocations.ValueSort( FSizeAndCountGreater() );
const float MaxPctDisplayed = 0.90f;
int32 CurrentIndex = 0;
uint64 DisplayedSoFar = 0;
UE_LOG( LogStats, Warning, TEXT( "Index, Size (Size MB), Count, UObject class" ) );
for( const auto& It : UObjectAllocations )
{
const FSizeAndCount& SizeAndCount = It.Value;
const FName& UObjectClass = It.Key;
UE_LOG( LogStats, Log, TEXT( "%2i, %llu (%.2f MB), %llu, %s" ),
CurrentIndex,
SizeAndCount.Size,
SizeAndCount.Size / 1024.0f / 1024.0f,
SizeAndCount.Count,
*UObjectClass.GetPlainNameString() );
// Dump stats
MemoryReport.AddColumn( TEXT( "%llu" ), SizeAndCount.Size );
MemoryReport.AddColumn( TEXT( "%.2f MB" ), SizeAndCount.Size / 1024.0f / 1024.0f );
MemoryReport.AddColumn( TEXT( "%llu" ), SizeAndCount.Count );
MemoryReport.AddColumn( *UObjectClass.GetPlainNameString() );
MemoryReport.CycleRow();
CurrentIndex++;
DisplayedSoFar += SizeAndCount.Size;
const float CurrentPct = (float)DisplayedSoFar / (float)TotalAllocatedMemory;
if( CurrentPct > MaxPctDisplayed )
{
break;
}
}
UE_LOG( LogStats, Warning, TEXT( "Allocated memory: %llu bytes (%.2f MB)" ), TotalAllocatedMemory, TotalAllocatedMemory / 1024.0f / 1024.0f );
// Add a total row.
MemoryReport.CycleRow();
MemoryReport.CycleRow();
MemoryReport.CycleRow();
MemoryReport.AddColumn( TEXT( "%llu" ), TotalAllocatedMemory );
MemoryReport.AddColumn( TEXT( "%.2f MB" ), TotalAllocatedMemory / 1024.0f / 1024.0f );
MemoryReport.AddColumn( TEXT( "%llu" ), NumAllocations );
MemoryReport.AddColumn( TEXT( "TOTAL" ) );
MemoryReport.CycleRow();
}
void FStatsMemoryDumpCommand::ProcessingScopedAllocations( const TMap<uint64, FAllocationInfo>& AllocationMap )
{
// This code is not optimized.
FScopeLogTime SLT( TEXT( "ProcessingScopedAllocations" ), nullptr, FScopeLogTime::ScopeLog_Seconds );
UE_LOG( LogStats, Warning, TEXT( "Processing scoped allocations" ) );
FDiagnosticTableViewer MemoryReport( *FDiagnosticTableViewer::GetUniqueTemporaryFilePath( TEXT( "MemoryReport-Scoped" ) ) );
// Write a row of headings for the table's columns.
MemoryReport.AddColumn( TEXT( "Size (bytes)" ) );
MemoryReport.AddColumn( TEXT( "Size (MB)" ) );
MemoryReport.AddColumn( TEXT( "Count" ) );
MemoryReport.AddColumn( TEXT( "Callstack" ) );
MemoryReport.CycleRow();
TMap<FName, FSizeAndCount> ScopedAllocations;
uint64 NumAllocations = 0;
uint64 TotalAllocatedMemory = 0;
for( const auto& It : AllocationMap )
{
const FAllocationInfo& Alloc = It.Value;
FSizeAndCount& SizeAndCount = ScopedAllocations.FindOrAdd( Alloc.EncodedCallstack );
SizeAndCount.Size += Alloc.Size;
SizeAndCount.Count += 1;
TotalAllocatedMemory += Alloc.Size;
NumAllocations++;
}
// Dump memory to the log.
ScopedAllocations.ValueSort( FSizeAndCountGreater() );
const float MaxPctDisplayed = 0.90f;
int32 CurrentIndex = 0;
uint64 DisplayedSoFar = 0;
UE_LOG( LogStats, Warning, TEXT( "Index, Size (Size MB), Count, Stat desc" ) );
for( const auto& It : ScopedAllocations )
{
const FSizeAndCount& SizeAndCount = It.Value;
const FName& EncodedCallstack = It.Key;
const FString AllocCallstack = GetCallstack( EncodedCallstack );
UE_LOG( LogStats, Log, TEXT( "%2i, %llu (%.2f MB), %llu, %s" ),
CurrentIndex,
SizeAndCount.Size,
SizeAndCount.Size / 1024.0f / 1024.0f,
SizeAndCount.Count,
*AllocCallstack );
// Dump stats
MemoryReport.AddColumn( TEXT( "%llu" ), SizeAndCount.Size );
MemoryReport.AddColumn( TEXT( "%.2f MB" ), SizeAndCount.Size / 1024.0f / 1024.0f );
MemoryReport.AddColumn( TEXT( "%llu" ), SizeAndCount.Count );
MemoryReport.AddColumn( *AllocCallstack );
MemoryReport.CycleRow();
CurrentIndex++;
DisplayedSoFar += SizeAndCount.Size;
const float CurrentPct = (float)DisplayedSoFar / (float)TotalAllocatedMemory;
if( CurrentPct > MaxPctDisplayed )
{
break;
}
}
UE_LOG( LogStats, Warning, TEXT( "Allocated memory: %llu bytes (%.2f MB)" ), TotalAllocatedMemory, TotalAllocatedMemory / 1024.0f / 1024.0f );
// Add a total row.
MemoryReport.CycleRow();
MemoryReport.CycleRow();
MemoryReport.CycleRow();
MemoryReport.AddColumn( TEXT( "%llu" ), TotalAllocatedMemory );
MemoryReport.AddColumn( TEXT( "%.2f MB" ), TotalAllocatedMemory / 1024.0f / 1024.0f );
MemoryReport.AddColumn( TEXT( "%llu" ), NumAllocations );
MemoryReport.AddColumn( TEXT( "TOTAL" ) );
MemoryReport.CycleRow();
}