void FSlateRHIResourceManager::CreateTextures( const TArray< const FSlateBrush* >& Resources )
{
DECLARE_SCOPE_CYCLE_COUNTER(TEXT("Loading Slate Textures"), STAT_Slate, STATGROUP_LoadTime);
TMap<FName,FNewTextureInfo> TextureInfoMap;
const uint32 Stride = GPixelFormats[PF_R8G8B8A8].BlockBytes;
for( int32 ResourceIndex = 0; ResourceIndex < Resources.Num(); ++ResourceIndex )
{
const FSlateBrush& Brush = *Resources[ResourceIndex];
const FName TextureName = Brush.GetResourceName();
if( TextureName != NAME_None && !Brush.HasUObject() && !Brush.IsDynamicallyLoaded() && !ResourceMap.Contains(TextureName) )
{
// Find the texture or add it if it doesnt exist (only load the texture once)
FNewTextureInfo& Info = TextureInfoMap.FindOrAdd( TextureName );
Info.bSrgb = (Brush.ImageType != ESlateBrushImageType::Linear);
// Only atlas the texture if none of the brushes that use it tile it and the image is srgb
Info.bShouldAtlas &= ( Brush.Tiling == ESlateBrushTileType::NoTile && Info.bSrgb && AtlasSize > 0 );
// Texture has been loaded if the texture data is valid
if( !Info.TextureData.IsValid() )
{
uint32 Width = 0;
uint32 Height = 0;
TArray<uint8> RawData;
bool bSucceeded = LoadTexture( Brush, Width, Height, RawData );
Info.TextureData = MakeShareable( new FSlateTextureData( Width, Height, Stride, RawData ) );
const bool bTooLargeForAtlas = (Width >= 256 || Height >= 256 || Width >= AtlasSize || Height >= AtlasSize );
Info.bShouldAtlas &= !bTooLargeForAtlas;
if( !bSucceeded || !ensureMsgf( Info.TextureData->GetRawBytes().Num() > 0, TEXT("Slate resource: (%s) contains no data"), *TextureName.ToString() ) )
{
TextureInfoMap.Remove( TextureName );
}
}
}
}
// Sort textures by size. The largest textures are atlased first which creates a more compact atlas
TextureInfoMap.ValueSort( FCompareFNewTextureInfoByTextureSize() );
for( TMap<FName,FNewTextureInfo>::TConstIterator It(TextureInfoMap); It; ++It )
{
const FNewTextureInfo& Info = It.Value();
FName TextureName = It.Key();
FString NameStr = TextureName.ToString();
checkSlow( TextureName != NAME_None );
FSlateShaderResourceProxy* NewTexture = GenerateTextureResource( Info );
ResourceMap.Add( TextureName, NewTexture );
}
}
void Unbind(UJavascriptDelegate* DelegateObject)
{
static FName NAME_Fire("Fire");
if (WeakObject.IsValid())
{
if (auto p = Cast<UMulticastDelegateProperty>(Property))
{
FScriptDelegate Delegate;
Delegate.BindUFunction(DelegateObject, NAME_Fire);
auto Target = p->GetPropertyValuePtr_InContainer(WeakObject.Get());
Target->Remove(Delegate);
}
else if (auto p = Cast<UDelegateProperty>(Property))
{
auto Target = p->GetPropertyValuePtr_InContainer(WeakObject.Get());
Target->Clear();
}
}
DelegateObject->JavascriptDelegate = nullptr;
DelegateObject->RemoveFromRoot();
DelegateObjects.Remove(DelegateObject);
if (!bAbandoned)
{
functions.Remove(DelegateObject->UniqueId);
}
}
bool FChunkManifestGenerator::LoadAssetRegistry(const FString& SandboxPath, const TSet<FName>* PackagesToKeep)
{
UE_LOG(LogChunkManifestGenerator, Display, TEXT("Loading asset registry."));
// Load generated registry for each platform
check(Platforms.Num() == 1);
for (auto Platform : Platforms)
{
/*FString PlatformSandboxPath = SandboxPath.Replace(TEXT("[Platform]"), *Platform->PlatformName());
FArchive* AssetRegistryReader = IFileManager::Get().CreateFileReader(*PlatformSandboxPath);*/
FString PlatformSandboxPath = SandboxPath.Replace(TEXT("[Platform]"), *Platform->PlatformName());
FArrayReader FileContents;
if (FFileHelper::LoadFileToArray(FileContents, *PlatformSandboxPath) == false)
{
continue;
}
FArchive* AssetRegistryReader = &FileContents;
TMap<FName, FAssetData*> SavedAssetRegistryData;
TArray<FDependsNode*> DependencyData;
if (AssetRegistryReader)
{
AssetRegistry.LoadRegistryData(*AssetRegistryReader, SavedAssetRegistryData, DependencyData);
}
for (auto& LoadedAssetData : AssetRegistryData)
{
if (PackagesToKeep &&
PackagesToKeep->Contains(LoadedAssetData.PackageName) == false)
{
continue;
}
FAssetData* FoundAssetData = SavedAssetRegistryData.FindRef(LoadedAssetData.ObjectPath);
if ( FoundAssetData )
{
LoadedAssetData.ChunkIDs.Append(FoundAssetData->ChunkIDs);
SavedAssetRegistryData.Remove(LoadedAssetData.ObjectPath);
delete FoundAssetData;
}
}
for (const auto& SavedAsset : SavedAssetRegistryData)
{
if (PackagesToKeep && PackagesToKeep->Contains(SavedAsset.Value->PackageName))
{
AssetRegistryData.Add(*SavedAsset.Value);
}
delete SavedAsset.Value;
}
SavedAssetRegistryData.Empty();
}
return true;
}
virtual void DisassociateSuppress(FLogCategoryBase* Destination)
{
FName* Name = Associations.Find(Destination);
if (Name)
{
verify(ReverseAssociations.Remove(*Name, Destination)==1);
verify(Associations.Remove(Destination) == 1);
}
}
void FSlateD3DTextureManager::CreateTextures( const TArray< const FSlateBrush* >& Resources )
{
TMap<FName,FNewTextureInfo> TextureInfoMap;
for( int32 ResourceIndex = 0; ResourceIndex < Resources.Num(); ++ResourceIndex )
{
const FSlateBrush& Brush = *Resources[ResourceIndex];
const FName TextureName = Brush.GetResourceName();
if( TextureName != NAME_None && !ResourceMap.Contains(TextureName) )
{
// Find the texture or add it if it doesn't exist (only load the texture once)
FNewTextureInfo& Info = TextureInfoMap.FindOrAdd( TextureName );
Info.bSrgb = (Brush.ImageType != ESlateBrushImageType::Linear);
// Only atlas the texture if none of the brushes that use it tile it
Info.bShouldAtlas &= (Brush.Tiling == ESlateBrushTileType::NoTile && Info.bSrgb );
if( !Info.TextureData.IsValid())
{
uint32 Width = 0;
uint32 Height = 0;
TArray<uint8> RawData;
bool bSucceeded = LoadTexture( Brush, Width, Height, RawData );
const uint32 Stride = 4; // RGBA
Info.TextureData = MakeShareable( new FSlateTextureData( Width, Height, Stride, RawData ) );
const bool bTooLargeForAtlas = (Width >= 256 || Height >= 256);
Info.bShouldAtlas &= !bTooLargeForAtlas;
if( !bSucceeded )
{
TextureInfoMap.Remove( TextureName );
}
}
}
}
TextureInfoMap.ValueSort( FCompareFNewTextureInfoByTextureSize() );
for( TMap<FName,FNewTextureInfo>::TConstIterator It(TextureInfoMap); It; ++It )
{
const FNewTextureInfo& Info = It.Value();
FName TextureName = It.Key();
FString NameStr = TextureName.ToString();
FSlateShaderResourceProxy* NewTexture = GenerateTextureResource( Info );
ResourceMap.Add( TextureName, NewTexture );
}
}
void FGAGameEffectContainer::RemoveEffect(FGAGameEffectHandle& HandleIn)
{
EGAEffectAggregation aggregatiopn = HandleIn.GetEffectRef().GameEffect->EffectAggregation;
UObject* Instigator = HandleIn.GetContextRef().Instigator.Get();
TSharedPtr<FGAGameEffect> effect = ActiveEffects.FindAndRemoveChecked(HandleIn);
if (effect.IsValid())
{
switch (aggregatiopn)
{
case EGAEffectAggregation::AggregateByInstigator:
{
TMap<FGAGameEffectHandle, TSharedPtr<FGAGameEffect>>* effects = InstigatorEffects.Find(Instigator);
TMap<FName, TSet<FGAGameEffectHandle>>* EffectByClass = InstigatorEffectHandles.Find(Instigator);
if (EffectByClass)
{
//Probabaly need another path for removing just single effect from stack.
EffectByClass->Remove(HandleIn.GetEffectSpec()->GetFName());
}
if (effects)
{
effects->FindAndRemoveChecked(HandleIn);
if (effects->Num() == 0)
{
InstigatorEffects.Remove(Instigator);
}
}
break;
}
case EGAEffectAggregation::AggregateByTarget:
{
//TargetEffects.FindAndRemoveChecked(HandleIn);
TSet<FGAGameEffectHandle>* Handles = TargetEffectByType.Find(HandleIn.GetEffectSpec()->GetFName());
//check aggregation type to know which effect to remove exactly ?
TargetEffectByType.Remove(HandleIn.GetEffectSpec()->GetFName());
break;
}
}
for (FGAGameEffectModifier& Modifier : effect->GameEffect->Modifiers)
{
if (Modifier.Attribute.IsValid())
{
FGAAttributeBase* Attribute = OwningComp->GetAttribute(Modifier.Attribute);
if (Attribute)
{
Attribute->RemoveBonus(HandleIn);
}
}
}
UE_LOG(GameAttributesEffects, Log, TEXT("FGAGameEffectContainer:: Removing Effect"))
effect.Reset();
}
}
void FGameplayDebugger::WorldDestroyed(UWorld* InWorld)
{
#if !(UE_BUILD_SHIPPING || UE_BUILD_TEST)
bool bIsServer = InWorld && InWorld->GetNetMode() < ENetMode::NM_Client; // (Only work on server)
if (!bIsServer)
{
return;
}
// remove global replicator from level
AllReplicatorsPerWorlds.Remove(InWorld);
#endif //!(UE_BUILD_SHIPPING || UE_BUILD_TEST)
}
void UActorComponent::PostEditUndo()
{
// Objects marked pending kill don't call PostEditChange() from UObject::PostEditUndo(),
// so they can leave an EditReregisterContexts entry around if they are deleted by an undo action.
if( IsPendingKill() )
{
// The reregister context won't bother attaching components that are 'pending kill'.
FComponentReregisterContext* ReregisterContext = EditReregisterContexts.FindRef(this);
if(ReregisterContext)
{
delete ReregisterContext;
EditReregisterContexts.Remove(this);
}
}
else
{
Owner = GetTypedOuter<AActor>();
bCanUseCachedOwner = true;
// Let the component be properly registered, after it was restored.
if (Owner)
{
Owner->AddOwnedComponent(this);
}
TArray<UObject*> Children;
GetObjectsWithOuter(this, Children);
for (UObject* Child : Children)
{
if (UActorComponent* ChildComponent = Cast<UActorComponent>(Child))
{
if (ChildComponent->Owner)
{
ChildComponent->Owner->RemoveOwnedComponent(ChildComponent);
}
ChildComponent->Owner = Owner;
if (Owner)
{
Owner->AddOwnedComponent(ChildComponent);
}
}
}
if (GetWorld())
{
GetWorld()->UpdateActorComponentEndOfFrameUpdateState(this);
}
}
Super::PostEditUndo();
}
void FGAGameEffectContainer::RemoveEffect(FGAGameEffectHandle& HandleIn)
{
EGAEffectAggregation aggregatiopn = HandleIn.GetEffectRef().GameEffect->EffectAggregation;
UObject* Instigator = HandleIn.GetContextRef().Instigator.Get();
TSharedPtr<FGAGameEffect> effect = ActiveEffects.FindAndRemoveChecked(HandleIn);
if (effect.IsValid())
{
switch (aggregatiopn)
{
case EGAEffectAggregation::AggregateByInstigator:
{
TMap<FGAGameEffectHandle, TSharedPtr<FGAGameEffect>>* effects = InstigatorEffects.Find(Instigator);
TMap<UClass*, FGAGameEffectHandle>* EffectByClass = InstigatorEffectHandles.Find(Instigator);
if (EffectByClass)
{
EffectByClass->Remove(HandleIn.GetEffectSpec()->StaticClass());
}
if (effects)
{
effects->FindAndRemoveChecked(HandleIn);
if (effects->Num() == 0)
{
InstigatorEffects.Remove(Instigator);
}
}
break;
}
case EGAEffectAggregation::AggregateByTarget:
{
TargetEffects.FindAndRemoveChecked(HandleIn);
break;
}
}
for (FGAGameEffectModifier& Modifier : effect->GameEffect->Modifiers)
{
if (Modifier.Attribute.IsValid())
{
FGAAttributeBase* Attribute = OwningComp->GetAttribute(Modifier.Attribute);
if (Attribute)
{
Attribute->RemoveBonus(HandleIn);
}
}
}
UE_LOG(GameAttributesEffects, Log, TEXT("FGAGameEffectContainer:: Removing Effect"))
effect.Reset();
}
}
void WriteMergeObjects( TFile *target ) {
cout << "Writing the merged data." << endl;
TIterator *nextobj = MergeObjects.MakeIterator();
TObjString *pathname_obj;
while( (pathname_obj = (TObjString *)nextobj->Next()) ) {
TString path,name;
SplitPathName(pathname_obj->String(),&path,&name);
TObject *obj = MergeObjects.GetValue(pathname_obj);
target->cd(path);
obj->Write( name );
delete obj;
}
MergeObjects.Clear();
target->Write();
// Temporarily let multiple root files remain if > 2GB
// Prevent Target_1.root Target_2.root, ... from happening.
// long long max_tree_size = 200000000000LL; // 200 GB
// if(TTree::GetMaxTreeSize() < max_tree_size ) {
// TTree::SetMaxTreeSize(max_tree_size);
// }
nextobj = MergeChains.MakeIterator();
TObjString *pathname_obj;
while( (pathname_obj = (TObjString *)nextobj->Next()) ) {
TString path,name;
SplitPathName(pathname_obj->String(),&path,&name);
TChain *ch = (TChain *)MergeChains.GetValue(pathname_obj);
target->cd(path);
ch->Merge(target,0,"KEEP");
delete ch;
// in case of multiple objects with same pathname, must remove
// this one from the list so we don't get the same (deleted)
// one next time we look up the same name
MergeChains.Remove(pathname_obj);
}
MergeChains.Clear();
InitializedMergeObjects = false;
}
void UActorComponent::PostEditUndo()
{
// Objects marked pending kill don't call PostEditChange() from UObject::PostEditUndo(),
// so they can leave an EditReregisterContexts entry around if they are deleted by an undo action.
if( IsPendingKill() )
{
// The reregister context won't bother attaching components that are 'pending kill'.
FComponentReregisterContext* ReregisterContext = EditReregisterContexts.FindRef(this);
if(ReregisterContext)
{
delete ReregisterContext;
EditReregisterContexts.Remove(this);
}
}
Super::PostEditUndo();
}
static void DeleteFromEnumerateUserFilesComplete(bool bWasSuccessful, const FUniqueNetId& UserId)
{
IOnlineSubsystem* OnlineSub = IOnlineSubsystem::Get();
check(OnlineSub);
IOnlineUserCloudPtr UserCloud = OnlineSub->GetUserCloudInterface();
UserCloud->ClearOnEnumerateUserFilesCompleteDelegate_Handle(GOnEnumerateUserFilesCompleteDelegateHandle);
GPerCloudDeleteFromEnumerateUserFilesCompleteDelegateHandles.Remove(UserCloud.Get());
if (bWasSuccessful)
{
TArray<FCloudFileHeader> UserFiles;
UserCloud->GetUserFileList(UserId, UserFiles);
for (int32 Idx=0; Idx < UserFiles.Num(); Idx++)
{
UserCloud->DeleteUserFile(UserId, UserFiles[Idx].FileName, true, true);
}
}
}
void UActorComponent::PostEditChangeProperty(FPropertyChangedEvent& PropertyChangedEvent)
{
FComponentReregisterContext* ReregisterContext = EditReregisterContexts.FindRef(this);
if(ReregisterContext)
{
delete ReregisterContext;
EditReregisterContexts.Remove(this);
}
// The component or its outer could be pending kill when calling PostEditChange when applying a transaction.
// Don't do do a full recreate in this situation, and instead simply detach.
if( IsPendingKill() )
{
// @todo UE4 james should this call UnregsiterComponent instead to remove itself from the RegisteteredComponents array on the owner?
ExecuteUnregisterEvents();
World = NULL;
}
Super::PostEditChangeProperty(PropertyChangedEvent);
}
/**
* Virtual destructor, free'ing allocated memory.
*/
FUntypedBulkData::~FUntypedBulkData()
{
check( LockStatus == LOCKSTATUS_Unlocked );
// Free memory.
if( bShouldFreeOnEmpty )
{
FMemory::Free( BulkData );
}
BulkData = NULL;
#if WITH_EDITOR
// Detach from archive.
if( AttachedAr )
{
AttachedAr->DetachBulkData( this, false );
check( AttachedAr == NULL );
}
#endif // WITH_EDITOR
#if TRACK_BULKDATA_USE
BulkDataToObjectMap.Remove( this );
#endif
}
void UActorComponent::ConsolidatedPostEditChange(const FPropertyChangedEvent& PropertyChangedEvent)
{
FComponentReregisterContext* ReregisterContext = EditReregisterContexts.FindRef(this);
if(ReregisterContext)
{
delete ReregisterContext;
EditReregisterContexts.Remove(this);
AActor* MyOwner = GetOwner();
if ( MyOwner && !MyOwner->IsTemplate() && PropertyChangedEvent.ChangeType != EPropertyChangeType::Interactive )
{
MyOwner->RerunConstructionScripts();
}
}
// The component or its outer could be pending kill when calling PostEditChange when applying a transaction.
// Don't do do a full recreate in this situation, and instead simply detach.
if( IsPendingKill() )
{
// @todo UE4 james should this call UnregisterComponent instead to remove itself from the RegisteredComponents array on the owner?
ExecuteUnregisterEvents();
World = NULL;
}
}
void UDeviceProfileManager::InitializeCVarsForActiveDeviceProfile()
{
// Find the device profile selector module used in this instance
FString DeviceProfileSelectionModule;
GConfig->GetString( TEXT("DeviceProfileManager"), TEXT("DeviceProfileSelectionModule"), DeviceProfileSelectionModule, GEngineIni );
FString SelectedPlatformDeviceProfileName = GetActiveProfileName();
UE_LOG(LogInit, Log, TEXT("Applying CVar settings loaded from the selected device profile: [%s]"), *SelectedPlatformDeviceProfileName);
// Load the device profile config
FConfigCacheIni::LoadGlobalIniFile(DeviceProfileFileName, TEXT("DeviceProfiles"));
TArray< FString > AvailableProfiles;
GConfig->GetSectionNames( DeviceProfileFileName, AvailableProfiles );
// Look up the ini for this tree as we are far too early to use the UObject system
AvailableProfiles.Remove( TEXT( "DeviceProfiles" ) );
// Next we need to create a hierarchy of CVars from the Selected Device Profile, to it's eldest parent
TMap<FString, FString> CVarsAlreadySetList;
// For each device profile, starting with the selected and working our way up the BaseProfileName tree,
// Find all CVars and set them
FString BaseDeviceProfileName = SelectedPlatformDeviceProfileName;
bool bReachedEndOfTree = BaseDeviceProfileName.IsEmpty();
while( bReachedEndOfTree == false )
{
FString CurrentSectionName = FString::Printf( TEXT("%s %s"), *BaseDeviceProfileName, *UDeviceProfile::StaticClass()->GetName() );
// Check the profile was available.
bool bProfileExists = AvailableProfiles.Contains( CurrentSectionName );
if( bProfileExists )
{
TArray< FString > CurrentProfilesCVars;
GConfig->GetArray( *CurrentSectionName, TEXT("CVars"), CurrentProfilesCVars, DeviceProfileFileName );
// Iterate over the profile and make sure we do not have duplicate CVars
{
TMap< FString, FString > ValidCVars;
for( TArray< FString >::TConstIterator CVarIt(CurrentProfilesCVars); CVarIt; ++CVarIt )
{
FString CVarKey, CVarValue;
if( (*CVarIt).Split( TEXT("="), &CVarKey, &CVarValue ) )
{
if( ValidCVars.Find( CVarKey ) )
{
ValidCVars.Remove( CVarKey );
}
ValidCVars.Add( CVarKey, CVarValue );
}
}
// Empty the current list, and replace with the processed CVars. This removes duplicates
CurrentProfilesCVars.Empty();
for( TMap< FString, FString >::TConstIterator ProcessedCVarIt(ValidCVars); ProcessedCVarIt; ++ProcessedCVarIt )
{
CurrentProfilesCVars.Add( FString::Printf( TEXT("%s=%s"), *ProcessedCVarIt.Key(), *ProcessedCVarIt.Value() ) );
}
}
// Iterate over this profiles cvars and set them if they haven't been already.
for( TArray< FString >::TConstIterator CVarIt(CurrentProfilesCVars); CVarIt; ++CVarIt )
{
FString CVarKey, CVarValue;
if( (*CVarIt).Split( TEXT("="), &CVarKey, &CVarValue ) )
{
if( !CVarsAlreadySetList.Find( CVarKey ) )
{
IConsoleVariable* CVar = IConsoleManager::Get().FindConsoleVariable(*CVarKey);
if( CVar )
{
UE_LOG(LogInit, Log, TEXT("Setting Device Profile CVar: [[%s:%s]]"), *CVarKey, *CVarValue);
CVar->Set( *CVarValue, ECVF_SetByDeviceProfile);
CVarsAlreadySetList.Add( CVarKey, CVarValue );
}
else
{
UE_LOG(LogInit, Warning, TEXT("Failed to find a registered CVar that matches the key: [%s]"), *CVarKey);
}
}
}
}
// Get the next device profile name, to look for CVars in, along the tree
FString NextBaseDeviceProfileName;
if( GConfig->GetString( *CurrentSectionName, TEXT("BaseProfileName"), NextBaseDeviceProfileName, DeviceProfileFileName ) )
{
BaseDeviceProfileName = NextBaseDeviceProfileName;
}
else
{
BaseDeviceProfileName.Empty();
}
}
// Check if we have inevitably reached the end of the device profile tree.
bReachedEndOfTree = !bProfileExists || BaseDeviceProfileName.IsEmpty();
}
}
void UPhyaCollisionHandler::HandlePhysicsCollisions_AssumesLocked(TArray<FCollisionNotifyInfo>& PendingCollisionNotifies)
{
if(GetWorld()->HasBegunPlay())
{
TMap< FPhyaBodyInstancePair, TSharedPtr<FPhyaPairInfo> > ExpiredPairHash = PairHash;
for(int32 InfoIdx=0; InfoIdx<PendingCollisionNotifies.Num(); InfoIdx++)
{
const FCollisionNotifyInfo& Info = PendingCollisionNotifies[InfoIdx];
FPhyaBodyInstancePair Pair(Info.Info0.GetBodyInstance(), Info.Info1.GetBodyInstance());
// Find pair in hash
TSharedPtr<FPhyaPairInfo> PairInfo = PairHash.FindRef(Pair);
// Existing pair
if(PairInfo.IsValid())
{
UE_LOG(LogTemp, Log, TEXT("EXISTING"));
ExpiredPairHash.Remove(Pair); // Not expired
}
// New pair
else
{
UE_LOG(LogTemp, Log, TEXT("NEW"));
PairInfo = MakeShareable( new FPhyaPairInfo );
PairHash.Add(Pair, PairInfo);
paImpact* Impact = paImpact::newImpact();
if(Impact != NULL)
{
Impact->setBody1(Bodies[0]);
paImpactDynamicData ImpactData;
ImpactData.relTangentSpeedAtImpact = 0; // No skid.
ImpactData.impactImpulse = 1.0;
Impact->setDynamicData(&ImpactData);
}
}
}
// Expire pairs from PairHash still in ExpiredPairHash
for( auto It = ExpiredPairHash.CreateConstIterator(); It; ++It )
{
UE_LOG(LogTemp, Log, TEXT("EXPIRE"));
FPhyaBodyInstancePair Pair = It.Key();
PairHash.Remove(Pair);
}
/*
float WorldTime = GetWorld()->GetTimeSeconds();
float TimeSinceLastTestImpact = WorldTime - LastTestImpactTime;
if(TimeSinceLastTestImpact > 1.f)
{
TestImpact();
LastTestImpactTime = WorldTime;
}
*/
}
}
void FRawProfilerSession::PrepareLoading()
{
SCOPE_LOG_TIME_FUNC();
const FString Filepath = DataFilepath + FStatConstants::StatsFileRawExtension;
const int64 Size = IFileManager::Get().FileSize( *Filepath );
if( Size < 4 )
{
UE_LOG( LogStats, Error, TEXT( "Could not open: %s" ), *Filepath );
return;
}
TAutoPtr<FArchive> FileReader( IFileManager::Get().CreateFileReader( *Filepath ) );
if( !FileReader )
{
UE_LOG( LogStats, Error, TEXT( "Could not open: %s" ), *Filepath );
return;
}
if( !Stream.ReadHeader( *FileReader ) )
{
UE_LOG( LogStats, Error, TEXT( "Could not open, bad magic: %s" ), *Filepath );
return;
}
const bool bIsFinalized = Stream.Header.IsFinalized();
check( bIsFinalized );
check( Stream.Header.Version == EStatMagicWithHeader::VERSION_5 );
StatsThreadStats.MarkAsLoaded();
TArray<FStatMessage> Messages;
if( Stream.Header.bRawStatsFile )
{
// Read metadata.
TArray<FStatMessage> MetadataMessages;
Stream.ReadFNamesAndMetadataMessages( *FileReader, MetadataMessages );
StatsThreadStats.ProcessMetaDataOnly( MetadataMessages );
const FName F00245 = FName(245, 245, 0);
const FName F11602 = FName(11602, 11602, 0);
const FName F06394 = FName(6394, 6394, 0);
const int64 CurrentFilePos = FileReader->Tell();
// Update profiler's metadata.
StatMetaData->UpdateFromStatsState( StatsThreadStats );
const uint32 GameThreadID = GetMetaData()->GetGameThreadID();
// Read frames offsets.
Stream.ReadFramesOffsets( *FileReader );
// Buffer used to store the compressed and decompressed data.
TArray<uint8> SrcArray;
TArray<uint8> DestArray;
const bool bHasCompressedData = Stream.Header.HasCompressedData();
check(bHasCompressedData);
TMap<int64, FStatPacketArray> CombinedHistory;
int64 TotalPacketSize = 0;
int64 MaximumPacketSize = 0;
// Read all packets sequentially, force by the memory profiler which is now a part of the raw stats.
// !!CAUTION!! Frame number in the raw stats is pointless, because it is time based, not frame based.
// Background threads usually execute time consuming operations, so the frame number won't be valid.
// Needs to be combined by the thread and the time, not by the frame number.
{
int64 FrameOffset0 = Stream.FramesInfo[0].FrameFileOffset;
FileReader->Seek( FrameOffset0 );
const int64 FileSize = FileReader->TotalSize();
while( FileReader->Tell() < FileSize )
{
// Read the compressed data.
FCompressedStatsData UncompressedData( SrcArray, DestArray );
*FileReader << UncompressedData;
if( UncompressedData.HasReachedEndOfCompressedData() )
{
break;
}
FMemoryReader MemoryReader( DestArray, true );
FStatPacket* StatPacket = new FStatPacket();
Stream.ReadStatPacket( MemoryReader, *StatPacket );
const int64 FrameNum = StatPacket->Frame;
FStatPacketArray& Frame = CombinedHistory.FindOrAdd(FrameNum);
// Check if we need to combine packets from the same thread.
FStatPacket** CombinedPacket = Frame.Packets.FindByPredicate([&](FStatPacket* Item) -> bool
{
return Item->ThreadId == StatPacket->ThreadId;
});
if( CombinedPacket )
{
(*CombinedPacket)->StatMessages += StatPacket->StatMessages;
}
else
{
Frame.Packets.Add(StatPacket);
}
const int64 CurrentPos = FileReader->Tell();
const int32 PctPos = int32(100.0f*CurrentPos/FileSize);
UE_LOG( LogStats, Log, TEXT( "%3i Processing FStatPacket: Frame %5i for thread %5i with %6i messages (%.1f MB)" ),
PctPos,
StatPacket->Frame,
StatPacket->ThreadId,
StatPacket->StatMessages.Num(),
StatPacket->StatMessages.GetAllocatedSize()/1024.0f/1024.0f );
const int64 PacketSize = StatPacket->StatMessages.GetAllocatedSize();
TotalPacketSize += PacketSize;
MaximumPacketSize = FMath::Max( MaximumPacketSize, PacketSize );
}
}
UE_LOG( LogStats, Log, TEXT( "TotalPacketSize: %.1f MB, Max: %1f MB" ),
TotalPacketSize/1024.0f/1024.0f,
MaximumPacketSize/1024.0f/1024.0f );
TArray<int64> Frames;
CombinedHistory.GenerateKeyArray(Frames);
Frames.Sort();
const int64 MiddleFrame = Frames[Frames.Num()/2];
// Remove all frames without the game thread messages.
for (int32 FrameIndex = 0; FrameIndex < Frames.Num(); ++FrameIndex)
{
const int64 TargetFrame = Frames[FrameIndex];
const FStatPacketArray& Frame = CombinedHistory.FindChecked( TargetFrame );
const double GameThreadTimeMS = GetMetaData()->ConvertCyclesToMS( GetFastThreadFrameTimeInternal( Frame, EThreadType::Game ) );
if (GameThreadTimeMS == 0.0f)
{
CombinedHistory.Remove( TargetFrame );
Frames.RemoveAt( FrameIndex );
FrameIndex--;
}
}
StatMetaData->SecondsPerCycle = GetSecondsPerCycle( CombinedHistory.FindChecked(MiddleFrame) );
check( StatMetaData->GetSecondsPerCycle() > 0.0 );
//const int32 FirstGameThreadFrame = FindFirstFrameWithGameThread( CombinedHistory, Frames );
// Prepare profiler frame.
{
SCOPE_LOG_TIME( TEXT( "Preparing profiler frames" ), nullptr );
// Prepare profiler frames.
double ElapsedTimeMS = 0;
for( int32 FrameIndex = 0; FrameIndex < Frames.Num(); ++FrameIndex )
{
const int64 TargetFrame = Frames[FrameIndex];
const FStatPacketArray& Frame = CombinedHistory.FindChecked(TargetFrame);
const double GameThreadTimeMS = GetMetaData()->ConvertCyclesToMS( GetFastThreadFrameTimeInternal(Frame,EThreadType::Game) );
if( GameThreadTimeMS == 0.0f )
{
continue;
}
const double RenderThreadTimeMS = GetMetaData()->ConvertCyclesToMS( GetFastThreadFrameTimeInternal(Frame,EThreadType::Renderer) );
// Update mini-view, convert from cycles to ms.
TMap<uint32, float> ThreadTimesMS;
ThreadTimesMS.Add( GameThreadID, GameThreadTimeMS );
ThreadTimesMS.Add( GetMetaData()->GetRenderThreadID()[0], RenderThreadTimeMS );
// Pass the reference to the stats' metadata.
OnAddThreadTime.ExecuteIfBound( FrameIndex, ThreadTimesMS, StatMetaData );
// Create a new profiler frame and add it to the stream.
ElapsedTimeMS += GameThreadTimeMS;
FProfilerFrame* ProfilerFrame = new FProfilerFrame( TargetFrame, GameThreadTimeMS, ElapsedTimeMS );
ProfilerFrame->ThreadTimesMS = ThreadTimesMS;
ProfilerStream.AddProfilerFrame( TargetFrame, ProfilerFrame );
}
}
// Process the raw stats data.
{
SCOPE_LOG_TIME( TEXT( "Processing the raw stats" ), nullptr );
double CycleCounterAdjustmentMS = 0.0f;
// Read the raw stats messages.
for( int32 FrameIndex = 0; FrameIndex < Frames.Num()-1; ++FrameIndex )
{
const int64 TargetFrame = Frames[FrameIndex];
const FStatPacketArray& Frame = CombinedHistory.FindChecked(TargetFrame);
FProfilerFrame* ProfilerFrame = ProfilerStream.GetProfilerFrame( FrameIndex );
UE_CLOG( FrameIndex % 8 == 0, LogStats, Log, TEXT( "Processing raw stats frame: %4i/%4i" ), FrameIndex, Frames.Num() );
ProcessStatPacketArray( Frame, *ProfilerFrame, FrameIndex ); // or ProfilerFrame->TargetFrame
// Find the first cycle counter for the game thread.
if( CycleCounterAdjustmentMS == 0.0f )
{
CycleCounterAdjustmentMS = ProfilerFrame->Root->CycleCounterStartTimeMS;
}
// Update thread time and mark profiler frame as valid and ready for use.
ProfilerFrame->MarkAsValid();
}
// Adjust all profiler frames.
ProfilerStream.AdjustCycleCounters( CycleCounterAdjustmentMS );
}
}
const int64 AllocatedSize = ProfilerStream.GetAllocatedSize();
// We have the whole metadata and basic information about the raw stats file, start ticking the profiler session.
//OnTickHandle = FTicker::GetCoreTicker().AddTicker( OnTick, 0.25f );
#if 0
if( SessionType == EProfilerSessionTypes::OfflineRaw )
{
// Broadcast that a capture file has been fully processed.
OnCaptureFileProcessed.ExecuteIfBound( GetInstanceID() );
}
#endif // 0
}
void FStatsMemoryDumpCommand::ProcessMemoryOperations( const TMap<int64, FStatPacketArray>& CombinedHistory )
{
// This is only example code, no fully implemented, may sometimes crash.
// This code is not optimized.
double PreviousSeconds = FPlatformTime::Seconds();
uint64 NumMemoryOperations = 0;
// Generate frames
TArray<int64> Frames;
CombinedHistory.GenerateKeyArray( Frames );
Frames.Sort();
// Raw stats callstack for this stat packet array.
TMap<FName, FStackState> StackStates;
// All allocation ordered by the sequence tag.
// There is an assumption that the sequence tag will not turn-around.
//TMap<uint32, FAllocationInfo> SequenceAllocationMap;
TArray<FAllocationInfo> SequenceAllocationArray;
// Pass 1.
// Read all stats messages, parse all memory operations and decode callstacks.
const int64 FirstFrame = 0;
PreviousSeconds -= NumSecondsBetweenLogs;
for( int32 FrameIndex = 0; FrameIndex < Frames.Num(); ++FrameIndex )
{
{
const double CurrentSeconds = FPlatformTime::Seconds();
if( CurrentSeconds > PreviousSeconds + NumSecondsBetweenLogs )
{
UE_LOG( LogStats, Warning, TEXT( "Processing frame %i/%i" ), FrameIndex+1, Frames.Num() );
PreviousSeconds = CurrentSeconds;
}
}
const int64 TargetFrame = Frames[FrameIndex];
const int64 Diff = TargetFrame - FirstFrame;
const FStatPacketArray& Frame = CombinedHistory.FindChecked( TargetFrame );
bool bAtLeastOnePacket = false;
for( int32 PacketIndex = 0; PacketIndex < Frame.Packets.Num(); PacketIndex++ )
{
{
const double CurrentSeconds = FPlatformTime::Seconds();
if( CurrentSeconds > PreviousSeconds + NumSecondsBetweenLogs )
{
UE_LOG( LogStats, Log, TEXT( "Processing packet %i/%i" ), PacketIndex, Frame.Packets.Num() );
PreviousSeconds = CurrentSeconds;
bAtLeastOnePacket = true;
}
}
const FStatPacket& StatPacket = *Frame.Packets[PacketIndex];
const FName& ThreadFName = StatsThreadStats.Threads.FindChecked( StatPacket.ThreadId );
const uint32 NewThreadID = ThreadIDtoStatID.FindChecked( StatPacket.ThreadId );
FStackState* StackState = StackStates.Find( ThreadFName );
if( !StackState )
{
StackState = &StackStates.Add( ThreadFName );
StackState->Stack.Add( ThreadFName );
StackState->Current = ThreadFName;
}
const FStatMessagesArray& Data = StatPacket.StatMessages;
int32 LastPct = 0;
const int32 NumDataElements = Data.Num();
const int32 OnerPercent = FMath::Max( NumDataElements / 100, 1024 );
bool bAtLeastOneMessage = false;
for( int32 Index = 0; Index < NumDataElements; Index++ )
{
if( Index % OnerPercent )
{
const double CurrentSeconds = FPlatformTime::Seconds();
if( CurrentSeconds > PreviousSeconds + NumSecondsBetweenLogs )
{
const int32 CurrentPct = int32( 100.0*(Index + 1) / NumDataElements );
UE_LOG( LogStats, Log, TEXT( "Processing %3i%% (%i/%i) stat messages" ), CurrentPct, Index, NumDataElements );
PreviousSeconds = CurrentSeconds;
bAtLeastOneMessage = true;
}
}
const FStatMessage& Item = Data[Index];
const EStatOperation::Type Op = Item.NameAndInfo.GetField<EStatOperation>();
const FName RawName = Item.NameAndInfo.GetRawName();
if( Op == EStatOperation::CycleScopeStart || Op == EStatOperation::CycleScopeEnd || Op == EStatOperation::Memory )
{
if( Op == EStatOperation::CycleScopeStart )
{
StackState->Stack.Add( RawName );
StackState->Current = RawName;
}
else if( Op == EStatOperation::Memory )
{
// Experimental code used only to test the implementation.
// First memory operation is Alloc or Free
const uint64 EncodedPtr = Item.GetValue_Ptr();
const bool bIsAlloc = (EncodedPtr & (uint64)EMemoryOperation::Alloc) != 0;
const bool bIsFree = (EncodedPtr & (uint64)EMemoryOperation::Free) != 0;
const uint64 Ptr = EncodedPtr & ~(uint64)EMemoryOperation::Mask;
if( bIsAlloc )
{
NumMemoryOperations++;
// @see FStatsMallocProfilerProxy::TrackAlloc
// After alloc ptr message there is always alloc size message and the sequence tag.
Index++;
const FStatMessage& AllocSizeMessage = Data[Index];
const int64 AllocSize = AllocSizeMessage.GetValue_int64();
// Read operation sequence tag.
Index++;
const FStatMessage& SequenceTagMessage = Data[Index];
const uint32 SequenceTag = SequenceTagMessage.GetValue_int64();
// Create a callstack.
TArray<FName> StatsBasedCallstack;
for( const auto& StackName : StackState->Stack )
{
StatsBasedCallstack.Add( StackName );
}
// Add a new allocation.
SequenceAllocationArray.Add(
FAllocationInfo(
Ptr,
AllocSize,
StatsBasedCallstack,
SequenceTag,
EMemoryOperation::Alloc,
StackState->bIsBrokenCallstack
) );
}
else if( bIsFree )
{
NumMemoryOperations++;
// Read operation sequence tag.
Index++;
const FStatMessage& SequenceTagMessage = Data[Index];
const uint32 SequenceTag = SequenceTagMessage.GetValue_int64();
// Create a callstack.
/*
TArray<FName> StatsBasedCallstack;
for( const auto& RawName : StackState->Stack )
{
StatsBasedCallstack.Add( RawName );
}
*/
// Add a new free.
SequenceAllocationArray.Add(
FAllocationInfo(
Ptr,
0,
TArray<FName>()/*StatsBasedCallstack*/,
SequenceTag,
EMemoryOperation::Free,
StackState->bIsBrokenCallstack
) );
}
else
{
UE_LOG( LogStats, Warning, TEXT( "Pointer from a memory operation is invalid" ) );
}
}
else if( Op == EStatOperation::CycleScopeEnd )
{
if( StackState->Stack.Num() > 1 )
{
const FName ScopeStart = StackState->Stack.Pop();
const FName ScopeEnd = Item.NameAndInfo.GetRawName();
check( ScopeStart == ScopeEnd );
StackState->Current = StackState->Stack.Last();
// The stack should be ok, but it may be partially broken.
// This will happen if memory profiling starts in the middle of executing a background thread.
StackState->bIsBrokenCallstack = false;
}
else
{
const FName ShortName = Item.NameAndInfo.GetShortName();
UE_LOG( LogStats, Warning, TEXT( "Broken cycle scope end %s/%s, current %s" ),
*ThreadFName.ToString(),
*ShortName.ToString(),
*StackState->Current.ToString() );
// The stack is completely broken, only has the thread name and the last cycle scope.
// Rollback to the thread node.
StackState->bIsBrokenCallstack = true;
StackState->Stack.Empty();
StackState->Stack.Add( ThreadFName );
StackState->Current = ThreadFName;
}
}
}
}
if( bAtLeastOneMessage )
{
PreviousSeconds -= NumSecondsBetweenLogs;
}
}
if( bAtLeastOnePacket )
{
PreviousSeconds -= NumSecondsBetweenLogs;
}
}
UE_LOG( LogStats, Warning, TEXT( "NumMemoryOperations: %llu" ), NumMemoryOperations );
UE_LOG( LogStats, Warning, TEXT( "SequenceAllocationNum: %i" ), SequenceAllocationArray.Num() );
// Pass 2.
/*
TMap<uint32,FAllocationInfo> UniqueSeq;
TMultiMap<uint32,FAllocationInfo> OriginalAllocs;
TMultiMap<uint32,FAllocationInfo> BrokenAllocs;
for( const FAllocationInfo& Alloc : SequenceAllocationArray )
{
const FAllocationInfo* Found = UniqueSeq.Find(Alloc.SequenceTag);
if( !Found )
{
UniqueSeq.Add(Alloc.SequenceTag,Alloc);
}
else
{
OriginalAllocs.Add(Alloc.SequenceTag, *Found);
BrokenAllocs.Add(Alloc.SequenceTag, Alloc);
}
}
*/
// Sort all memory operation by the sequence tag, iterate through all operation and generate memory usage.
SequenceAllocationArray.Sort( TLess<FAllocationInfo>() );
// Alive allocations.
TMap<uint64, FAllocationInfo> AllocationMap;
TMultiMap<uint64, FAllocationInfo> FreeWithoutAllocMap;
TMultiMap<uint64, FAllocationInfo> DuplicatedAllocMap;
int32 NumDuplicatedMemoryOperations = 0;
int32 NumFWAMemoryOperations = 0; // FreeWithoutAlloc
UE_LOG( LogStats, Warning, TEXT( "Generating memory operations map" ) );
const int32 NumSequenceAllocations = SequenceAllocationArray.Num();
const int32 OnePercent = FMath::Max( NumSequenceAllocations / 100, 1024 );
for( int32 Index = 0; Index < NumSequenceAllocations; Index++ )
{
if( Index % OnePercent )
{
const double CurrentSeconds = FPlatformTime::Seconds();
if( CurrentSeconds > PreviousSeconds + NumSecondsBetweenLogs )
{
const int32 CurrentPct = int32( 100.0*(Index + 1) / NumSequenceAllocations );
UE_LOG( LogStats, Log, TEXT( "Processing allocations %3i%% (%10i/%10i)" ), CurrentPct, Index + 1, NumSequenceAllocations );
PreviousSeconds = CurrentSeconds;
}
}
const FAllocationInfo& Alloc = SequenceAllocationArray[Index];
const EMemoryOperation MemOp = Alloc.Op;
const uint64 Ptr = Alloc.Ptr;
const int64 Size = Alloc.Size;
const uint32 SequenceTag = Alloc.SequenceTag;
if( MemOp == EMemoryOperation::Alloc )
{
const FAllocationInfo* Found = AllocationMap.Find( Ptr );
if( !Found )
{
AllocationMap.Add( Ptr, Alloc );
}
else
{
const FAllocationInfo* FoundAndFreed = FreeWithoutAllocMap.Find( Found->Ptr );
const FAllocationInfo* FoundAndAllocated = FreeWithoutAllocMap.Find( Alloc.Ptr );
#if _DEBUG
if( FoundAndFreed )
{
const FString FoundAndFreedCallstack = GetCallstack( FoundAndFreed->EncodedCallstack );
}
if( FoundAndAllocated )
{
const FString FoundAndAllocatedCallstack = GetCallstack( FoundAndAllocated->EncodedCallstack );
}
NumDuplicatedMemoryOperations++;
const FString FoundCallstack = GetCallstack( Found->EncodedCallstack );
const FString AllocCallstack = GetCallstack( Alloc.EncodedCallstack );
#endif // _DEBUG
// Replace pointer.
AllocationMap.Add( Ptr, Alloc );
// Store the old pointer.
DuplicatedAllocMap.Add( Ptr, *Found );
}
}
else if( MemOp == EMemoryOperation::Free )
{
const FAllocationInfo* Found = AllocationMap.Find( Ptr );
if( Found )
{
const bool bIsValid = Alloc.SequenceTag > Found->SequenceTag;
if( !bIsValid )
{
UE_LOG( LogStats, Warning, TEXT( "InvalidFree Ptr: %llu, Seq: %i/%i" ), Ptr, SequenceTag, Found->SequenceTag );
}
AllocationMap.Remove( Ptr );
}
else
{
FreeWithoutAllocMap.Add( Ptr, Alloc );
NumFWAMemoryOperations++;
}
}
}
UE_LOG( LogStats, Warning, TEXT( "NumDuplicatedMemoryOperations: %i" ), NumDuplicatedMemoryOperations );
UE_LOG( LogStats, Warning, TEXT( "NumFWAMemoryOperations: %i" ), NumFWAMemoryOperations );
// Dump problematic allocations
DuplicatedAllocMap.ValueSort( FAllocationInfoGreater() );
//FreeWithoutAllocMap
uint64 TotalDuplicatedMemory = 0;
for( const auto& It : DuplicatedAllocMap )
{
const FAllocationInfo& Alloc = It.Value;
TotalDuplicatedMemory += Alloc.Size;
}
UE_LOG( LogStats, Warning, TEXT( "Dumping duplicated alloc map" ) );
const float MaxPctDisplayed = 0.80f;
uint64 DisplayedSoFar = 0;
for( const auto& It : DuplicatedAllocMap )
{
const FAllocationInfo& Alloc = It.Value;
const FString AllocCallstack = GetCallstack( Alloc.EncodedCallstack );
UE_LOG( LogStats, Log, TEXT( "%lli (%.2f MB) %s" ), Alloc.Size, Alloc.Size / 1024.0f / 1024.0f, *AllocCallstack );
DisplayedSoFar += Alloc.Size;
const float CurrentPct = (float)DisplayedSoFar / (float)TotalDuplicatedMemory;
if( CurrentPct > MaxPctDisplayed )
{
break;
}
}
GenerateMemoryUsageReport( AllocationMap );
}