/**
* Nulls out references to a given object
*
* @param InObject - Object to null references to
*/
void NullReferencesToObject(UObject* InObject)
{
TArray<UObject*> ReplaceableObjects;
TMap<UObject*, UObject*> ReplacementMap;
ReplacementMap.Add(InObject, NULL);
ReplacementMap.GenerateKeyArray(ReplaceableObjects);
// Find all the properties (and their corresponding objects) that refer to any of the objects to be replaced
TMap< UObject*, TArray<UProperty*> > ReferencingPropertiesMap;
for (FObjectIterator ObjIter; ObjIter; ++ObjIter)
{
UObject* CurObject = *ObjIter;
// Find the referencers of the objects to be replaced
FFindReferencersArchive FindRefsArchive(CurObject, ReplaceableObjects);
// Inform the object referencing any of the objects to be replaced about the properties that are being forcefully
// changed, and store both the object doing the referencing as well as the properties that were changed in a map (so that
// we can correctly call PostEditChange later)
TMap<UObject*, int32> CurNumReferencesMap;
TMultiMap<UObject*, UProperty*> CurReferencingPropertiesMMap;
if (FindRefsArchive.GetReferenceCounts(CurNumReferencesMap, CurReferencingPropertiesMMap) > 0)
{
TArray<UProperty*> CurReferencedProperties;
CurReferencingPropertiesMMap.GenerateValueArray(CurReferencedProperties);
ReferencingPropertiesMap.Add(CurObject, CurReferencedProperties);
for (TArray<UProperty*>::TConstIterator RefPropIter(CurReferencedProperties); RefPropIter; ++RefPropIter)
{
CurObject->PreEditChange(*RefPropIter);
}
}
}
// Iterate over the map of referencing objects/changed properties, forcefully replacing the references and then
// alerting the referencing objects the change has completed via PostEditChange
int32 NumObjsReplaced = 0;
for (TMap< UObject*, TArray<UProperty*> >::TConstIterator MapIter(ReferencingPropertiesMap); MapIter; ++MapIter)
{
++NumObjsReplaced;
UObject* CurReplaceObj = MapIter.Key();
const TArray<UProperty*>& RefPropArray = MapIter.Value();
FArchiveReplaceObjectRef<UObject> ReplaceAr(CurReplaceObj, ReplacementMap, false, true, false);
for (TArray<UProperty*>::TConstIterator RefPropIter(RefPropArray); RefPropIter; ++RefPropIter)
{
FPropertyChangedEvent PropertyEvent(*RefPropIter);
CurReplaceObj->PostEditChangeProperty(PropertyEvent);
}
if (!CurReplaceObj->HasAnyFlags(RF_Transient) && CurReplaceObj->GetOutermost() != GetTransientPackage())
{
if (!CurReplaceObj->RootPackageHasAnyFlags(PKG_CompiledIn))
{
CurReplaceObj->MarkPackageDirty();
}
}
}
}
void FBlueprintCompileReinstancer::ReplaceInstancesOfClass(UClass* OldClass, UClass* NewClass, UObject* OriginalCDO, TSet<UObject*>* ObjectsThatShouldUseOldStuff)
{
USelection* SelectedActors;
bool bSelectionChanged = false;
TArray<UObject*> ObjectsToReplace;
const bool bLogConversions = false; // for debugging
// Map of old objects to new objects
TMap<UObject*, UObject*> OldToNewInstanceMap;
TMap<UClass*, UClass*> OldToNewClassMap;
OldToNewClassMap.Add(OldClass, NewClass);
TMap<FStringAssetReference, UObject*> ReinstancedObjectsWeakReferenceMap;
// actors being replace
TArray<FActorReplacementHelper> ReplacementActors;
// A list of objects (e.g. Blueprints) that potentially have editors open that we need to refresh
TArray<UObject*> PotentialEditorsForRefreshing;
// A list of component owners that need their construction scripts re-ran (because a component of theirs has been reinstanced)
TSet<AActor*> OwnersToReconstruct;
// Set global flag to let system know we are reconstructing blueprint instances
TGuardValue<bool> GuardTemplateNameFlag(GIsReconstructingBlueprintInstances, true);
struct FObjectRemappingHelper
{
void OnObjectsReplaced(const TMap<UObject*, UObject*>& InReplacedObjects)
{
ReplacedObjects.Append(InReplacedObjects);
}
TMap<UObject*, UObject*> ReplacedObjects;
} ObjectRemappingHelper;
FDelegateHandle OnObjectsReplacedHandle = GEditor->OnObjectsReplaced().AddRaw(&ObjectRemappingHelper,&FObjectRemappingHelper::OnObjectsReplaced);
{ BP_SCOPED_COMPILER_EVENT_STAT(EKismetReinstancerStats_ReplaceInstancesOfClass);
const bool bIncludeDerivedClasses = false;
GetObjectsOfClass(OldClass, ObjectsToReplace, bIncludeDerivedClasses);
SelectedActors = GEditor->GetSelectedActors();
SelectedActors->BeginBatchSelectOperation();
SelectedActors->Modify();
// Then fix 'real' (non archetype) instances of the class
for (UObject* OldObject : ObjectsToReplace)
{
// Skip non-archetype instances, EXCEPT for component templates
const bool bIsComponent = NewClass->IsChildOf(UActorComponent::StaticClass());
if ((!bIsComponent && OldObject->IsTemplate()) || OldObject->IsPendingKill())
{
continue;
}
UBlueprint* CorrespondingBlueprint = Cast<UBlueprint>(OldObject->GetClass()->ClassGeneratedBy);
UObject* OldBlueprintDebugObject = nullptr;
// If this object is being debugged, cache it off so we can preserve the 'object being debugged' association
if ((CorrespondingBlueprint != nullptr) && (CorrespondingBlueprint->GetObjectBeingDebugged() == OldObject))
{
OldBlueprintDebugObject = OldObject;
}
AActor* OldActor = Cast<AActor>(OldObject);
UObject* NewUObject = nullptr;
// if the object to replace is an actor...
if (OldActor != nullptr)
{
FVector Location = FVector::ZeroVector;
FRotator Rotation = FRotator::ZeroRotator;
if (USceneComponent* OldRootComponent = OldActor->GetRootComponent())
{
Location = OldActor->GetActorLocation();
Rotation = OldActor->GetActorRotation();
}
// If this actor was spawned from an Archetype, we spawn the new actor from the new version of that archetype
UObject* OldArchetype = OldActor->GetArchetype();
UWorld* World = OldActor->GetWorld();
AActor* NewArchetype = Cast<AActor>(OldToNewInstanceMap.FindRef(OldArchetype));
// Check that either this was an instance of the class directly, or we found a new archetype for it
check(OldArchetype == OldClass->GetDefaultObject() || NewArchetype);
// Spawn the new actor instance, in the same level as the original, but deferring running the construction script until we have transferred modified properties
ULevel* ActorLevel = OldActor->GetLevel();
UClass** MappedClass = OldToNewClassMap.Find(OldActor->GetClass());
UClass* SpawnClass = MappedClass ? *MappedClass : NewClass;
FActorSpawnParameters SpawnInfo;
SpawnInfo.OverrideLevel = ActorLevel;
SpawnInfo.Template = NewArchetype;
SpawnInfo.bNoCollisionFail = true;
SpawnInfo.bDeferConstruction = true;
// Temporarily remove the deprecated flag so we can respawn the Blueprint in the level
const bool bIsClassDeprecated = SpawnClass->HasAnyClassFlags(CLASS_Deprecated);
SpawnClass->ClassFlags &= ~CLASS_Deprecated;
AActor* NewActor = World->SpawnActor(SpawnClass, &Location, &Rotation, SpawnInfo);
// Reassign the deprecated flag if it was previously assigned
if (bIsClassDeprecated)
{
SpawnClass->ClassFlags |= CLASS_Deprecated;
}
check(NewActor != nullptr);
NewUObject = NewActor;
// store the new actor for the second pass (NOTE: this detaches
// OldActor from all child/parent attachments)
//
// running the NewActor's construction-script is saved for that
// second pass (because the construction-script may reference
// another instance that hasn't been replaced yet).
ReplacementActors.Add(FActorReplacementHelper(NewActor, OldActor));
ReinstancedObjectsWeakReferenceMap.Add(OldObject, NewUObject);
OldActor->DestroyConstructedComponents(); // don't want to serialize components from the old actor
// Unregister native components so we don't copy any sub-components they generate for themselves (like UCameraComponent does)
OldActor->UnregisterAllComponents();
// Unregister any native components, might have cached state based on properties we are going to overwrite
NewActor->UnregisterAllComponents();
UEditorEngine::CopyPropertiesForUnrelatedObjects(OldActor, NewActor);
// reset properties/streams
NewActor->ResetPropertiesForConstruction();
// register native components
NewActor->RegisterAllComponents();
//
// clean up the old actor (unselect it, remove it from the world, etc.)...
if (OldActor->IsSelected())
{
GEditor->SelectActor(OldActor, /*bInSelected =*/false, /*bNotify =*/false);
bSelectionChanged = true;
}
if (GEditor->Layers.IsValid()) // ensure(NULL != GEditor->Layers) ?? While cooking the Layers is NULL.
{
GEditor->Layers->DisassociateActorFromLayers(OldActor);
}
World->EditorDestroyActor(OldActor, /*bShouldModifyLevel =*/true);
OldToNewInstanceMap.Add(OldActor, NewActor);
}
else
{
FName OldName(OldObject->GetFName());
OldObject->Rename(NULL, OldObject->GetOuter(), REN_DoNotDirty | REN_DontCreateRedirectors);
NewUObject = NewObject<UObject>(OldObject->GetOuter(), NewClass, OldName);
check(NewUObject != nullptr);
UEditorEngine::CopyPropertiesForUnrelatedObjects(OldObject, NewUObject);
if (UAnimInstance* AnimTree = Cast<UAnimInstance>(NewUObject))
{
// Initialising the anim instance isn't enough to correctly set up the skeletal mesh again in a
// paused world, need to initialise the skeletal mesh component that contains the anim instance.
if (USkeletalMeshComponent* SkelComponent = Cast<USkeletalMeshComponent>(AnimTree->GetOuter()))
{
SkelComponent->InitAnim(true);
}
}
OldObject->RemoveFromRoot();
OldObject->MarkPendingKill();
OldToNewInstanceMap.Add(OldObject, NewUObject);
if (bIsComponent)
{
UActorComponent* Component = Cast<UActorComponent>(NewUObject);
AActor* OwningActor = Component->GetOwner();
if (OwningActor)
{
OwningActor->ResetOwnedComponents();
// Check to see if they have an editor that potentially needs to be refreshed
if (OwningActor->GetClass()->ClassGeneratedBy)
{
PotentialEditorsForRefreshing.AddUnique(OwningActor->GetClass()->ClassGeneratedBy);
}
// we need to keep track of actor instances that need
// their construction scripts re-ran (since we've just
// replaced a component they own)
OwnersToReconstruct.Add(OwningActor);
}
}
}
// If this original object came from a blueprint and it was in the selected debug set, change the debugging to the new object.
if ((CorrespondingBlueprint) && (OldBlueprintDebugObject) && (NewUObject))
{
CorrespondingBlueprint->SetObjectBeingDebugged(NewUObject);
}
if (bLogConversions)
{
UE_LOG(LogBlueprint, Log, TEXT("Converted instance '%s' to '%s'"), *OldObject->GetPathName(), *NewUObject->GetPathName());
}
}
}
GEditor->OnObjectsReplaced().Remove(OnObjectsReplacedHandle);
// Now replace any pointers to the old archetypes/instances with pointers to the new one
TArray<UObject*> SourceObjects;
TArray<UObject*> DstObjects;
OldToNewInstanceMap.GenerateKeyArray(SourceObjects);
OldToNewInstanceMap.GenerateValueArray(DstObjects); // Also look for references in new spawned objects.
SourceObjects.Append(DstObjects);
FReplaceReferenceHelper::IncludeCDO(OldClass, NewClass, OldToNewInstanceMap, SourceObjects, OriginalCDO);
FReplaceReferenceHelper::FindAndReplaceReferences(SourceObjects, ObjectsThatShouldUseOldStuff, ObjectsToReplace, OldToNewInstanceMap, ReinstancedObjectsWeakReferenceMap);
{ BP_SCOPED_COMPILER_EVENT_STAT(EKismetReinstancerStats_ReplacementConstruction);
// the process of setting up new replacement actors is split into two
// steps (this here, is the second)...
//
// the "finalization" here runs the replacement actor's construction-
// script and is left until late to account for a scenario where the
// construction-script attempts to modify another instance of the
// same class... if this were to happen above, in the ObjectsToReplace
// loop, then accessing that other instance would cause an assert in
// UProperty::ContainerPtrToValuePtrInternal() (which appropriatly
// complains that the other instance's type doesn't match because it
// hasn't been replaced yet... that's why we wait until after
// FArchiveReplaceObjectRef to run construction-scripts).
for (FActorReplacementHelper& ReplacementActor : ReplacementActors)
{
ReplacementActor.Finalize(ObjectRemappingHelper.ReplacedObjects);
}
}
SelectedActors->EndBatchSelectOperation();
if (bSelectionChanged)
{
GEditor->NoteSelectionChange();
}
if (GEditor)
{
// Refresh any editors for objects that we've updated components for
for (auto BlueprintAsset : PotentialEditorsForRefreshing)
{
FBlueprintEditor* BlueprintEditor = static_cast<FBlueprintEditor*>(FAssetEditorManager::Get().FindEditorForAsset(BlueprintAsset, /*bFocusIfOpen =*/false));
if (BlueprintEditor)
{
BlueprintEditor->RefreshEditors();
}
}
}
// in the case where we're replacing component instances, we need to make
// sure to re-run their owner's construction scripts
for (AActor* ActorInstance : OwnersToReconstruct)
{
ActorInstance->RerunConstructionScripts();
}
}
void FStatsMemoryDumpCommand::ProcessMemoryOperations( const TMap<int64, FStatPacketArray>& CombinedHistory )
{
// This is only example code, no fully implemented, may sometimes crash.
// This code is not optimized.
double PreviousSeconds = FPlatformTime::Seconds();
uint64 NumMemoryOperations = 0;
// Generate frames
TArray<int64> Frames;
CombinedHistory.GenerateKeyArray( Frames );
Frames.Sort();
// Raw stats callstack for this stat packet array.
TMap<FName, FStackState> StackStates;
// All allocation ordered by the sequence tag.
// There is an assumption that the sequence tag will not turn-around.
//TMap<uint32, FAllocationInfo> SequenceAllocationMap;
TArray<FAllocationInfo> SequenceAllocationArray;
// Pass 1.
// Read all stats messages, parse all memory operations and decode callstacks.
const int64 FirstFrame = 0;
PreviousSeconds -= NumSecondsBetweenLogs;
for( int32 FrameIndex = 0; FrameIndex < Frames.Num(); ++FrameIndex )
{
{
const double CurrentSeconds = FPlatformTime::Seconds();
if( CurrentSeconds > PreviousSeconds + NumSecondsBetweenLogs )
{
UE_LOG( LogStats, Warning, TEXT( "Processing frame %i/%i" ), FrameIndex+1, Frames.Num() );
PreviousSeconds = CurrentSeconds;
}
}
const int64 TargetFrame = Frames[FrameIndex];
const int64 Diff = TargetFrame - FirstFrame;
const FStatPacketArray& Frame = CombinedHistory.FindChecked( TargetFrame );
bool bAtLeastOnePacket = false;
for( int32 PacketIndex = 0; PacketIndex < Frame.Packets.Num(); PacketIndex++ )
{
{
const double CurrentSeconds = FPlatformTime::Seconds();
if( CurrentSeconds > PreviousSeconds + NumSecondsBetweenLogs )
{
UE_LOG( LogStats, Log, TEXT( "Processing packet %i/%i" ), PacketIndex, Frame.Packets.Num() );
PreviousSeconds = CurrentSeconds;
bAtLeastOnePacket = true;
}
}
const FStatPacket& StatPacket = *Frame.Packets[PacketIndex];
const FName& ThreadFName = StatsThreadStats.Threads.FindChecked( StatPacket.ThreadId );
const uint32 NewThreadID = ThreadIDtoStatID.FindChecked( StatPacket.ThreadId );
FStackState* StackState = StackStates.Find( ThreadFName );
if( !StackState )
{
StackState = &StackStates.Add( ThreadFName );
StackState->Stack.Add( ThreadFName );
StackState->Current = ThreadFName;
}
const FStatMessagesArray& Data = StatPacket.StatMessages;
int32 LastPct = 0;
const int32 NumDataElements = Data.Num();
const int32 OnerPercent = FMath::Max( NumDataElements / 100, 1024 );
bool bAtLeastOneMessage = false;
for( int32 Index = 0; Index < NumDataElements; Index++ )
{
if( Index % OnerPercent )
{
const double CurrentSeconds = FPlatformTime::Seconds();
if( CurrentSeconds > PreviousSeconds + NumSecondsBetweenLogs )
{
const int32 CurrentPct = int32( 100.0*(Index + 1) / NumDataElements );
UE_LOG( LogStats, Log, TEXT( "Processing %3i%% (%i/%i) stat messages" ), CurrentPct, Index, NumDataElements );
PreviousSeconds = CurrentSeconds;
bAtLeastOneMessage = true;
}
}
const FStatMessage& Item = Data[Index];
const EStatOperation::Type Op = Item.NameAndInfo.GetField<EStatOperation>();
const FName RawName = Item.NameAndInfo.GetRawName();
if( Op == EStatOperation::CycleScopeStart || Op == EStatOperation::CycleScopeEnd || Op == EStatOperation::Memory )
{
if( Op == EStatOperation::CycleScopeStart )
{
StackState->Stack.Add( RawName );
StackState->Current = RawName;
}
else if( Op == EStatOperation::Memory )
{
// Experimental code used only to test the implementation.
// First memory operation is Alloc or Free
const uint64 EncodedPtr = Item.GetValue_Ptr();
const bool bIsAlloc = (EncodedPtr & (uint64)EMemoryOperation::Alloc) != 0;
const bool bIsFree = (EncodedPtr & (uint64)EMemoryOperation::Free) != 0;
const uint64 Ptr = EncodedPtr & ~(uint64)EMemoryOperation::Mask;
if( bIsAlloc )
{
NumMemoryOperations++;
// @see FStatsMallocProfilerProxy::TrackAlloc
// After alloc ptr message there is always alloc size message and the sequence tag.
Index++;
const FStatMessage& AllocSizeMessage = Data[Index];
const int64 AllocSize = AllocSizeMessage.GetValue_int64();
// Read operation sequence tag.
Index++;
const FStatMessage& SequenceTagMessage = Data[Index];
const uint32 SequenceTag = SequenceTagMessage.GetValue_int64();
// Create a callstack.
TArray<FName> StatsBasedCallstack;
for( const auto& StackName : StackState->Stack )
{
StatsBasedCallstack.Add( StackName );
}
// Add a new allocation.
SequenceAllocationArray.Add(
FAllocationInfo(
Ptr,
AllocSize,
StatsBasedCallstack,
SequenceTag,
EMemoryOperation::Alloc,
StackState->bIsBrokenCallstack
) );
}
else if( bIsFree )
{
NumMemoryOperations++;
// Read operation sequence tag.
Index++;
const FStatMessage& SequenceTagMessage = Data[Index];
const uint32 SequenceTag = SequenceTagMessage.GetValue_int64();
// Create a callstack.
/*
TArray<FName> StatsBasedCallstack;
for( const auto& RawName : StackState->Stack )
{
StatsBasedCallstack.Add( RawName );
}
*/
// Add a new free.
SequenceAllocationArray.Add(
FAllocationInfo(
Ptr,
0,
TArray<FName>()/*StatsBasedCallstack*/,
SequenceTag,
EMemoryOperation::Free,
StackState->bIsBrokenCallstack
) );
}
else
{
UE_LOG( LogStats, Warning, TEXT( "Pointer from a memory operation is invalid" ) );
}
}
else if( Op == EStatOperation::CycleScopeEnd )
{
if( StackState->Stack.Num() > 1 )
{
const FName ScopeStart = StackState->Stack.Pop();
const FName ScopeEnd = Item.NameAndInfo.GetRawName();
check( ScopeStart == ScopeEnd );
StackState->Current = StackState->Stack.Last();
// The stack should be ok, but it may be partially broken.
// This will happen if memory profiling starts in the middle of executing a background thread.
StackState->bIsBrokenCallstack = false;
}
else
{
const FName ShortName = Item.NameAndInfo.GetShortName();
UE_LOG( LogStats, Warning, TEXT( "Broken cycle scope end %s/%s, current %s" ),
*ThreadFName.ToString(),
*ShortName.ToString(),
*StackState->Current.ToString() );
// The stack is completely broken, only has the thread name and the last cycle scope.
// Rollback to the thread node.
StackState->bIsBrokenCallstack = true;
StackState->Stack.Empty();
StackState->Stack.Add( ThreadFName );
StackState->Current = ThreadFName;
}
}
}
}
if( bAtLeastOneMessage )
{
PreviousSeconds -= NumSecondsBetweenLogs;
}
}
if( bAtLeastOnePacket )
{
PreviousSeconds -= NumSecondsBetweenLogs;
}
}
UE_LOG( LogStats, Warning, TEXT( "NumMemoryOperations: %llu" ), NumMemoryOperations );
UE_LOG( LogStats, Warning, TEXT( "SequenceAllocationNum: %i" ), SequenceAllocationArray.Num() );
// Pass 2.
/*
TMap<uint32,FAllocationInfo> UniqueSeq;
TMultiMap<uint32,FAllocationInfo> OriginalAllocs;
TMultiMap<uint32,FAllocationInfo> BrokenAllocs;
for( const FAllocationInfo& Alloc : SequenceAllocationArray )
{
const FAllocationInfo* Found = UniqueSeq.Find(Alloc.SequenceTag);
if( !Found )
{
UniqueSeq.Add(Alloc.SequenceTag,Alloc);
}
else
{
OriginalAllocs.Add(Alloc.SequenceTag, *Found);
BrokenAllocs.Add(Alloc.SequenceTag, Alloc);
}
}
*/
// Sort all memory operation by the sequence tag, iterate through all operation and generate memory usage.
SequenceAllocationArray.Sort( TLess<FAllocationInfo>() );
// Alive allocations.
TMap<uint64, FAllocationInfo> AllocationMap;
TMultiMap<uint64, FAllocationInfo> FreeWithoutAllocMap;
TMultiMap<uint64, FAllocationInfo> DuplicatedAllocMap;
int32 NumDuplicatedMemoryOperations = 0;
int32 NumFWAMemoryOperations = 0; // FreeWithoutAlloc
UE_LOG( LogStats, Warning, TEXT( "Generating memory operations map" ) );
const int32 NumSequenceAllocations = SequenceAllocationArray.Num();
const int32 OnePercent = FMath::Max( NumSequenceAllocations / 100, 1024 );
for( int32 Index = 0; Index < NumSequenceAllocations; Index++ )
{
if( Index % OnePercent )
{
const double CurrentSeconds = FPlatformTime::Seconds();
if( CurrentSeconds > PreviousSeconds + NumSecondsBetweenLogs )
{
const int32 CurrentPct = int32( 100.0*(Index + 1) / NumSequenceAllocations );
UE_LOG( LogStats, Log, TEXT( "Processing allocations %3i%% (%10i/%10i)" ), CurrentPct, Index + 1, NumSequenceAllocations );
PreviousSeconds = CurrentSeconds;
}
}
const FAllocationInfo& Alloc = SequenceAllocationArray[Index];
const EMemoryOperation MemOp = Alloc.Op;
const uint64 Ptr = Alloc.Ptr;
const int64 Size = Alloc.Size;
const uint32 SequenceTag = Alloc.SequenceTag;
if( MemOp == EMemoryOperation::Alloc )
{
const FAllocationInfo* Found = AllocationMap.Find( Ptr );
if( !Found )
{
AllocationMap.Add( Ptr, Alloc );
}
else
{
const FAllocationInfo* FoundAndFreed = FreeWithoutAllocMap.Find( Found->Ptr );
const FAllocationInfo* FoundAndAllocated = FreeWithoutAllocMap.Find( Alloc.Ptr );
#if _DEBUG
if( FoundAndFreed )
{
const FString FoundAndFreedCallstack = GetCallstack( FoundAndFreed->EncodedCallstack );
}
if( FoundAndAllocated )
{
const FString FoundAndAllocatedCallstack = GetCallstack( FoundAndAllocated->EncodedCallstack );
}
NumDuplicatedMemoryOperations++;
const FString FoundCallstack = GetCallstack( Found->EncodedCallstack );
const FString AllocCallstack = GetCallstack( Alloc.EncodedCallstack );
#endif // _DEBUG
// Replace pointer.
AllocationMap.Add( Ptr, Alloc );
// Store the old pointer.
DuplicatedAllocMap.Add( Ptr, *Found );
}
}
else if( MemOp == EMemoryOperation::Free )
{
const FAllocationInfo* Found = AllocationMap.Find( Ptr );
if( Found )
{
const bool bIsValid = Alloc.SequenceTag > Found->SequenceTag;
if( !bIsValid )
{
UE_LOG( LogStats, Warning, TEXT( "InvalidFree Ptr: %llu, Seq: %i/%i" ), Ptr, SequenceTag, Found->SequenceTag );
}
AllocationMap.Remove( Ptr );
}
else
{
FreeWithoutAllocMap.Add( Ptr, Alloc );
NumFWAMemoryOperations++;
}
}
}
UE_LOG( LogStats, Warning, TEXT( "NumDuplicatedMemoryOperations: %i" ), NumDuplicatedMemoryOperations );
UE_LOG( LogStats, Warning, TEXT( "NumFWAMemoryOperations: %i" ), NumFWAMemoryOperations );
// Dump problematic allocations
DuplicatedAllocMap.ValueSort( FAllocationInfoGreater() );
//FreeWithoutAllocMap
uint64 TotalDuplicatedMemory = 0;
for( const auto& It : DuplicatedAllocMap )
{
const FAllocationInfo& Alloc = It.Value;
TotalDuplicatedMemory += Alloc.Size;
}
UE_LOG( LogStats, Warning, TEXT( "Dumping duplicated alloc map" ) );
const float MaxPctDisplayed = 0.80f;
uint64 DisplayedSoFar = 0;
for( const auto& It : DuplicatedAllocMap )
{
const FAllocationInfo& Alloc = It.Value;
const FString AllocCallstack = GetCallstack( Alloc.EncodedCallstack );
UE_LOG( LogStats, Log, TEXT( "%lli (%.2f MB) %s" ), Alloc.Size, Alloc.Size / 1024.0f / 1024.0f, *AllocCallstack );
DisplayedSoFar += Alloc.Size;
const float CurrentPct = (float)DisplayedSoFar / (float)TotalDuplicatedMemory;
if( CurrentPct > MaxPctDisplayed )
{
break;
}
}
GenerateMemoryUsageReport( AllocationMap );
}
void FStatsMemoryDumpCommand::InternalRun()
{
FParse::Value( FCommandLine::Get(), TEXT( "-INFILE=" ), SourceFilepath );
const int64 Size = IFileManager::Get().FileSize( *SourceFilepath );
if( Size < 4 )
{
UE_LOG( LogStats, Error, TEXT( "Could not open: %s" ), *SourceFilepath );
return;
}
TAutoPtr<FArchive> FileReader( IFileManager::Get().CreateFileReader( *SourceFilepath ) );
if( !FileReader )
{
UE_LOG( LogStats, Error, TEXT( "Could not open: %s" ), *SourceFilepath );
return;
}
if( !Stream.ReadHeader( *FileReader ) )
{
UE_LOG( LogStats, Error, TEXT( "Could not open, bad magic: %s" ), *SourceFilepath );
return;
}
UE_LOG( LogStats, Warning, TEXT( "Reading a raw stats file for memory profiling: %s" ), *SourceFilepath );
const bool bIsFinalized = Stream.Header.IsFinalized();
check( bIsFinalized );
check( Stream.Header.Version == EStatMagicWithHeader::VERSION_5 );
StatsThreadStats.MarkAsLoaded();
TArray<FStatMessage> Messages;
if( Stream.Header.bRawStatsFile )
{
FScopeLogTime SLT( TEXT( "FStatsMemoryDumpCommand::InternalRun" ), nullptr, FScopeLogTime::ScopeLog_Seconds );
// Read metadata.
TArray<FStatMessage> MetadataMessages;
Stream.ReadFNamesAndMetadataMessages( *FileReader, MetadataMessages );
StatsThreadStats.ProcessMetaDataOnly( MetadataMessages );
// Find all UObject metadata messages.
for( const auto& Meta : MetadataMessages )
{
FName LongName = Meta.NameAndInfo.GetRawName();
const FString Desc = FStatNameAndInfo::GetShortNameFrom( LongName ).GetPlainNameString();
const bool bContainsUObject = Desc.Contains( TEXT( "//" ) );
if( bContainsUObject )
{
UObjectNames.Add( LongName );
}
}
const int64 CurrentFilePos = FileReader->Tell();
// Update profiler's metadata.
CreateThreadsMapping();
// Read frames offsets.
Stream.ReadFramesOffsets( *FileReader );
// Buffer used to store the compressed and decompressed data.
TArray<uint8> SrcArray;
TArray<uint8> DestArray;
const bool bHasCompressedData = Stream.Header.HasCompressedData();
check( bHasCompressedData );
TMap<int64, FStatPacketArray> CombinedHistory;
int64 TotalDataSize = 0;
int64 TotalStatMessagesNum = 0;
int64 MaximumPacketSize = 0;
int64 TotalPacketsNum = 0;
// Read all packets sequentially, force by the memory profiler which is now a part of the raw stats.
// !!CAUTION!! Frame number in the raw stats is pointless, because it is time based, not frame based.
// Background threads usually execute time consuming operations, so the frame number won't be valid.
// Needs to be combined by the thread and the time, not by the frame number.
{
// Display log information once per 5 seconds to avoid spamming.
double PreviousSeconds = FPlatformTime::Seconds();
const int64 FrameOffset0 = Stream.FramesInfo[0].FrameFileOffset;
FileReader->Seek( FrameOffset0 );
const int64 FileSize = FileReader->TotalSize();
while( FileReader->Tell() < FileSize )
{
// Read the compressed data.
FCompressedStatsData UncompressedData( SrcArray, DestArray );
*FileReader << UncompressedData;
if( UncompressedData.HasReachedEndOfCompressedData() )
{
break;
}
FMemoryReader MemoryReader( DestArray, true );
FStatPacket* StatPacket = new FStatPacket();
Stream.ReadStatPacket( MemoryReader, *StatPacket );
const int64 StatPacketFrameNum = StatPacket->Frame;
FStatPacketArray& Frame = CombinedHistory.FindOrAdd( StatPacketFrameNum );
// Check if we need to combine packets from the same thread.
FStatPacket** CombinedPacket = Frame.Packets.FindByPredicate( [&]( FStatPacket* Item ) -> bool
{
return Item->ThreadId == StatPacket->ThreadId;
} );
const int64 PacketSize = StatPacket->StatMessages.GetAllocatedSize();
TotalStatMessagesNum += StatPacket->StatMessages.Num();
if( CombinedPacket )
{
TotalDataSize -= (*CombinedPacket)->StatMessages.GetAllocatedSize();
(*CombinedPacket)->StatMessages += StatPacket->StatMessages;
TotalDataSize += (*CombinedPacket)->StatMessages.GetAllocatedSize();
delete StatPacket;
}
else
{
Frame.Packets.Add( StatPacket );
TotalDataSize += PacketSize;
}
const double CurrentSeconds = FPlatformTime::Seconds();
if( CurrentSeconds > PreviousSeconds + NumSecondsBetweenLogs )
{
const int32 PctPos = int32( 100.0*FileReader->Tell() / FileSize );
UE_LOG( LogStats, Log, TEXT( "%3i%% %10llu (%.1f MB) read messages, last read frame %4i" ), PctPos, TotalStatMessagesNum, TotalDataSize / 1024.0f / 1024.0f, StatPacketFrameNum );
PreviousSeconds = CurrentSeconds;
}
MaximumPacketSize = FMath::Max( MaximumPacketSize, PacketSize );
TotalPacketsNum++;
}
}
// Dump frame stats
for( const auto& It : CombinedHistory )
{
const int64 FrameNum = It.Key;
int64 FramePacketsSize = 0;
int64 FrameStatMessages = 0;
int64 FramePackets = It.Value.Packets.Num(); // Threads
for( const auto& It2 : It.Value.Packets )
{
FramePacketsSize += It2->StatMessages.GetAllocatedSize();
FrameStatMessages += It2->StatMessages.Num();
}
UE_LOG( LogStats, Warning, TEXT( "Frame: %10llu/%3lli Size: %.1f MB / %10lli" ),
FrameNum,
FramePackets,
FramePacketsSize / 1024.0f / 1024.0f,
FrameStatMessages );
}
UE_LOG( LogStats, Warning, TEXT( "TotalPacketSize: %.1f MB, Max: %1f MB" ),
TotalDataSize / 1024.0f / 1024.0f,
MaximumPacketSize / 1024.0f / 1024.0f );
TArray<int64> Frames;
CombinedHistory.GenerateKeyArray( Frames );
Frames.Sort();
const int64 MiddleFrame = Frames[Frames.Num() / 2];
ProcessMemoryOperations( CombinedHistory );
}
}
void FRawProfilerSession::PrepareLoading()
{
SCOPE_LOG_TIME_FUNC();
const FString Filepath = DataFilepath + FStatConstants::StatsFileRawExtension;
const int64 Size = IFileManager::Get().FileSize( *Filepath );
if( Size < 4 )
{
UE_LOG( LogStats, Error, TEXT( "Could not open: %s" ), *Filepath );
return;
}
TAutoPtr<FArchive> FileReader( IFileManager::Get().CreateFileReader( *Filepath ) );
if( !FileReader )
{
UE_LOG( LogStats, Error, TEXT( "Could not open: %s" ), *Filepath );
return;
}
if( !Stream.ReadHeader( *FileReader ) )
{
UE_LOG( LogStats, Error, TEXT( "Could not open, bad magic: %s" ), *Filepath );
return;
}
const bool bIsFinalized = Stream.Header.IsFinalized();
check( bIsFinalized );
check( Stream.Header.Version == EStatMagicWithHeader::VERSION_5 );
StatsThreadStats.MarkAsLoaded();
TArray<FStatMessage> Messages;
if( Stream.Header.bRawStatsFile )
{
// Read metadata.
TArray<FStatMessage> MetadataMessages;
Stream.ReadFNamesAndMetadataMessages( *FileReader, MetadataMessages );
StatsThreadStats.ProcessMetaDataOnly( MetadataMessages );
const FName F00245 = FName(245, 245, 0);
const FName F11602 = FName(11602, 11602, 0);
const FName F06394 = FName(6394, 6394, 0);
const int64 CurrentFilePos = FileReader->Tell();
// Update profiler's metadata.
StatMetaData->UpdateFromStatsState( StatsThreadStats );
const uint32 GameThreadID = GetMetaData()->GetGameThreadID();
// Read frames offsets.
Stream.ReadFramesOffsets( *FileReader );
// Buffer used to store the compressed and decompressed data.
TArray<uint8> SrcArray;
TArray<uint8> DestArray;
const bool bHasCompressedData = Stream.Header.HasCompressedData();
check(bHasCompressedData);
TMap<int64, FStatPacketArray> CombinedHistory;
int64 TotalPacketSize = 0;
int64 MaximumPacketSize = 0;
// Read all packets sequentially, force by the memory profiler which is now a part of the raw stats.
// !!CAUTION!! Frame number in the raw stats is pointless, because it is time based, not frame based.
// Background threads usually execute time consuming operations, so the frame number won't be valid.
// Needs to be combined by the thread and the time, not by the frame number.
{
int64 FrameOffset0 = Stream.FramesInfo[0].FrameFileOffset;
FileReader->Seek( FrameOffset0 );
const int64 FileSize = FileReader->TotalSize();
while( FileReader->Tell() < FileSize )
{
// Read the compressed data.
FCompressedStatsData UncompressedData( SrcArray, DestArray );
*FileReader << UncompressedData;
if( UncompressedData.HasReachedEndOfCompressedData() )
{
break;
}
FMemoryReader MemoryReader( DestArray, true );
FStatPacket* StatPacket = new FStatPacket();
Stream.ReadStatPacket( MemoryReader, *StatPacket );
const int64 FrameNum = StatPacket->Frame;
FStatPacketArray& Frame = CombinedHistory.FindOrAdd(FrameNum);
// Check if we need to combine packets from the same thread.
FStatPacket** CombinedPacket = Frame.Packets.FindByPredicate([&](FStatPacket* Item) -> bool
{
return Item->ThreadId == StatPacket->ThreadId;
});
if( CombinedPacket )
{
(*CombinedPacket)->StatMessages += StatPacket->StatMessages;
}
else
{
Frame.Packets.Add(StatPacket);
}
const int64 CurrentPos = FileReader->Tell();
const int32 PctPos = int32(100.0f*CurrentPos/FileSize);
UE_LOG( LogStats, Log, TEXT( "%3i Processing FStatPacket: Frame %5i for thread %5i with %6i messages (%.1f MB)" ),
PctPos,
StatPacket->Frame,
StatPacket->ThreadId,
StatPacket->StatMessages.Num(),
StatPacket->StatMessages.GetAllocatedSize()/1024.0f/1024.0f );
const int64 PacketSize = StatPacket->StatMessages.GetAllocatedSize();
TotalPacketSize += PacketSize;
MaximumPacketSize = FMath::Max( MaximumPacketSize, PacketSize );
}
}
UE_LOG( LogStats, Log, TEXT( "TotalPacketSize: %.1f MB, Max: %1f MB" ),
TotalPacketSize/1024.0f/1024.0f,
MaximumPacketSize/1024.0f/1024.0f );
TArray<int64> Frames;
CombinedHistory.GenerateKeyArray(Frames);
Frames.Sort();
const int64 MiddleFrame = Frames[Frames.Num()/2];
// Remove all frames without the game thread messages.
for (int32 FrameIndex = 0; FrameIndex < Frames.Num(); ++FrameIndex)
{
const int64 TargetFrame = Frames[FrameIndex];
const FStatPacketArray& Frame = CombinedHistory.FindChecked( TargetFrame );
const double GameThreadTimeMS = GetMetaData()->ConvertCyclesToMS( GetFastThreadFrameTimeInternal( Frame, EThreadType::Game ) );
if (GameThreadTimeMS == 0.0f)
{
CombinedHistory.Remove( TargetFrame );
Frames.RemoveAt( FrameIndex );
FrameIndex--;
}
}
StatMetaData->SecondsPerCycle = GetSecondsPerCycle( CombinedHistory.FindChecked(MiddleFrame) );
check( StatMetaData->GetSecondsPerCycle() > 0.0 );
//const int32 FirstGameThreadFrame = FindFirstFrameWithGameThread( CombinedHistory, Frames );
// Prepare profiler frame.
{
SCOPE_LOG_TIME( TEXT( "Preparing profiler frames" ), nullptr );
// Prepare profiler frames.
double ElapsedTimeMS = 0;
for( int32 FrameIndex = 0; FrameIndex < Frames.Num(); ++FrameIndex )
{
const int64 TargetFrame = Frames[FrameIndex];
const FStatPacketArray& Frame = CombinedHistory.FindChecked(TargetFrame);
const double GameThreadTimeMS = GetMetaData()->ConvertCyclesToMS( GetFastThreadFrameTimeInternal(Frame,EThreadType::Game) );
if( GameThreadTimeMS == 0.0f )
{
continue;
}
const double RenderThreadTimeMS = GetMetaData()->ConvertCyclesToMS( GetFastThreadFrameTimeInternal(Frame,EThreadType::Renderer) );
// Update mini-view, convert from cycles to ms.
TMap<uint32, float> ThreadTimesMS;
ThreadTimesMS.Add( GameThreadID, GameThreadTimeMS );
ThreadTimesMS.Add( GetMetaData()->GetRenderThreadID()[0], RenderThreadTimeMS );
// Pass the reference to the stats' metadata.
OnAddThreadTime.ExecuteIfBound( FrameIndex, ThreadTimesMS, StatMetaData );
// Create a new profiler frame and add it to the stream.
ElapsedTimeMS += GameThreadTimeMS;
FProfilerFrame* ProfilerFrame = new FProfilerFrame( TargetFrame, GameThreadTimeMS, ElapsedTimeMS );
ProfilerFrame->ThreadTimesMS = ThreadTimesMS;
ProfilerStream.AddProfilerFrame( TargetFrame, ProfilerFrame );
}
}
// Process the raw stats data.
{
SCOPE_LOG_TIME( TEXT( "Processing the raw stats" ), nullptr );
double CycleCounterAdjustmentMS = 0.0f;
// Read the raw stats messages.
for( int32 FrameIndex = 0; FrameIndex < Frames.Num()-1; ++FrameIndex )
{
const int64 TargetFrame = Frames[FrameIndex];
const FStatPacketArray& Frame = CombinedHistory.FindChecked(TargetFrame);
FProfilerFrame* ProfilerFrame = ProfilerStream.GetProfilerFrame( FrameIndex );
UE_CLOG( FrameIndex % 8 == 0, LogStats, Log, TEXT( "Processing raw stats frame: %4i/%4i" ), FrameIndex, Frames.Num() );
ProcessStatPacketArray( Frame, *ProfilerFrame, FrameIndex ); // or ProfilerFrame->TargetFrame
// Find the first cycle counter for the game thread.
if( CycleCounterAdjustmentMS == 0.0f )
{
CycleCounterAdjustmentMS = ProfilerFrame->Root->CycleCounterStartTimeMS;
}
// Update thread time and mark profiler frame as valid and ready for use.
ProfilerFrame->MarkAsValid();
}
// Adjust all profiler frames.
ProfilerStream.AdjustCycleCounters( CycleCounterAdjustmentMS );
}
}
const int64 AllocatedSize = ProfilerStream.GetAllocatedSize();
// We have the whole metadata and basic information about the raw stats file, start ticking the profiler session.
//OnTickHandle = FTicker::GetCoreTicker().AddTicker( OnTick, 0.25f );
#if 0
if( SessionType == EProfilerSessionTypes::OfflineRaw )
{
// Broadcast that a capture file has been fully processed.
OnCaptureFileProcessed.ExecuteIfBound( GetInstanceID() );
}
#endif // 0
}