void FBlueprintCompileReinstancer::ReplaceInstancesOfClass(UClass* OldClass, UClass* NewClass, UObject*	OriginalCDO, TSet<UObject*>* ObjectsThatShouldUseOldStuff)
{
	USelection* SelectedActors;
	bool bSelectionChanged = false;
	TArray<UObject*> ObjectsToReplace;
	const bool bLogConversions = false; // for debugging

	// Map of old objects to new objects
	TMap<UObject*, UObject*> OldToNewInstanceMap;
	TMap<UClass*, UClass*>   OldToNewClassMap;
	OldToNewClassMap.Add(OldClass, NewClass);

	TMap<FStringAssetReference, UObject*> ReinstancedObjectsWeakReferenceMap;

	// actors being replace
	TArray<FActorReplacementHelper> ReplacementActors;

	// A list of objects (e.g. Blueprints) that potentially have editors open that we need to refresh
	TArray<UObject*> PotentialEditorsForRefreshing;

	// A list of component owners that need their construction scripts re-ran (because a component of theirs has been reinstanced)
	TSet<AActor*> OwnersToReconstruct;

	// Set global flag to let system know we are reconstructing blueprint instances
	TGuardValue<bool> GuardTemplateNameFlag(GIsReconstructingBlueprintInstances, true);

	struct FObjectRemappingHelper
	{
		void OnObjectsReplaced(const TMap<UObject*, UObject*>& InReplacedObjects)
		{
			ReplacedObjects.Append(InReplacedObjects);
		}

		TMap<UObject*, UObject*> ReplacedObjects;
	} ObjectRemappingHelper;

	FDelegateHandle OnObjectsReplacedHandle = GEditor->OnObjectsReplaced().AddRaw(&ObjectRemappingHelper,&FObjectRemappingHelper::OnObjectsReplaced);

	{ BP_SCOPED_COMPILER_EVENT_STAT(EKismetReinstancerStats_ReplaceInstancesOfClass);


		const bool bIncludeDerivedClasses = false;
		GetObjectsOfClass(OldClass, ObjectsToReplace, bIncludeDerivedClasses);
	
		SelectedActors = GEditor->GetSelectedActors();
		SelectedActors->BeginBatchSelectOperation();
		SelectedActors->Modify();
		
		// Then fix 'real' (non archetype) instances of the class
		for (UObject* OldObject : ObjectsToReplace)
		{
			// Skip non-archetype instances, EXCEPT for component templates
			const bool bIsComponent = NewClass->IsChildOf(UActorComponent::StaticClass());
			if ((!bIsComponent && OldObject->IsTemplate()) || OldObject->IsPendingKill())
			{
				continue;
			}

			UBlueprint* CorrespondingBlueprint  = Cast<UBlueprint>(OldObject->GetClass()->ClassGeneratedBy);
			UObject*    OldBlueprintDebugObject = nullptr;
			// If this object is being debugged, cache it off so we can preserve the 'object being debugged' association
			if ((CorrespondingBlueprint != nullptr) && (CorrespondingBlueprint->GetObjectBeingDebugged() == OldObject))
			{
				OldBlueprintDebugObject = OldObject;
			}

			AActor*  OldActor = Cast<AActor>(OldObject);
			UObject* NewUObject = nullptr;
			// if the object to replace is an actor...
			if (OldActor != nullptr)
			{
				FVector  Location = FVector::ZeroVector;
				FRotator Rotation = FRotator::ZeroRotator;
				if (USceneComponent* OldRootComponent = OldActor->GetRootComponent())
				{
					Location = OldActor->GetActorLocation();
					Rotation = OldActor->GetActorRotation();
				}

				// If this actor was spawned from an Archetype, we spawn the new actor from the new version of that archetype
				UObject* OldArchetype = OldActor->GetArchetype();
				UWorld*  World        = OldActor->GetWorld();
				AActor*  NewArchetype = Cast<AActor>(OldToNewInstanceMap.FindRef(OldArchetype));
				// Check that either this was an instance of the class directly, or we found a new archetype for it
				check(OldArchetype == OldClass->GetDefaultObject() || NewArchetype);

				// Spawn the new actor instance, in the same level as the original, but deferring running the construction script until we have transferred modified properties
				ULevel*  ActorLevel  = OldActor->GetLevel();
				UClass** MappedClass = OldToNewClassMap.Find(OldActor->GetClass());
				UClass*  SpawnClass  = MappedClass ? *MappedClass : NewClass;

				FActorSpawnParameters SpawnInfo;
				SpawnInfo.OverrideLevel      = ActorLevel;
				SpawnInfo.Template           = NewArchetype;
				SpawnInfo.bNoCollisionFail   = true;
				SpawnInfo.bDeferConstruction = true;

				// Temporarily remove the deprecated flag so we can respawn the Blueprint in the level
				const bool bIsClassDeprecated = SpawnClass->HasAnyClassFlags(CLASS_Deprecated);
				SpawnClass->ClassFlags &= ~CLASS_Deprecated;

				AActor* NewActor = World->SpawnActor(SpawnClass, &Location, &Rotation, SpawnInfo);
				// Reassign the deprecated flag if it was previously assigned
				if (bIsClassDeprecated)
				{
					SpawnClass->ClassFlags |= CLASS_Deprecated;
				}

				check(NewActor != nullptr);
				NewUObject = NewActor;
				// store the new actor for the second pass (NOTE: this detaches 
				// OldActor from all child/parent attachments)
				//
				// running the NewActor's construction-script is saved for that 
				// second pass (because the construction-script may reference 
				// another instance that hasn't been replaced yet).
				ReplacementActors.Add(FActorReplacementHelper(NewActor, OldActor));

				ReinstancedObjectsWeakReferenceMap.Add(OldObject, NewUObject);

				OldActor->DestroyConstructedComponents(); // don't want to serialize components from the old actor
				// Unregister native components so we don't copy any sub-components they generate for themselves (like UCameraComponent does)
				OldActor->UnregisterAllComponents();

				// Unregister any native components, might have cached state based on properties we are going to overwrite
				NewActor->UnregisterAllComponents(); 

				UEditorEngine::CopyPropertiesForUnrelatedObjects(OldActor, NewActor);
				// reset properties/streams
				NewActor->ResetPropertiesForConstruction(); 
				// register native components
				NewActor->RegisterAllComponents(); 

				// 
				// clean up the old actor (unselect it, remove it from the world, etc.)...

				if (OldActor->IsSelected())
				{
					GEditor->SelectActor(OldActor, /*bInSelected =*/false, /*bNotify =*/false);
					bSelectionChanged = true;
				}
				if (GEditor->Layers.IsValid()) // ensure(NULL != GEditor->Layers) ?? While cooking the Layers is NULL.
				{
					GEditor->Layers->DisassociateActorFromLayers(OldActor);
				}

 				World->EditorDestroyActor(OldActor, /*bShouldModifyLevel =*/true);
 				OldToNewInstanceMap.Add(OldActor, NewActor);
			}
			else
			{
				FName OldName(OldObject->GetFName());
				OldObject->Rename(NULL, OldObject->GetOuter(), REN_DoNotDirty | REN_DontCreateRedirectors);
				NewUObject = NewObject<UObject>(OldObject->GetOuter(), NewClass, OldName);
				check(NewUObject != nullptr);

				UEditorEngine::CopyPropertiesForUnrelatedObjects(OldObject, NewUObject);

				if (UAnimInstance* AnimTree = Cast<UAnimInstance>(NewUObject))
				{
					// Initialising the anim instance isn't enough to correctly set up the skeletal mesh again in a
					// paused world, need to initialise the skeletal mesh component that contains the anim instance.
					if (USkeletalMeshComponent* SkelComponent = Cast<USkeletalMeshComponent>(AnimTree->GetOuter()))
					{
						SkelComponent->InitAnim(true);
					}
				}

				OldObject->RemoveFromRoot();
				OldObject->MarkPendingKill();

				OldToNewInstanceMap.Add(OldObject, NewUObject);

				if (bIsComponent)
				{
					UActorComponent* Component = Cast<UActorComponent>(NewUObject);
					AActor* OwningActor = Component->GetOwner();
					if (OwningActor)
					{
						OwningActor->ResetOwnedComponents();

						// Check to see if they have an editor that potentially needs to be refreshed
						if (OwningActor->GetClass()->ClassGeneratedBy)
						{
							PotentialEditorsForRefreshing.AddUnique(OwningActor->GetClass()->ClassGeneratedBy);
						}

						// we need to keep track of actor instances that need 
						// their construction scripts re-ran (since we've just 
						// replaced a component they own)
						OwnersToReconstruct.Add(OwningActor);
					}
				}
			}

			// If this original object came from a blueprint and it was in the selected debug set, change the debugging to the new object.
			if ((CorrespondingBlueprint) && (OldBlueprintDebugObject) && (NewUObject))
			{
				CorrespondingBlueprint->SetObjectBeingDebugged(NewUObject);
			}

			if (bLogConversions)
			{
				UE_LOG(LogBlueprint, Log, TEXT("Converted instance '%s' to '%s'"), *OldObject->GetPathName(), *NewUObject->GetPathName());
			}
		}
	}

	GEditor->OnObjectsReplaced().Remove(OnObjectsReplacedHandle);

	// Now replace any pointers to the old archetypes/instances with pointers to the new one
	TArray<UObject*> SourceObjects;
	TArray<UObject*> DstObjects;

	OldToNewInstanceMap.GenerateKeyArray(SourceObjects);
	OldToNewInstanceMap.GenerateValueArray(DstObjects); // Also look for references in new spawned objects.

	SourceObjects.Append(DstObjects);

	FReplaceReferenceHelper::IncludeCDO(OldClass, NewClass, OldToNewInstanceMap, SourceObjects, OriginalCDO);
	FReplaceReferenceHelper::FindAndReplaceReferences(SourceObjects, ObjectsThatShouldUseOldStuff, ObjectsToReplace, OldToNewInstanceMap, ReinstancedObjectsWeakReferenceMap);

	{ BP_SCOPED_COMPILER_EVENT_STAT(EKismetReinstancerStats_ReplacementConstruction);

		// the process of setting up new replacement actors is split into two 
		// steps (this here, is the second)...
		// 
		// the "finalization" here runs the replacement actor's construction-
		// script and is left until late to account for a scenario where the 
		// construction-script attempts to modify another instance of the 
		// same class... if this were to happen above, in the ObjectsToReplace 
		// loop, then accessing that other instance would cause an assert in 
		// UProperty::ContainerPtrToValuePtrInternal() (which appropriatly 
		// complains that the other instance's type doesn't match because it 
		// hasn't been replaced yet... that's why we wait until after 
		// FArchiveReplaceObjectRef to run construction-scripts).
		for (FActorReplacementHelper& ReplacementActor : ReplacementActors)
		{
			ReplacementActor.Finalize(ObjectRemappingHelper.ReplacedObjects);
		}
	}

	SelectedActors->EndBatchSelectOperation();
	if (bSelectionChanged)
	{
		GEditor->NoteSelectionChange();
	}

	if (GEditor)
	{
		// Refresh any editors for objects that we've updated components for
		for (auto BlueprintAsset : PotentialEditorsForRefreshing)
		{
			FBlueprintEditor* BlueprintEditor = static_cast<FBlueprintEditor*>(FAssetEditorManager::Get().FindEditorForAsset(BlueprintAsset, /*bFocusIfOpen =*/false));
			if (BlueprintEditor)
			{
				BlueprintEditor->RefreshEditors();
			}
		}
	}

	// in the case where we're replacing component instances, we need to make 
	// sure to re-run their owner's construction scripts
	for (AActor* ActorInstance : OwnersToReconstruct)
	{
		ActorInstance->RerunConstructionScripts();
	}
}
	/**
	* Nulls out references to a given object
	*
	* @param InObject - Object to null references to
	*/
	void NullReferencesToObject(UObject* InObject)
	{
		TArray<UObject*> ReplaceableObjects;
		TMap<UObject*, UObject*> ReplacementMap;
		ReplacementMap.Add(InObject, NULL);
		ReplacementMap.GenerateKeyArray(ReplaceableObjects);

		// Find all the properties (and their corresponding objects) that refer to any of the objects to be replaced
		TMap< UObject*, TArray<UProperty*> > ReferencingPropertiesMap;
		for (FObjectIterator ObjIter; ObjIter; ++ObjIter)
		{
			UObject* CurObject = *ObjIter;

			// Find the referencers of the objects to be replaced
			FFindReferencersArchive FindRefsArchive(CurObject, ReplaceableObjects);

			// Inform the object referencing any of the objects to be replaced about the properties that are being forcefully
			// changed, and store both the object doing the referencing as well as the properties that were changed in a map (so that
			// we can correctly call PostEditChange later)
			TMap<UObject*, int32> CurNumReferencesMap;
			TMultiMap<UObject*, UProperty*> CurReferencingPropertiesMMap;
			if (FindRefsArchive.GetReferenceCounts(CurNumReferencesMap, CurReferencingPropertiesMMap) > 0)
			{
				TArray<UProperty*> CurReferencedProperties;
				CurReferencingPropertiesMMap.GenerateValueArray(CurReferencedProperties);
				ReferencingPropertiesMap.Add(CurObject, CurReferencedProperties);
				for (TArray<UProperty*>::TConstIterator RefPropIter(CurReferencedProperties); RefPropIter; ++RefPropIter)
				{
					CurObject->PreEditChange(*RefPropIter);
				}
			}

		}

		// Iterate over the map of referencing objects/changed properties, forcefully replacing the references and then
		// alerting the referencing objects the change has completed via PostEditChange
		int32 NumObjsReplaced = 0;
		for (TMap< UObject*, TArray<UProperty*> >::TConstIterator MapIter(ReferencingPropertiesMap); MapIter; ++MapIter)
		{
			++NumObjsReplaced;

			UObject* CurReplaceObj = MapIter.Key();
			const TArray<UProperty*>& RefPropArray = MapIter.Value();

			FArchiveReplaceObjectRef<UObject> ReplaceAr(CurReplaceObj, ReplacementMap, false, true, false);

			for (TArray<UProperty*>::TConstIterator RefPropIter(RefPropArray); RefPropIter; ++RefPropIter)
			{
				FPropertyChangedEvent PropertyEvent(*RefPropIter);
				CurReplaceObj->PostEditChangeProperty(PropertyEvent);
			}

			if (!CurReplaceObj->HasAnyFlags(RF_Transient) && CurReplaceObj->GetOutermost() != GetTransientPackage())
			{
				if (!CurReplaceObj->RootPackageHasAnyFlags(PKG_CompiledIn))
				{
					CurReplaceObj->MarkPackageDirty();
				}
			}
		}
	}
Example #3
0
void FStatsMemoryDumpCommand::ProcessMemoryOperations( const TMap<int64, FStatPacketArray>& CombinedHistory )
{
	// This is only example code, no fully implemented, may sometimes crash.
	// This code is not optimized. 
	double PreviousSeconds = FPlatformTime::Seconds();
	uint64 NumMemoryOperations = 0;

	// Generate frames
	TArray<int64> Frames;
	CombinedHistory.GenerateKeyArray( Frames );
	Frames.Sort();

	// Raw stats callstack for this stat packet array.
	TMap<FName, FStackState> StackStates;

	// All allocation ordered by the sequence tag.
	// There is an assumption that the sequence tag will not turn-around.
	//TMap<uint32, FAllocationInfo> SequenceAllocationMap;
	TArray<FAllocationInfo> SequenceAllocationArray;

	// Pass 1.
	// Read all stats messages, parse all memory operations and decode callstacks.
	const int64 FirstFrame = 0;
	PreviousSeconds -= NumSecondsBetweenLogs;
	for( int32 FrameIndex = 0; FrameIndex < Frames.Num(); ++FrameIndex )
	{
        {
            const double CurrentSeconds = FPlatformTime::Seconds();
            if( CurrentSeconds > PreviousSeconds + NumSecondsBetweenLogs )
            {
                UE_LOG( LogStats, Warning, TEXT( "Processing frame %i/%i" ), FrameIndex+1, Frames.Num() );
                PreviousSeconds = CurrentSeconds;
            }
        }

		const int64 TargetFrame = Frames[FrameIndex];
		const int64 Diff = TargetFrame - FirstFrame;
		const FStatPacketArray& Frame = CombinedHistory.FindChecked( TargetFrame );

		bool bAtLeastOnePacket = false;
		for( int32 PacketIndex = 0; PacketIndex < Frame.Packets.Num(); PacketIndex++ )
		{
            {
                const double CurrentSeconds = FPlatformTime::Seconds();
                if( CurrentSeconds > PreviousSeconds + NumSecondsBetweenLogs )
                {
                    UE_LOG( LogStats, Log, TEXT( "Processing packet %i/%i" ), PacketIndex, Frame.Packets.Num() );
                    PreviousSeconds = CurrentSeconds;
                    bAtLeastOnePacket = true;
                }
            }

			const FStatPacket& StatPacket = *Frame.Packets[PacketIndex];
			const FName& ThreadFName = StatsThreadStats.Threads.FindChecked( StatPacket.ThreadId );
			const uint32 NewThreadID = ThreadIDtoStatID.FindChecked( StatPacket.ThreadId );

			FStackState* StackState = StackStates.Find( ThreadFName );
			if( !StackState )
			{
				StackState = &StackStates.Add( ThreadFName );
				StackState->Stack.Add( ThreadFName );
				StackState->Current = ThreadFName;
			}

			const FStatMessagesArray& Data = StatPacket.StatMessages;

			int32 LastPct = 0;
			const int32 NumDataElements = Data.Num();
			const int32 OnerPercent = FMath::Max( NumDataElements / 100, 1024 );
			bool bAtLeastOneMessage = false;
			for( int32 Index = 0; Index < NumDataElements; Index++ )
			{
				if( Index % OnerPercent )
				{
					const double CurrentSeconds = FPlatformTime::Seconds();
					if( CurrentSeconds > PreviousSeconds + NumSecondsBetweenLogs )
					{
						const int32 CurrentPct = int32( 100.0*(Index + 1) / NumDataElements );
						UE_LOG( LogStats, Log, TEXT( "Processing %3i%% (%i/%i) stat messages" ), CurrentPct, Index, NumDataElements );
						PreviousSeconds = CurrentSeconds;
						bAtLeastOneMessage = true;
					}
				}

				const FStatMessage& Item = Data[Index];

				const EStatOperation::Type Op = Item.NameAndInfo.GetField<EStatOperation>();
				const FName RawName = Item.NameAndInfo.GetRawName();

				if( Op == EStatOperation::CycleScopeStart || Op == EStatOperation::CycleScopeEnd || Op == EStatOperation::Memory )
				{
					if( Op == EStatOperation::CycleScopeStart )
					{
						StackState->Stack.Add( RawName );
						StackState->Current = RawName;
					}
					else if( Op == EStatOperation::Memory )
					{
						// Experimental code used only to test the implementation.
						// First memory operation is Alloc or Free
						const uint64 EncodedPtr = Item.GetValue_Ptr();
						const bool bIsAlloc = (EncodedPtr & (uint64)EMemoryOperation::Alloc) != 0;
						const bool bIsFree = (EncodedPtr & (uint64)EMemoryOperation::Free) != 0;
						const uint64 Ptr = EncodedPtr & ~(uint64)EMemoryOperation::Mask;
						if( bIsAlloc )
						{
							NumMemoryOperations++;
							// @see FStatsMallocProfilerProxy::TrackAlloc
							// After alloc ptr message there is always alloc size message and the sequence tag.
							Index++;
							const FStatMessage& AllocSizeMessage = Data[Index];
							const int64 AllocSize = AllocSizeMessage.GetValue_int64();

							// Read operation sequence tag.
							Index++;
							const FStatMessage& SequenceTagMessage = Data[Index];
							const uint32 SequenceTag = SequenceTagMessage.GetValue_int64();

							// Create a callstack.
							TArray<FName> StatsBasedCallstack;
							for( const auto& StackName : StackState->Stack )
							{
								StatsBasedCallstack.Add( StackName );
							}

							// Add a new allocation.
							SequenceAllocationArray.Add(
								FAllocationInfo(
								Ptr,
								AllocSize,
								StatsBasedCallstack,
								SequenceTag,
								EMemoryOperation::Alloc,
								StackState->bIsBrokenCallstack
								) );
						}
						else if( bIsFree )
						{
							NumMemoryOperations++;
							// Read operation sequence tag.
							Index++;
							const FStatMessage& SequenceTagMessage = Data[Index];
							const uint32 SequenceTag = SequenceTagMessage.GetValue_int64();

							// Create a callstack.
							/*
							TArray<FName> StatsBasedCallstack;
							for( const auto& RawName : StackState->Stack )
							{
								StatsBasedCallstack.Add( RawName );
							}
							*/

							// Add a new free.
							SequenceAllocationArray.Add(
								FAllocationInfo(
								Ptr,		
								0,
								TArray<FName>()/*StatsBasedCallstack*/,					
								SequenceTag,
								EMemoryOperation::Free,
								StackState->bIsBrokenCallstack
								) );
						}
						else
						{
							UE_LOG( LogStats, Warning, TEXT( "Pointer from a memory operation is invalid" ) );
						}
					}
					else if( Op == EStatOperation::CycleScopeEnd )
					{
						if( StackState->Stack.Num() > 1 )
						{
							const FName ScopeStart = StackState->Stack.Pop();
							const FName ScopeEnd = Item.NameAndInfo.GetRawName();

							check( ScopeStart == ScopeEnd );

							StackState->Current = StackState->Stack.Last();

							// The stack should be ok, but it may be partially broken.
							// This will happen if memory profiling starts in the middle of executing a background thread.
							StackState->bIsBrokenCallstack = false;
						}
						else
						{
							const FName ShortName = Item.NameAndInfo.GetShortName();

							UE_LOG( LogStats, Warning, TEXT( "Broken cycle scope end %s/%s, current %s" ),
									*ThreadFName.ToString(),
									*ShortName.ToString(),
									*StackState->Current.ToString() );

							// The stack is completely broken, only has the thread name and the last cycle scope.
							// Rollback to the thread node.
							StackState->bIsBrokenCallstack = true;
							StackState->Stack.Empty();
							StackState->Stack.Add( ThreadFName );
							StackState->Current = ThreadFName;
						}
					}
				}
			}
			if( bAtLeastOneMessage )
			{
				PreviousSeconds -= NumSecondsBetweenLogs;
			}
		}
		if( bAtLeastOnePacket )
		{
			PreviousSeconds -= NumSecondsBetweenLogs;
		}
	}

	UE_LOG( LogStats, Warning, TEXT( "NumMemoryOperations:   %llu" ), NumMemoryOperations );
	UE_LOG( LogStats, Warning, TEXT( "SequenceAllocationNum: %i" ), SequenceAllocationArray.Num() );

	// Pass 2.
	/*
	TMap<uint32,FAllocationInfo> UniqueSeq;
	TMultiMap<uint32,FAllocationInfo> OriginalAllocs;
	TMultiMap<uint32,FAllocationInfo> BrokenAllocs;
	for( const FAllocationInfo& Alloc : SequenceAllocationArray )
	{
		const FAllocationInfo* Found = UniqueSeq.Find(Alloc.SequenceTag);
		if( !Found )
		{
			UniqueSeq.Add(Alloc.SequenceTag,Alloc);
		}
		else
		{
			OriginalAllocs.Add(Alloc.SequenceTag, *Found);
			BrokenAllocs.Add(Alloc.SequenceTag, Alloc);
		}
	}
	*/

	// Sort all memory operation by the sequence tag, iterate through all operation and generate memory usage.
	SequenceAllocationArray.Sort( TLess<FAllocationInfo>() );

	// Alive allocations.
	TMap<uint64, FAllocationInfo> AllocationMap;
	TMultiMap<uint64, FAllocationInfo> FreeWithoutAllocMap;
	TMultiMap<uint64, FAllocationInfo> DuplicatedAllocMap;
	int32 NumDuplicatedMemoryOperations = 0;
	int32 NumFWAMemoryOperations = 0; // FreeWithoutAlloc

	UE_LOG( LogStats, Warning, TEXT( "Generating memory operations map" ) );
	const int32 NumSequenceAllocations = SequenceAllocationArray.Num();
	const int32 OnePercent = FMath::Max( NumSequenceAllocations / 100, 1024 );
	for( int32 Index = 0; Index < NumSequenceAllocations; Index++ )
	{
		if( Index % OnePercent )
		{
			const double CurrentSeconds = FPlatformTime::Seconds();
			if( CurrentSeconds > PreviousSeconds + NumSecondsBetweenLogs )
			{
				const int32 CurrentPct = int32( 100.0*(Index + 1) / NumSequenceAllocations );
				UE_LOG( LogStats, Log, TEXT( "Processing allocations %3i%% (%10i/%10i)" ), CurrentPct, Index + 1, NumSequenceAllocations );
				PreviousSeconds = CurrentSeconds;
			}
		}

		const FAllocationInfo& Alloc = SequenceAllocationArray[Index];
		const EMemoryOperation MemOp = Alloc.Op;
		const uint64 Ptr = Alloc.Ptr;
		const int64 Size = Alloc.Size;
		const uint32 SequenceTag = Alloc.SequenceTag;

		if( MemOp == EMemoryOperation::Alloc )
		{
			const FAllocationInfo* Found = AllocationMap.Find( Ptr );

			if( !Found )
			{
				AllocationMap.Add( Ptr, Alloc );
			}
			else
			{
				const FAllocationInfo* FoundAndFreed = FreeWithoutAllocMap.Find( Found->Ptr );
				const FAllocationInfo* FoundAndAllocated = FreeWithoutAllocMap.Find( Alloc.Ptr );

#if	_DEBUG
				if( FoundAndFreed )
				{
					const FString FoundAndFreedCallstack = GetCallstack( FoundAndFreed->EncodedCallstack );
				}

				if( FoundAndAllocated )
				{
					const FString FoundAndAllocatedCallstack = GetCallstack( FoundAndAllocated->EncodedCallstack );
				}

				NumDuplicatedMemoryOperations++;


				const FString FoundCallstack = GetCallstack( Found->EncodedCallstack );
				const FString AllocCallstack = GetCallstack( Alloc.EncodedCallstack );
#endif // _DEBUG

				// Replace pointer.
				AllocationMap.Add( Ptr, Alloc );
				// Store the old pointer.
				DuplicatedAllocMap.Add( Ptr, *Found );
			}
		}
		else if( MemOp == EMemoryOperation::Free )
		{
			const FAllocationInfo* Found = AllocationMap.Find( Ptr );
			if( Found )
			{
				const bool bIsValid = Alloc.SequenceTag > Found->SequenceTag;
				if( !bIsValid )
				{
					UE_LOG( LogStats, Warning, TEXT( "InvalidFree Ptr: %llu, Seq: %i/%i" ), Ptr, SequenceTag, Found->SequenceTag );
				}
				AllocationMap.Remove( Ptr );
			}
			else
			{
				FreeWithoutAllocMap.Add( Ptr, Alloc );
				NumFWAMemoryOperations++;
			}
		}
	}

	UE_LOG( LogStats, Warning, TEXT( "NumDuplicatedMemoryOperations: %i" ), NumDuplicatedMemoryOperations );
	UE_LOG( LogStats, Warning, TEXT( "NumFWAMemoryOperations:        %i" ), NumFWAMemoryOperations );

	// Dump problematic allocations
	DuplicatedAllocMap.ValueSort( FAllocationInfoGreater() );
	//FreeWithoutAllocMap

	uint64 TotalDuplicatedMemory = 0;
	for( const auto& It : DuplicatedAllocMap )
	{
		const FAllocationInfo& Alloc = It.Value;
		TotalDuplicatedMemory += Alloc.Size;
	}

	UE_LOG( LogStats, Warning, TEXT( "Dumping duplicated alloc map" ) );
	const float MaxPctDisplayed = 0.80f;
	uint64 DisplayedSoFar = 0;
	for( const auto& It : DuplicatedAllocMap )
	{
		const FAllocationInfo& Alloc = It.Value;
		const FString AllocCallstack = GetCallstack( Alloc.EncodedCallstack );
		UE_LOG( LogStats, Log, TEXT( "%lli (%.2f MB) %s" ), Alloc.Size, Alloc.Size / 1024.0f / 1024.0f, *AllocCallstack );

		DisplayedSoFar += Alloc.Size;

		const float CurrentPct = (float)DisplayedSoFar / (float)TotalDuplicatedMemory;
		if( CurrentPct > MaxPctDisplayed )
		{
			break;
		}
	}

	GenerateMemoryUsageReport( AllocationMap );
}
Example #4
0
void FStatsMemoryDumpCommand::InternalRun()
{
	FParse::Value( FCommandLine::Get(), TEXT( "-INFILE=" ), SourceFilepath );

	const int64 Size = IFileManager::Get().FileSize( *SourceFilepath );
	if( Size < 4 )
	{
		UE_LOG( LogStats, Error, TEXT( "Could not open: %s" ), *SourceFilepath );
		return;
	}
	TAutoPtr<FArchive> FileReader( IFileManager::Get().CreateFileReader( *SourceFilepath ) );
	if( !FileReader )
	{
		UE_LOG( LogStats, Error, TEXT( "Could not open: %s" ), *SourceFilepath );
		return;
	}

	if( !Stream.ReadHeader( *FileReader ) )
	{
		UE_LOG( LogStats, Error, TEXT( "Could not open, bad magic: %s" ), *SourceFilepath );
		return;
	}

	UE_LOG( LogStats, Warning, TEXT( "Reading a raw stats file for memory profiling: %s" ), *SourceFilepath );

	const bool bIsFinalized = Stream.Header.IsFinalized();
	check( bIsFinalized );
	check( Stream.Header.Version == EStatMagicWithHeader::VERSION_5 );
	StatsThreadStats.MarkAsLoaded();

	TArray<FStatMessage> Messages;
	if( Stream.Header.bRawStatsFile )
	{
		FScopeLogTime SLT( TEXT( "FStatsMemoryDumpCommand::InternalRun" ), nullptr, FScopeLogTime::ScopeLog_Seconds );

		// Read metadata.
		TArray<FStatMessage> MetadataMessages;
		Stream.ReadFNamesAndMetadataMessages( *FileReader, MetadataMessages );
		StatsThreadStats.ProcessMetaDataOnly( MetadataMessages );

		// Find all UObject metadata messages.
		for( const auto& Meta : MetadataMessages )
		{
			FName LongName = Meta.NameAndInfo.GetRawName();
			const FString Desc = FStatNameAndInfo::GetShortNameFrom( LongName ).GetPlainNameString();
			const bool bContainsUObject = Desc.Contains( TEXT( "//" ) );
			if( bContainsUObject )
			{
				UObjectNames.Add( LongName );
			}
		}

		const int64 CurrentFilePos = FileReader->Tell();

		// Update profiler's metadata.
		CreateThreadsMapping();

		// Read frames offsets.
		Stream.ReadFramesOffsets( *FileReader );

		// Buffer used to store the compressed and decompressed data.
		TArray<uint8> SrcArray;
		TArray<uint8> DestArray;
		const bool bHasCompressedData = Stream.Header.HasCompressedData();
		check( bHasCompressedData );

		TMap<int64, FStatPacketArray> CombinedHistory;
		int64 TotalDataSize = 0;
		int64 TotalStatMessagesNum = 0;
		int64 MaximumPacketSize = 0;
		int64 TotalPacketsNum = 0;
		// Read all packets sequentially, force by the memory profiler which is now a part of the raw stats.
		// !!CAUTION!! Frame number in the raw stats is pointless, because it is time based, not frame based.
		// Background threads usually execute time consuming operations, so the frame number won't be valid.
		// Needs to be combined by the thread and the time, not by the frame number.
		{
			// Display log information once per 5 seconds to avoid spamming.
			double PreviousSeconds = FPlatformTime::Seconds();
			const int64 FrameOffset0 = Stream.FramesInfo[0].FrameFileOffset;
			FileReader->Seek( FrameOffset0 );

			const int64 FileSize = FileReader->TotalSize();

			while( FileReader->Tell() < FileSize )
			{
				// Read the compressed data.
				FCompressedStatsData UncompressedData( SrcArray, DestArray );
				*FileReader << UncompressedData;
				if( UncompressedData.HasReachedEndOfCompressedData() )
				{
					break;
				}

				FMemoryReader MemoryReader( DestArray, true );

				FStatPacket* StatPacket = new FStatPacket();
				Stream.ReadStatPacket( MemoryReader, *StatPacket );

				const int64 StatPacketFrameNum = StatPacket->Frame;
				FStatPacketArray& Frame = CombinedHistory.FindOrAdd( StatPacketFrameNum );

				// Check if we need to combine packets from the same thread.
				FStatPacket** CombinedPacket = Frame.Packets.FindByPredicate( [&]( FStatPacket* Item ) -> bool
				{
					return Item->ThreadId == StatPacket->ThreadId;
				} );

				const int64 PacketSize = StatPacket->StatMessages.GetAllocatedSize();
				TotalStatMessagesNum += StatPacket->StatMessages.Num();

				if( CombinedPacket )
				{
					TotalDataSize -= (*CombinedPacket)->StatMessages.GetAllocatedSize();
					(*CombinedPacket)->StatMessages += StatPacket->StatMessages;
					TotalDataSize += (*CombinedPacket)->StatMessages.GetAllocatedSize();

					delete StatPacket;
				}
				else
				{
					Frame.Packets.Add( StatPacket );
					TotalDataSize += PacketSize;
				}

				const double CurrentSeconds = FPlatformTime::Seconds();
				if( CurrentSeconds > PreviousSeconds + NumSecondsBetweenLogs )
				{
					const int32 PctPos = int32( 100.0*FileReader->Tell() / FileSize );
					UE_LOG( LogStats, Log, TEXT( "%3i%% %10llu (%.1f MB) read messages, last read frame %4i" ), PctPos, TotalStatMessagesNum, TotalDataSize / 1024.0f / 1024.0f, StatPacketFrameNum );
					PreviousSeconds = CurrentSeconds;
				}
			
				MaximumPacketSize = FMath::Max( MaximumPacketSize, PacketSize );			
				TotalPacketsNum++;
			}
		}

		// Dump frame stats
		for( const auto& It : CombinedHistory )
		{
			const int64 FrameNum = It.Key;
			int64 FramePacketsSize = 0;
			int64 FrameStatMessages = 0;
			int64 FramePackets = It.Value.Packets.Num(); // Threads
			for( const auto& It2 : It.Value.Packets )
			{
				FramePacketsSize += It2->StatMessages.GetAllocatedSize();
				FrameStatMessages += It2->StatMessages.Num();
			}

			UE_LOG( LogStats, Warning, TEXT( "Frame: %10llu/%3lli Size: %.1f MB / %10lli" ), 
					FrameNum, 
					FramePackets, 
					FramePacketsSize / 1024.0f / 1024.0f,
					FrameStatMessages );
		}

		UE_LOG( LogStats, Warning, TEXT( "TotalPacketSize: %.1f MB, Max: %1f MB" ),
				TotalDataSize / 1024.0f / 1024.0f,
				MaximumPacketSize / 1024.0f / 1024.0f );

		TArray<int64> Frames;
		CombinedHistory.GenerateKeyArray( Frames );
		Frames.Sort();
		const int64 MiddleFrame = Frames[Frames.Num() / 2];

		ProcessMemoryOperations( CombinedHistory );
	}
}
void FRawProfilerSession::PrepareLoading()
{
	SCOPE_LOG_TIME_FUNC();

	const FString Filepath = DataFilepath + FStatConstants::StatsFileRawExtension;
	const int64 Size = IFileManager::Get().FileSize( *Filepath );
	if( Size < 4 )
	{
		UE_LOG( LogStats, Error, TEXT( "Could not open: %s" ), *Filepath );
		return;
	}
	TAutoPtr<FArchive> FileReader( IFileManager::Get().CreateFileReader( *Filepath ) );
	if( !FileReader )
	{
		UE_LOG( LogStats, Error, TEXT( "Could not open: %s" ), *Filepath );
		return;
	}

	if( !Stream.ReadHeader( *FileReader ) )
	{
		UE_LOG( LogStats, Error, TEXT( "Could not open, bad magic: %s" ), *Filepath );
		return;
	}

	const bool bIsFinalized = Stream.Header.IsFinalized();
	check( bIsFinalized );
	check( Stream.Header.Version == EStatMagicWithHeader::VERSION_5 );
	StatsThreadStats.MarkAsLoaded();

	TArray<FStatMessage> Messages;
	if( Stream.Header.bRawStatsFile )
	{
		// Read metadata.
		TArray<FStatMessage> MetadataMessages;
		Stream.ReadFNamesAndMetadataMessages( *FileReader, MetadataMessages );
		StatsThreadStats.ProcessMetaDataOnly( MetadataMessages );

		const FName F00245 = FName(245, 245, 0);
		
		const FName F11602 = FName(11602, 11602, 0);
		const FName F06394 = FName(6394, 6394, 0);

		const int64 CurrentFilePos = FileReader->Tell();

		// Update profiler's metadata.
		StatMetaData->UpdateFromStatsState( StatsThreadStats );
		const uint32 GameThreadID = GetMetaData()->GetGameThreadID();

		// Read frames offsets.
		Stream.ReadFramesOffsets( *FileReader );

		// Buffer used to store the compressed and decompressed data.
		TArray<uint8> SrcArray;
		TArray<uint8> DestArray;
		const bool bHasCompressedData = Stream.Header.HasCompressedData();
		check(bHasCompressedData);

		TMap<int64, FStatPacketArray> CombinedHistory;
		int64 TotalPacketSize = 0;
		int64 MaximumPacketSize = 0;
		// Read all packets sequentially, force by the memory profiler which is now a part of the raw stats.
		// !!CAUTION!! Frame number in the raw stats is pointless, because it is time based, not frame based.
		// Background threads usually execute time consuming operations, so the frame number won't be valid.
		// Needs to be combined by the thread and the time, not by the frame number.
		{
			int64 FrameOffset0 = Stream.FramesInfo[0].FrameFileOffset;
			FileReader->Seek( FrameOffset0 );

			const int64 FileSize = FileReader->TotalSize();

			while( FileReader->Tell() < FileSize )
			{
				// Read the compressed data.
				FCompressedStatsData UncompressedData( SrcArray, DestArray );
				*FileReader << UncompressedData;
				if( UncompressedData.HasReachedEndOfCompressedData() )
				{
					break;
				}

				FMemoryReader MemoryReader( DestArray, true );

				FStatPacket* StatPacket = new FStatPacket();
				Stream.ReadStatPacket( MemoryReader, *StatPacket );
				
				const int64 FrameNum = StatPacket->Frame;
				FStatPacketArray& Frame = CombinedHistory.FindOrAdd(FrameNum);
			
				// Check if we need to combine packets from the same thread.
				FStatPacket** CombinedPacket = Frame.Packets.FindByPredicate([&](FStatPacket* Item) -> bool
				{
					return Item->ThreadId == StatPacket->ThreadId;
				});
				
				if( CombinedPacket )
				{
					(*CombinedPacket)->StatMessages += StatPacket->StatMessages;
				}
				else
				{
					Frame.Packets.Add(StatPacket);
				}

				const int64 CurrentPos = FileReader->Tell();
				const int32 PctPos = int32(100.0f*CurrentPos/FileSize);

				UE_LOG( LogStats, Log, TEXT( "%3i Processing FStatPacket: Frame %5i for thread %5i with %6i messages (%.1f MB)" ), 
					PctPos, 
					StatPacket->Frame, 
					StatPacket->ThreadId, 
					StatPacket->StatMessages.Num(), 
					StatPacket->StatMessages.GetAllocatedSize()/1024.0f/1024.0f );

				const int64 PacketSize = StatPacket->StatMessages.GetAllocatedSize();
				TotalPacketSize += PacketSize;
				MaximumPacketSize = FMath::Max( MaximumPacketSize, PacketSize );
			}
		}

		UE_LOG( LogStats, Log, TEXT( "TotalPacketSize: %.1f MB, Max: %1f MB" ), 
			TotalPacketSize/1024.0f/1024.0f, 
			MaximumPacketSize/1024.0f/1024.0f );

		TArray<int64> Frames;
		CombinedHistory.GenerateKeyArray(Frames);
		Frames.Sort();
		const int64 MiddleFrame = Frames[Frames.Num()/2];


		// Remove all frames without the game thread messages.
		for (int32 FrameIndex = 0; FrameIndex < Frames.Num(); ++FrameIndex)
		{
			const int64 TargetFrame = Frames[FrameIndex];
			const FStatPacketArray& Frame = CombinedHistory.FindChecked( TargetFrame );

			const double GameThreadTimeMS = GetMetaData()->ConvertCyclesToMS( GetFastThreadFrameTimeInternal( Frame, EThreadType::Game ) );

			if (GameThreadTimeMS == 0.0f)
			{
				CombinedHistory.Remove( TargetFrame );
				Frames.RemoveAt( FrameIndex );
				FrameIndex--;
			}
		}
		
	
		StatMetaData->SecondsPerCycle = GetSecondsPerCycle( CombinedHistory.FindChecked(MiddleFrame) );
		check( StatMetaData->GetSecondsPerCycle() > 0.0 );

		//const int32 FirstGameThreadFrame = FindFirstFrameWithGameThread( CombinedHistory, Frames );

		// Prepare profiler frame.
		{
			SCOPE_LOG_TIME( TEXT( "Preparing profiler frames" ), nullptr );

			// Prepare profiler frames.
			double ElapsedTimeMS = 0;

			for( int32 FrameIndex = 0; FrameIndex < Frames.Num(); ++FrameIndex )
			{
				const int64 TargetFrame = Frames[FrameIndex];
				const FStatPacketArray& Frame = CombinedHistory.FindChecked(TargetFrame);

				const double GameThreadTimeMS = GetMetaData()->ConvertCyclesToMS( GetFastThreadFrameTimeInternal(Frame,EThreadType::Game) );

				if( GameThreadTimeMS == 0.0f )
				{
					continue;
				}

				const double RenderThreadTimeMS = GetMetaData()->ConvertCyclesToMS( GetFastThreadFrameTimeInternal(Frame,EThreadType::Renderer) );

				// Update mini-view, convert from cycles to ms.
				TMap<uint32, float> ThreadTimesMS;
				ThreadTimesMS.Add( GameThreadID, GameThreadTimeMS );
				ThreadTimesMS.Add( GetMetaData()->GetRenderThreadID()[0], RenderThreadTimeMS );

				// Pass the reference to the stats' metadata.
				OnAddThreadTime.ExecuteIfBound( FrameIndex, ThreadTimesMS, StatMetaData );

				// Create a new profiler frame and add it to the stream.
				ElapsedTimeMS += GameThreadTimeMS;
				FProfilerFrame* ProfilerFrame = new FProfilerFrame( TargetFrame, GameThreadTimeMS, ElapsedTimeMS );
				ProfilerFrame->ThreadTimesMS = ThreadTimesMS;
				ProfilerStream.AddProfilerFrame( TargetFrame, ProfilerFrame );
			}
		}
	
		// Process the raw stats data.
		{
			SCOPE_LOG_TIME( TEXT( "Processing the raw stats" ), nullptr );

			double CycleCounterAdjustmentMS = 0.0f;

			// Read the raw stats messages.
			for( int32 FrameIndex = 0; FrameIndex < Frames.Num()-1; ++FrameIndex )
			{
				const int64 TargetFrame = Frames[FrameIndex];
				const FStatPacketArray& Frame = CombinedHistory.FindChecked(TargetFrame);

				FProfilerFrame* ProfilerFrame = ProfilerStream.GetProfilerFrame( FrameIndex );

				UE_CLOG( FrameIndex % 8 == 0, LogStats, Log, TEXT( "Processing raw stats frame: %4i/%4i" ), FrameIndex, Frames.Num() );

				ProcessStatPacketArray( Frame, *ProfilerFrame, FrameIndex ); // or ProfilerFrame->TargetFrame

				// Find the first cycle counter for the game thread.
				if( CycleCounterAdjustmentMS == 0.0f )
				{
					CycleCounterAdjustmentMS = ProfilerFrame->Root->CycleCounterStartTimeMS;
				}

				// Update thread time and mark profiler frame as valid and ready for use.
				ProfilerFrame->MarkAsValid();
			}

			// Adjust all profiler frames.
			ProfilerStream.AdjustCycleCounters( CycleCounterAdjustmentMS );
		}
	}

	const int64 AllocatedSize = ProfilerStream.GetAllocatedSize();

	// We have the whole metadata and basic information about the raw stats file, start ticking the profiler session.
	//OnTickHandle = FTicker::GetCoreTicker().AddTicker( OnTick, 0.25f );

#if	0
	if( SessionType == EProfilerSessionTypes::OfflineRaw )
	{
		// Broadcast that a capture file has been fully processed.
		OnCaptureFileProcessed.ExecuteIfBound( GetInstanceID() );
	}
#endif // 0
}