USceneCapturer::USceneCapturer()
: ImageWrapperModule( FModuleManager::LoadModuleChecked<IImageWrapperModule>( FName( "ImageWrapper" ) ) )
, bIsTicking( false )
, CapturePlayerController( NULL )
, CaptureGameMode( NULL )
, hAngIncrement( FStereoPanoramaManager::HorizontalAngularIncrement->GetFloat() )
, vAngIncrement( FStereoPanoramaManager::VerticalAngularIncrement->GetFloat() )
, eyeSeparation( FStereoPanoramaManager::EyeSeparation->GetFloat() )
, NumberOfHorizontalSteps( ( int32 )( 360.0f / hAngIncrement ) )
, NumberOfVerticalSteps( ( int32 )( 180.0f / vAngIncrement ) + 1 ) /* Need an extra b/c we only grab half of the top & bottom slices */
, SphericalAtlasWidth( FStereoPanoramaManager::StepCaptureWidth->GetInt() )
, SphericalAtlasHeight( SphericalAtlasWidth / 2)
, bForceAlpha( FStereoPanoramaManager::ForceAlpha->GetInt() != 0 )
, bEnableBilerp( FStereoPanoramaManager::EnableBilerp->GetInt() != 0 )
, SSMethod( FMath::Clamp<int32>(FStereoPanoramaManager::SuperSamplingMethod->GetInt(), 0, ARRAY_COUNT(g_ssPatterns)) )
, bOverrideInitialYaw( FStereoPanoramaManager::ShouldOverrideInitialYaw->GetInt() != 0 )
, ForcedInitialYaw( FRotator::ClampAxis(FStereoPanoramaManager::ForcedInitialYaw->GetFloat()) )
, OutputDir( FStereoPanoramaManager::OutputDir->GetString().IsEmpty() ? FPaths::GameSavedDir() / TEXT("StereoPanorama") : FStereoPanoramaManager::OutputDir->GetString() )
, dbgDisableOffsetRotation( FStereoPanoramaManager::FadeStereoToZeroAtSides->GetInt() != 0 )
{
//NOTE: ikrimae: Keeping the old sampling mechanism just until we're sure the new way is always better
dbgMatchCaptureSliceFovToAtlasSliceFov = false;
float captureHFov = 0, captureVFov = 0;
if (dbgMatchCaptureSliceFovToAtlasSliceFov)
{
//Slicing Technique 1: Match Capture Slice StripWidth to match the pixel dimensions of AtlasWidth/NumHorizSteps & s.t. stripwidth/stripheight fovs match hAngIncr & vAngIncr
// Legacy technique but allows setting the strip width to match atlas slice width
// Pretty wasteful and will break if CaptureHFov & hangIncr/vAngIncr diverge greatly b/c resultant texture will exceed GPU bounds
// StripHeight is computed based on solving CpxV = CpxH * SpxV / SpxH
// CpxV = CV * SpxV / SV
// captureVfov = 2 * atan( tan(captureHfov / 2) * (SpxV / SpxH) )
sliceHFov = hAngIncrement;
sliceVFov = vAngIncrement;
//TODO: ikrimae: Also do a quick test to see if there are issues with setting fov to something really small ( < 1 degree)
// And it does. Current noted issues: screen space effects like SSAO, AA, SSR are all off
// local eyeadaptation also causes problems. Should probably turn off all PostProcess effects
// small fovs cause floating point errors in the sampling function (probably a bug b/c no thought put towards that)
captureHFov = FStereoPanoramaManager::CaptureHorizontalFOV->GetFloat();
ensure(captureHFov >= hAngIncrement);
//TODO: ikrimae: In hindsight, there's no reason that strip size should be this at all. Just select a square FOV larger than hAngIncr & vAngIncr
// and then sample the resulting plane accordingly. Remember when updating to this to recheck the math in resample function. Might
// have made assumptions about capture slice dimensions matching the sample strips
StripWidth = SphericalAtlasWidth / NumberOfHorizontalSteps;
//The scenecapture cube won't allow horizontal & vertical fov to not match the aspect ratio so we have to compute the right dimensions here for square pixels
StripHeight = StripWidth * FMath::Tan(FMath::DegreesToRadians(vAngIncrement / 2.0f)) / FMath::Tan(FMath::DegreesToRadians(hAngIncrement / 2.0f));
const FVector2D slicePlaneDim = FVector2D(
2.0f * FMath::Tan(FMath::DegreesToRadians(hAngIncrement) / 2.0f),
2.0f * FMath::Tan(FMath::DegreesToRadians(vAngIncrement) / 2.0f));
const float capturePlaneWidth = 2.0f * FMath::Tan(FMath::DegreesToRadians(captureHFov) / 2.0f);
//TODO: ikrimae: This is just to let the rest of the existing code work. Sampling rate of the slice can be whatever.
// Ex: To match the highest sampling frequency of the spherical atlas, it should match the area of differential patch
// at ray direction of pixel(0,1) in the atlas
//Need stripwidth/slicePlaneDim.X = capturewidth / capturePlaneDim.X
CaptureWidth = capturePlaneWidth * StripWidth / slicePlaneDim.X;
CaptureHeight = CaptureWidth * StripHeight / StripWidth;
captureVFov = FMath::RadiansToDegrees(2 * FMath::Atan(FMath::Tan(FMath::DegreesToRadians(captureHFov / 2.0f)) * CaptureHeight / CaptureWidth));
//float dbgCapturePlaneDimY = 2.0f * FMath::Tan(FMath::DegreesToRadians(captureVFov) / 2.0f);
//float dbgCaptureHeight = dbgCapturePlaneDimY * StripHeight / slicePlaneDim.Y;
}
else
{
//Slicing Technique 2: Each slice is a determined square FOV at a configured preset resolution.
// Strip Width/Strip Height is determined based on hAngIncrement & vAngIncrement
// Just make sure pixels/captureHFov >= pixels/hAngIncr && pixels/vAngIncr
captureVFov = captureHFov = FStereoPanoramaManager::CaptureHorizontalFOV->GetFloat();
sliceVFov = sliceHFov = captureHFov;
ensure(captureHFov >= FMath::Max(hAngIncrement, vAngIncrement));
//TODO: ikrimae: Re-do for floating point accuracy
const FVector2D slicePlaneDim = FVector2D(
2.0f * FMath::Tan(FMath::DegreesToRadians(hAngIncrement) / 2.0f),
2.0f * FMath::Tan(FMath::DegreesToRadians(vAngIncrement) / 2.0f));
const FVector2D capturePlaneDim = FVector2D(
2.0f * FMath::Tan(FMath::DegreesToRadians(captureHFov) / 2.0f),
2.0f * FMath::Tan(FMath::DegreesToRadians(captureVFov) / 2.0f));
CaptureHeight = CaptureWidth = FStereoPanoramaManager::CaptureSlicePixelWidth->GetInt();
StripWidth = CaptureWidth * slicePlaneDim.X / capturePlaneDim.X;
StripHeight = CaptureHeight * slicePlaneDim.Y / capturePlaneDim.Y;
//TODO: ikrimae: Come back and check for the actual right sampling rate
check(StripWidth >= (SphericalAtlasWidth / NumberOfHorizontalSteps) &&
StripHeight >= (SphericalAtlasHeight / NumberOfVerticalSteps));
//Ensure Width/Height is always even
StripWidth += StripWidth & 1;
StripHeight += StripHeight & 1;
}
UnprojectedAtlasWidth = NumberOfHorizontalSteps * StripWidth;
UnprojectedAtlasHeight = NumberOfVerticalSteps * StripHeight;
//NOTE: ikrimae: Ensure that the main gameview is > CaptureWidth x CaptureHeight. Bug in UE4 that won't re-alloc scene render targets to the correct size
// when the scenecapture component > current window render target. https://answers.unrealengine.com/questions/80531/scene-capture-2d-max-resolution.html
//TODO: ikrimae: Ensure that r.SceneRenderTargetResizeMethod=2
FSystemResolution::RequestResolutionChange(CaptureWidth, CaptureHeight, EWindowMode::Windowed);
for( int CaptureIndex = 0; CaptureIndex < FStereoPanoramaManager::ConcurrentCaptures->GetInt(); CaptureIndex++ )
{
FString LeftCounter = FString::Printf( TEXT( "LeftEyeCaptureComponent_%04d" ), CaptureIndex );
USceneCaptureComponent2D* LeftEyeCaptureComponent = CreateDefaultSubobject<USceneCaptureComponent2D>( *LeftCounter );
InitCaptureComponent( LeftEyeCaptureComponent, captureHFov, captureVFov, EStereoscopicPass::eSSP_LEFT_EYE );
LeftEyeCaptureComponents.Add( LeftEyeCaptureComponent );
FString RightCounter = FString::Printf( TEXT( "RightEyeCaptureComponent_%04d" ), CaptureIndex );
USceneCaptureComponent2D* RightEyeCaptureComponent = CreateDefaultSubobject<USceneCaptureComponent2D>( *RightCounter );
InitCaptureComponent( RightEyeCaptureComponent, captureHFov, captureVFov, EStereoscopicPass::eSSP_RIGHT_EYE );
RightEyeCaptureComponents.Add( RightEyeCaptureComponent );
}
CurrentStep = 0;
TotalSteps = 0;
FrameDescriptors = TEXT( "FrameNumber, GameClock, TimeTaken(s)" LINE_TERMINATOR );
CaptureStep = ECaptureStep::Reset;
}
/**
Attach a GTCaptureComponent to a pawn
*/
UGTCaptureComponent* UGTCaptureComponent::Create(APawn* InPawn, TArray<FString> Modes)
{
UWorld* World = FUE4CVServer::Get().GetGameWorld();
UGTCaptureComponent* GTCapturer = NewObject<UGTCaptureComponent>();
GTCapturer->bIsActive = true;
// check(GTCapturer->IsComponentTickEnabled() == true);
GTCapturer->Pawn = InPawn; // This GTCapturer should depend on the Pawn and be released together with the Pawn.
// This snippet is from Engine/Source/Runtime/Engine/Private/Components/SceneComponent.cpp, AttachTo
FAttachmentTransformRules AttachmentRules(EAttachmentRule::KeepRelative, false);
ConvertAttachLocation(EAttachLocation::KeepRelativeOffset, AttachmentRules.LocationRule, AttachmentRules.RotationRule, AttachmentRules.ScaleRule);
GTCapturer->AttachToComponent(InPawn->GetRootComponent(), AttachmentRules);
// GTCapturer->AddToRoot();
GTCapturer->RegisterComponentWithWorld(World);
GTCapturer->SetTickableWhenPaused(true);
for (FString Mode : Modes)
{
// DEPRECATED_FORGAME(4.6, "CaptureComponent2D should not be accessed directly, please use GetCaptureComponent2D() function instead. CaptureComponent2D will soon be private and your code will not compile.")
USceneCaptureComponent2D* CaptureComponent = NewObject<USceneCaptureComponent2D>();
CaptureComponent->bIsActive = false; // Disable it by default for performance consideration
GTCapturer->CaptureComponents.Add(Mode, CaptureComponent);
// CaptureComponent needs to be attached to somewhere immediately, otherwise it will be gc-ed
CaptureComponent->AttachToComponent(GTCapturer, AttachmentRules);
InitCaptureComponent(CaptureComponent);
UMaterial* Material = GetMaterial(Mode);
if (Mode == "lit") // For rendered images
{
// FViewMode::Lit(CaptureComponent->ShowFlags);
CaptureComponent->TextureTarget->TargetGamma = GEngine->GetDisplayGamma();
// float DisplayGamma = SceneViewport->GetDisplayGamma();
}
else if (Mode == "default")
{
continue;
}
else // for ground truth
{
CaptureComponent->TextureTarget->TargetGamma = 1;
if (Mode == "object_mask") // For object mask
{
// FViewMode::Lit(CaptureComponent->ShowFlags);
FViewMode::VertexColor(CaptureComponent->ShowFlags);
}
else if (Mode == "wireframe") // For object mask
{
FViewMode::Wireframe(CaptureComponent->ShowFlags);
}
else
{
check(Material);
// GEngine->GetDisplayGamma(), the default gamma is 2.2
// CaptureComponent->TextureTarget->TargetGamma = 2.2;
FViewMode::PostProcess(CaptureComponent->ShowFlags);
CaptureComponent->PostProcessSettings.AddBlendable(Material, 1);
// Instead of switching post-process materials, we create several SceneCaptureComponent, so that we can capture different GT within the same frame.
}
}
}
return GTCapturer;
}
