USceneCapturer::USceneCapturer() : ImageWrapperModule( FModuleManager::LoadModuleChecked<IImageWrapperModule>( FName( "ImageWrapper" ) ) ) , bIsTicking( false ) , CapturePlayerController( NULL ) , CaptureGameMode( NULL ) , hAngIncrement( FStereoPanoramaManager::HorizontalAngularIncrement->GetFloat() ) , vAngIncrement( FStereoPanoramaManager::VerticalAngularIncrement->GetFloat() ) , eyeSeparation( FStereoPanoramaManager::EyeSeparation->GetFloat() ) , NumberOfHorizontalSteps( ( int32 )( 360.0f / hAngIncrement ) ) , NumberOfVerticalSteps( ( int32 )( 180.0f / vAngIncrement ) + 1 ) /* Need an extra b/c we only grab half of the top & bottom slices */ , SphericalAtlasWidth( FStereoPanoramaManager::StepCaptureWidth->GetInt() ) , SphericalAtlasHeight( SphericalAtlasWidth / 2) , bForceAlpha( FStereoPanoramaManager::ForceAlpha->GetInt() != 0 ) , bEnableBilerp( FStereoPanoramaManager::EnableBilerp->GetInt() != 0 ) , SSMethod( FMath::Clamp<int32>(FStereoPanoramaManager::SuperSamplingMethod->GetInt(), 0, ARRAY_COUNT(g_ssPatterns)) ) , bOverrideInitialYaw( FStereoPanoramaManager::ShouldOverrideInitialYaw->GetInt() != 0 ) , ForcedInitialYaw( FRotator::ClampAxis(FStereoPanoramaManager::ForcedInitialYaw->GetFloat()) ) , OutputDir( FStereoPanoramaManager::OutputDir->GetString().IsEmpty() ? FPaths::GameSavedDir() / TEXT("StereoPanorama") : FStereoPanoramaManager::OutputDir->GetString() ) , dbgDisableOffsetRotation( FStereoPanoramaManager::FadeStereoToZeroAtSides->GetInt() != 0 ) { //NOTE: ikrimae: Keeping the old sampling mechanism just until we're sure the new way is always better dbgMatchCaptureSliceFovToAtlasSliceFov = false; float captureHFov = 0, captureVFov = 0; if (dbgMatchCaptureSliceFovToAtlasSliceFov) { //Slicing Technique 1: Match Capture Slice StripWidth to match the pixel dimensions of AtlasWidth/NumHorizSteps & s.t. stripwidth/stripheight fovs match hAngIncr & vAngIncr // Legacy technique but allows setting the strip width to match atlas slice width // Pretty wasteful and will break if CaptureHFov & hangIncr/vAngIncr diverge greatly b/c resultant texture will exceed GPU bounds // StripHeight is computed based on solving CpxV = CpxH * SpxV / SpxH // CpxV = CV * SpxV / SV // captureVfov = 2 * atan( tan(captureHfov / 2) * (SpxV / SpxH) ) sliceHFov = hAngIncrement; sliceVFov = vAngIncrement; //TODO: ikrimae: Also do a quick test to see if there are issues with setting fov to something really small ( < 1 degree) // And it does. Current noted issues: screen space effects like SSAO, AA, SSR are all off // local eyeadaptation also causes problems. Should probably turn off all PostProcess effects // small fovs cause floating point errors in the sampling function (probably a bug b/c no thought put towards that) captureHFov = FStereoPanoramaManager::CaptureHorizontalFOV->GetFloat(); ensure(captureHFov >= hAngIncrement); //TODO: ikrimae: In hindsight, there's no reason that strip size should be this at all. Just select a square FOV larger than hAngIncr & vAngIncr // and then sample the resulting plane accordingly. Remember when updating to this to recheck the math in resample function. Might // have made assumptions about capture slice dimensions matching the sample strips StripWidth = SphericalAtlasWidth / NumberOfHorizontalSteps; //The scenecapture cube won't allow horizontal & vertical fov to not match the aspect ratio so we have to compute the right dimensions here for square pixels StripHeight = StripWidth * FMath::Tan(FMath::DegreesToRadians(vAngIncrement / 2.0f)) / FMath::Tan(FMath::DegreesToRadians(hAngIncrement / 2.0f)); const FVector2D slicePlaneDim = FVector2D( 2.0f * FMath::Tan(FMath::DegreesToRadians(hAngIncrement) / 2.0f), 2.0f * FMath::Tan(FMath::DegreesToRadians(vAngIncrement) / 2.0f)); const float capturePlaneWidth = 2.0f * FMath::Tan(FMath::DegreesToRadians(captureHFov) / 2.0f); //TODO: ikrimae: This is just to let the rest of the existing code work. Sampling rate of the slice can be whatever. // Ex: To match the highest sampling frequency of the spherical atlas, it should match the area of differential patch // at ray direction of pixel(0,1) in the atlas //Need stripwidth/slicePlaneDim.X = capturewidth / capturePlaneDim.X CaptureWidth = capturePlaneWidth * StripWidth / slicePlaneDim.X; CaptureHeight = CaptureWidth * StripHeight / StripWidth; captureVFov = FMath::RadiansToDegrees(2 * FMath::Atan(FMath::Tan(FMath::DegreesToRadians(captureHFov / 2.0f)) * CaptureHeight / CaptureWidth)); //float dbgCapturePlaneDimY = 2.0f * FMath::Tan(FMath::DegreesToRadians(captureVFov) / 2.0f); //float dbgCaptureHeight = dbgCapturePlaneDimY * StripHeight / slicePlaneDim.Y; } else { //Slicing Technique 2: Each slice is a determined square FOV at a configured preset resolution. // Strip Width/Strip Height is determined based on hAngIncrement & vAngIncrement // Just make sure pixels/captureHFov >= pixels/hAngIncr && pixels/vAngIncr captureVFov = captureHFov = FStereoPanoramaManager::CaptureHorizontalFOV->GetFloat(); sliceVFov = sliceHFov = captureHFov; ensure(captureHFov >= FMath::Max(hAngIncrement, vAngIncrement)); //TODO: ikrimae: Re-do for floating point accuracy const FVector2D slicePlaneDim = FVector2D( 2.0f * FMath::Tan(FMath::DegreesToRadians(hAngIncrement) / 2.0f), 2.0f * FMath::Tan(FMath::DegreesToRadians(vAngIncrement) / 2.0f)); const FVector2D capturePlaneDim = FVector2D( 2.0f * FMath::Tan(FMath::DegreesToRadians(captureHFov) / 2.0f), 2.0f * FMath::Tan(FMath::DegreesToRadians(captureVFov) / 2.0f)); CaptureHeight = CaptureWidth = FStereoPanoramaManager::CaptureSlicePixelWidth->GetInt(); StripWidth = CaptureWidth * slicePlaneDim.X / capturePlaneDim.X; StripHeight = CaptureHeight * slicePlaneDim.Y / capturePlaneDim.Y; //TODO: ikrimae: Come back and check for the actual right sampling rate check(StripWidth >= (SphericalAtlasWidth / NumberOfHorizontalSteps) && StripHeight >= (SphericalAtlasHeight / NumberOfVerticalSteps)); //Ensure Width/Height is always even StripWidth += StripWidth & 1; StripHeight += StripHeight & 1; } UnprojectedAtlasWidth = NumberOfHorizontalSteps * StripWidth; UnprojectedAtlasHeight = NumberOfVerticalSteps * StripHeight; //NOTE: ikrimae: Ensure that the main gameview is > CaptureWidth x CaptureHeight. Bug in UE4 that won't re-alloc scene render targets to the correct size // when the scenecapture component > current window render target. https://answers.unrealengine.com/questions/80531/scene-capture-2d-max-resolution.html //TODO: ikrimae: Ensure that r.SceneRenderTargetResizeMethod=2 FSystemResolution::RequestResolutionChange(CaptureWidth, CaptureHeight, EWindowMode::Windowed); for( int CaptureIndex = 0; CaptureIndex < FStereoPanoramaManager::ConcurrentCaptures->GetInt(); CaptureIndex++ ) { FString LeftCounter = FString::Printf( TEXT( "LeftEyeCaptureComponent_%04d" ), CaptureIndex ); USceneCaptureComponent2D* LeftEyeCaptureComponent = CreateDefaultSubobject<USceneCaptureComponent2D>( *LeftCounter ); InitCaptureComponent( LeftEyeCaptureComponent, captureHFov, captureVFov, EStereoscopicPass::eSSP_LEFT_EYE ); LeftEyeCaptureComponents.Add( LeftEyeCaptureComponent ); FString RightCounter = FString::Printf( TEXT( "RightEyeCaptureComponent_%04d" ), CaptureIndex ); USceneCaptureComponent2D* RightEyeCaptureComponent = CreateDefaultSubobject<USceneCaptureComponent2D>( *RightCounter ); InitCaptureComponent( RightEyeCaptureComponent, captureHFov, captureVFov, EStereoscopicPass::eSSP_RIGHT_EYE ); RightEyeCaptureComponents.Add( RightEyeCaptureComponent ); } CurrentStep = 0; TotalSteps = 0; FrameDescriptors = TEXT( "FrameNumber, GameClock, TimeTaken(s)" LINE_TERMINATOR ); CaptureStep = ECaptureStep::Reset; }
/** Attach a GTCaptureComponent to a pawn */ UGTCaptureComponent* UGTCaptureComponent::Create(APawn* InPawn, TArray<FString> Modes) { UWorld* World = FUE4CVServer::Get().GetGameWorld(); UGTCaptureComponent* GTCapturer = NewObject<UGTCaptureComponent>(); GTCapturer->bIsActive = true; // check(GTCapturer->IsComponentTickEnabled() == true); GTCapturer->Pawn = InPawn; // This GTCapturer should depend on the Pawn and be released together with the Pawn. // This snippet is from Engine/Source/Runtime/Engine/Private/Components/SceneComponent.cpp, AttachTo FAttachmentTransformRules AttachmentRules(EAttachmentRule::KeepRelative, false); ConvertAttachLocation(EAttachLocation::KeepRelativeOffset, AttachmentRules.LocationRule, AttachmentRules.RotationRule, AttachmentRules.ScaleRule); GTCapturer->AttachToComponent(InPawn->GetRootComponent(), AttachmentRules); // GTCapturer->AddToRoot(); GTCapturer->RegisterComponentWithWorld(World); GTCapturer->SetTickableWhenPaused(true); for (FString Mode : Modes) { // DEPRECATED_FORGAME(4.6, "CaptureComponent2D should not be accessed directly, please use GetCaptureComponent2D() function instead. CaptureComponent2D will soon be private and your code will not compile.") USceneCaptureComponent2D* CaptureComponent = NewObject<USceneCaptureComponent2D>(); CaptureComponent->bIsActive = false; // Disable it by default for performance consideration GTCapturer->CaptureComponents.Add(Mode, CaptureComponent); // CaptureComponent needs to be attached to somewhere immediately, otherwise it will be gc-ed CaptureComponent->AttachToComponent(GTCapturer, AttachmentRules); InitCaptureComponent(CaptureComponent); UMaterial* Material = GetMaterial(Mode); if (Mode == "lit") // For rendered images { // FViewMode::Lit(CaptureComponent->ShowFlags); CaptureComponent->TextureTarget->TargetGamma = GEngine->GetDisplayGamma(); // float DisplayGamma = SceneViewport->GetDisplayGamma(); } else if (Mode == "default") { continue; } else // for ground truth { CaptureComponent->TextureTarget->TargetGamma = 1; if (Mode == "object_mask") // For object mask { // FViewMode::Lit(CaptureComponent->ShowFlags); FViewMode::VertexColor(CaptureComponent->ShowFlags); } else if (Mode == "wireframe") // For object mask { FViewMode::Wireframe(CaptureComponent->ShowFlags); } else { check(Material); // GEngine->GetDisplayGamma(), the default gamma is 2.2 // CaptureComponent->TextureTarget->TargetGamma = 2.2; FViewMode::PostProcess(CaptureComponent->ShowFlags); CaptureComponent->PostProcessSettings.AddBlendable(Material, 1); // Instead of switching post-process materials, we create several SceneCaptureComponent, so that we can capture different GT within the same frame. } } } return GTCapturer; }