void USceneCapturer::CaptureComponent( int32 CurrentHorizontalStep, int32 CurrentVerticalStep, FString Folder, USceneCaptureComponent2D* CaptureComponent, TArray<FColor>& Atlas )
{
TArray<FColor> SurfaceData;
{
SCOPE_CYCLE_COUNTER( STAT_SPReadStrip );
FTextureRenderTargetResource* RenderTarget = CaptureComponent->TextureTarget->GameThread_GetRenderTargetResource();
//TODO: ikrimae: Might need to validate that this divides evenly. Might not matter
int32 CenterX = CaptureWidth / 2;
int32 CenterY = CaptureHeight / 2;
SurfaceData.AddUninitialized( StripWidth * StripHeight );
// Read pixels
FIntRect Area( CenterX - ( StripWidth / 2 ), CenterY - ( StripHeight / 2 ), CenterX + ( StripWidth / 2 ), CenterY + ( StripHeight / 2) );
auto readSurfaceDataFlags = FReadSurfaceDataFlags();
readSurfaceDataFlags.SetLinearToGamma(false);
RenderTarget->ReadPixelsPtr( SurfaceData.GetData(), readSurfaceDataFlags, Area );
}
// Copy off strip to atlas texture
CopyToUnprojAtlas( CurrentHorizontalStep, CurrentVerticalStep, Atlas, SurfaceData );
if( FStereoPanoramaManager::GenerateDebugImages->GetInt() != 0 )
{
SCOPE_CYCLE_COUNTER( STAT_SPSavePNG );
// Generate name
FString TickString = FString::Printf( TEXT( "_%05d_%04d_%04d" ), CurrentFrameCount, CurrentHorizontalStep, CurrentVerticalStep );
FString CaptureName = OutputDir / Timestamp / Folder / TickString + TEXT( ".png" );
UE_LOG( LogStereoPanorama, Log, TEXT( "Writing snapshot: %s" ), *CaptureName );
// Write out PNG
if (FStereoPanoramaManager::GenerateDebugImages->GetInt() == 2)
{
//Read Whole Capture Buffer
IImageWrapperPtr ImageWrapper = ImageWrapperModule.CreateImageWrapper( EImageFormat::PNG );
TArray<FColor> SurfaceDataWhole;
SurfaceDataWhole.AddUninitialized(CaptureWidth * CaptureHeight);
// Read pixels
FTextureRenderTargetResource* RenderTarget = CaptureComponent->TextureTarget->GameThread_GetRenderTargetResource();
RenderTarget->ReadPixelsPtr(SurfaceDataWhole.GetData(), FReadSurfaceDataFlags());
// Force alpha value
if (bForceAlpha)
{
for (FColor& Color : SurfaceDataWhole)
{
Color.A = 255;
}
}
ImageWrapper->SetRaw(SurfaceDataWhole.GetData(), SurfaceDataWhole.GetAllocatedSize(), CaptureWidth, CaptureHeight, ERGBFormat::BGRA, 8);
const TArray<uint8>& PNGData = ImageWrapper->GetCompressed(100);
FFileHelper::SaveArrayToFile(PNGData, *CaptureName);
ImageWrapper.Reset();
}
else
{
if (bForceAlpha)
{
for (FColor& Color : SurfaceData)
{
Color.A = 255;
}
}
IImageWrapperPtr ImageWrapper = ImageWrapperModule.CreateImageWrapper(EImageFormat::PNG);
ImageWrapper->SetRaw(SurfaceData.GetData(), SurfaceData.GetAllocatedSize(), StripWidth, StripHeight, ERGBFormat::BGRA, 8);
const TArray<uint8>& PNGData = ImageWrapper->GetCompressed(100);
FFileHelper::SaveArrayToFile( PNGData, *CaptureName );
ImageWrapper.Reset();
}
}
}
TArray<FColor> USceneCapturer::SaveAtlas(FString Folder, const TArray<FColor>& SurfaceData)
{
SCOPE_CYCLE_COUNTER( STAT_SPSavePNG );
TArray<FColor> SphericalAtlas;
SphericalAtlas.AddZeroed(SphericalAtlasWidth * SphericalAtlasHeight);
const FVector2D slicePlaneDim = FVector2D(
2.0f * FMath::Tan(FMath::DegreesToRadians(sliceHFov) / 2.0f),
2.0f * FMath::Tan(FMath::DegreesToRadians(sliceVFov) / 2.0f));
//For each direction,
// Find corresponding slice
// Calculate intersection of slice plane
// Calculate intersection UVs by projecting onto plane tangents
// Supersample that UV coordinate from the unprojected atlas
{
SCOPE_CYCLE_COUNTER(STAT_SPSampleSpherical);
// Dump out how long the process took
const FDateTime SamplingStartTime = FDateTime::UtcNow();
UE_LOG(LogStereoPanorama, Log, TEXT("Sampling atlas..."));
for (int32 y = 0; y < SphericalAtlasHeight; y++)
{
for (int32 x = 0; x < SphericalAtlasWidth; x++)
{
FLinearColor samplePixelAccum = FLinearColor(0, 0, 0, 0);
//TODO: ikrimae: Seems that bilinear filtering sans supersampling is good enough. Supersampling sans bilerp seems best.
// After more tests, come back to optimize by folding supersampling in and remove this outer sampling loop.
const auto& ssPattern = g_ssPatterns[SSMethod];
for (int32 SampleCount = 0; SampleCount < ssPattern.numSamples; SampleCount++)
{
const float sampleU = ((float)x + ssPattern.ssOffsets[SampleCount].X) / SphericalAtlasWidth;
const float sampleV = ((float)y + ssPattern.ssOffsets[SampleCount].Y) / SphericalAtlasHeight;
const float sampleTheta = sampleU * 360.0f;
const float samplePhi = sampleV * 180.0f;
const FVector sampleDir = FVector(
FMath::Sin(FMath::DegreesToRadians(samplePhi)) * FMath::Cos(FMath::DegreesToRadians(sampleTheta)),
FMath::Sin(FMath::DegreesToRadians(samplePhi)) * FMath::Sin(FMath::DegreesToRadians(sampleTheta)),
FMath::Cos(FMath::DegreesToRadians(samplePhi)));
//TODO: ikrimae: ugh, ugly.
const int32 sliceXIndex = FMath::TruncToInt(FRotator::ClampAxis(sampleTheta + hAngIncrement / 2.0f) / hAngIncrement);
int32 sliceYIndex = 0;
//Slice Selection = slice with max{sampleDir dot sliceNormal }
{
float largestCosAngle = 0;
for (int VerticalStep = 0; VerticalStep < NumberOfVerticalSteps; VerticalStep++)
{
const FVector2D sliceCenterThetaPhi = FVector2D(
hAngIncrement * sliceXIndex,
vAngIncrement * VerticalStep);
//TODO: ikrimae: There has got to be a faster way. Rethink reparametrization later
const FVector sliceDir = FVector(
FMath::Sin(FMath::DegreesToRadians(sliceCenterThetaPhi.Y)) * FMath::Cos(FMath::DegreesToRadians(sliceCenterThetaPhi.X)),
FMath::Sin(FMath::DegreesToRadians(sliceCenterThetaPhi.Y)) * FMath::Sin(FMath::DegreesToRadians(sliceCenterThetaPhi.X)),
FMath::Cos(FMath::DegreesToRadians(sliceCenterThetaPhi.Y)));
const float cosAngle = sampleDir | sliceDir;
if (cosAngle > largestCosAngle)
{
largestCosAngle = cosAngle;
sliceYIndex = VerticalStep;
}
}
}
const FVector2D sliceCenterThetaPhi = FVector2D(
hAngIncrement * sliceXIndex,
vAngIncrement * sliceYIndex);
//TODO: ikrimae: Reparameterize with an inverse mapping (e.g. project from slice pixels onto final u,v coordinates.
// Should make code simpler and faster b/c reduces to handful of sin/cos calcs per slice.
// Supersampling will be more difficult though.
const FVector sliceDir = FVector(
FMath::Sin(FMath::DegreesToRadians(sliceCenterThetaPhi.Y)) * FMath::Cos(FMath::DegreesToRadians(sliceCenterThetaPhi.X)),
FMath::Sin(FMath::DegreesToRadians(sliceCenterThetaPhi.Y)) * FMath::Sin(FMath::DegreesToRadians(sliceCenterThetaPhi.X)),
FMath::Cos(FMath::DegreesToRadians(sliceCenterThetaPhi.Y)));
const FPlane slicePlane = FPlane(sliceDir, -sliceDir);
//Tangents from partial derivatives of sphere equation
const FVector slicePlanePhiTangent = FVector(
FMath::Cos(FMath::DegreesToRadians(sliceCenterThetaPhi.Y)) * FMath::Cos(FMath::DegreesToRadians(sliceCenterThetaPhi.X)),
FMath::Cos(FMath::DegreesToRadians(sliceCenterThetaPhi.Y)) * FMath::Sin(FMath::DegreesToRadians(sliceCenterThetaPhi.X)),
-FMath::Sin(FMath::DegreesToRadians(sliceCenterThetaPhi.Y))).GetSafeNormal();
//Should be reconstructed to get around discontinuity of theta tangent at nodal points
const FVector slicePlaneThetaTangent = (sliceDir ^ slicePlanePhiTangent).GetSafeNormal();
//const FVector slicePlaneThetaTangent = FVector(
// -FMath::Sin(FMath::DegreesToRadians(sliceCenterThetaPhi.Y)) * FMath::Sin(FMath::DegreesToRadians(sliceCenterThetaPhi.X)),
// FMath::Sin(FMath::DegreesToRadians(sliceCenterThetaPhi.Y)) * FMath::Cos(FMath::DegreesToRadians(sliceCenterThetaPhi.X)),
// 0).SafeNormal();
check(!slicePlaneThetaTangent.IsZero() && !slicePlanePhiTangent.IsZero());
const double t = (double)-slicePlane.W / (sampleDir | sliceDir);
const FVector sliceIntersection = FVector(t * sampleDir.X, t * sampleDir.Y, t * sampleDir.Z);
//Calculate scalar projection of sliceIntersection onto tangent vectors. a dot b / |b| = a dot b when tangent vectors are normalized
//Then reparameterize to U,V of the sliceplane based on slice plane dimensions
const float sliceU = (sliceIntersection | slicePlaneThetaTangent) / slicePlaneDim.X;
const float sliceV = (sliceIntersection | slicePlanePhiTangent) / slicePlaneDim.Y;
check(sliceU >= -(0.5f + KINDA_SMALL_NUMBER) &&
sliceU <= (0.5f + KINDA_SMALL_NUMBER));
check(sliceV >= -(0.5f + KINDA_SMALL_NUMBER) &&
sliceV <= (0.5f + KINDA_SMALL_NUMBER));
//TODO: ikrimae: Supersample/bilinear filter
const int32 slicePixelX = FMath::TruncToInt(dbgMatchCaptureSliceFovToAtlasSliceFov ? sliceU * StripWidth : sliceU * CaptureWidth);
const int32 slicePixelY = FMath::TruncToInt(dbgMatchCaptureSliceFovToAtlasSliceFov ? sliceV * StripHeight : sliceV * CaptureHeight);
FLinearColor slicePixelSample;
if (bEnableBilerp)
{
//TODO: ikrimae: Clean up later; too tired now
const int32 sliceCenterPixelX = (sliceXIndex + 0.5f) * StripWidth;
const int32 sliceCenterPixelY = (sliceYIndex + 0.5f) * StripHeight;
const FIntPoint atlasSampleTL(sliceCenterPixelX + FMath::Clamp(slicePixelX , -StripWidth/2, StripWidth/2), sliceCenterPixelY + FMath::Clamp(slicePixelY , -StripHeight/2, StripHeight/2));
const FIntPoint atlasSampleTR(sliceCenterPixelX + FMath::Clamp(slicePixelX + 1, -StripWidth/2, StripWidth/2), sliceCenterPixelY + FMath::Clamp(slicePixelY , -StripHeight/2, StripHeight/2));
const FIntPoint atlasSampleBL(sliceCenterPixelX + FMath::Clamp(slicePixelX , -StripWidth/2, StripWidth/2), sliceCenterPixelY + FMath::Clamp(slicePixelY + 1, -StripHeight/2, StripHeight/2));
const FIntPoint atlasSampleBR(sliceCenterPixelX + FMath::Clamp(slicePixelX + 1, -StripWidth/2, StripWidth/2), sliceCenterPixelY + FMath::Clamp(slicePixelY + 1, -StripHeight/2, StripHeight/2));
const FColor pixelColorTL = SurfaceData[atlasSampleTL.Y * UnprojectedAtlasWidth + atlasSampleTL.X];
const FColor pixelColorTR = SurfaceData[atlasSampleTR.Y * UnprojectedAtlasWidth + atlasSampleTR.X];
const FColor pixelColorBL = SurfaceData[atlasSampleBL.Y * UnprojectedAtlasWidth + atlasSampleBL.X];
const FColor pixelColorBR = SurfaceData[atlasSampleBR.Y * UnprojectedAtlasWidth + atlasSampleBR.X];
const float fracX = FMath::Frac(dbgMatchCaptureSliceFovToAtlasSliceFov ? sliceU * StripWidth : sliceU * CaptureWidth);
const float fracY = FMath::Frac(dbgMatchCaptureSliceFovToAtlasSliceFov ? sliceV * StripHeight : sliceV * CaptureHeight);
//Reinterpret as linear (a.k.a dont apply srgb inversion)
slicePixelSample = FMath::BiLerp(
pixelColorTL.ReinterpretAsLinear(), pixelColorTR.ReinterpretAsLinear(),
pixelColorBL.ReinterpretAsLinear(), pixelColorBR.ReinterpretAsLinear(),
fracX, fracY);
}
else
{
const int32 sliceCenterPixelX = (sliceXIndex + 0.5f) * StripWidth;
const int32 sliceCenterPixelY = (sliceYIndex + 0.5f) * StripHeight;
const int32 atlasSampleX = sliceCenterPixelX + slicePixelX;
const int32 atlasSampleY = sliceCenterPixelY + slicePixelY;
slicePixelSample = SurfaceData[atlasSampleY * UnprojectedAtlasWidth + atlasSampleX].ReinterpretAsLinear();
}
samplePixelAccum += slicePixelSample;
////Output color map of projections
//const FColor debugEquiColors[12] = {
// FColor(205, 180, 76),
// FColor(190, 88, 202),
// FColor(127, 185, 194),
// FColor(90, 54, 47),
// FColor(197, 88, 53),
// FColor(197, 75, 124),
// FColor(130, 208, 72),
// FColor(136, 211, 153),
// FColor(126, 130, 207),
// FColor(83, 107, 59),
// FColor(200, 160, 157),
// FColor(80, 66, 106)
//};
//samplePixelAccum = ssPattern.numSamples * debugEquiColors[sliceYIndex * 4 + sliceXIndex];
}
SphericalAtlas[y * SphericalAtlasWidth + x] = (samplePixelAccum / ssPattern.numSamples).Quantize();
// Force alpha value
if (bForceAlpha)
{
SphericalAtlas[y * SphericalAtlasWidth + x].A = 255;
}
}
}
//Blit the first column into the last column to make the stereo image seamless at theta=360
for (int32 y = 0; y < SphericalAtlasHeight; y++)
{
SphericalAtlas[y * SphericalAtlasWidth + (SphericalAtlasWidth - 1)] = SphericalAtlas[y * SphericalAtlasWidth + 0];
}
const FTimespan SamplingDuration = FDateTime::UtcNow() - SamplingStartTime;
UE_LOG(LogStereoPanorama, Log, TEXT("...done! Duration: %g seconds"), SamplingDuration.GetTotalSeconds());
}
// Generate name
FString FrameString = FString::Printf( TEXT( "%s_%05d.png" ), *Folder, CurrentFrameCount );
FString AtlasName = OutputDir / Timestamp / FrameString;
UE_LOG( LogStereoPanorama, Log, TEXT( "Writing atlas: %s" ), *AtlasName );
// Write out PNG
//TODO: ikrimae: Use threads to write out the images for performance
IImageWrapperPtr ImageWrapper = ImageWrapperModule.CreateImageWrapper( EImageFormat::PNG );
ImageWrapper->SetRaw(SphericalAtlas.GetData(), SphericalAtlas.GetAllocatedSize(), SphericalAtlasWidth, SphericalAtlasHeight, ERGBFormat::BGRA, 8);
const TArray<uint8>& PNGData = ImageWrapper->GetCompressed(100);
FFileHelper::SaveArrayToFile( PNGData, *AtlasName );
if (FStereoPanoramaManager::GenerateDebugImages->GetInt() != 0)
{
FString FrameStringUnprojected = FString::Printf(TEXT("%s_%05d_Unprojected.png"), *Folder, CurrentFrameCount);
FString AtlasNameUnprojected = OutputDir / Timestamp / FrameStringUnprojected;
ImageWrapper->SetRaw(SurfaceData.GetData(), SurfaceData.GetAllocatedSize(), UnprojectedAtlasWidth, UnprojectedAtlasHeight, ERGBFormat::BGRA, 8);
const TArray<uint8>& PNGDataUnprojected = ImageWrapper->GetCompressed(100);
FFileHelper::SaveArrayToFile(PNGData, *AtlasNameUnprojected);
}
ImageWrapper.Reset();
UE_LOG( LogStereoPanorama, Log, TEXT( " ... done!" ), *AtlasName );
return SphericalAtlas;
}