// block dimensions : widthStride, heightStride
// texture dims : width, height, x offset, y offset
static void WriteSwizzler(char*& p, u32 format, API_TYPE ApiType)
{
// left, top, of source rectangle within source texture
// width of the destination rectangle, scale_factor (1 or 2)
WRITE(p, "uniform int4 position;\n");
int blkW = TexDecoder_GetBlockWidthInTexels(format);
int blkH = TexDecoder_GetBlockHeightInTexels(format);
int samples = GetEncodedSampleCount(format);
if (ApiType == API_OPENGL)
{
WRITE(p, "#define samp0 samp9\n");
WRITE(p, "SAMPLER_BINDING(9) uniform sampler2DArray samp0;\n");
WRITE(p, " out vec4 ocol0;\n");
WRITE(p, "void main()\n");
}
else // D3D
{
WRITE(p,"sampler samp0 : register(s0);\n");
WRITE(p, "Texture2D Tex0 : register(t0);\n");
WRITE(p,"void main(\n");
WRITE(p," out float4 ocol0 : SV_Target)\n");
}
WRITE(p, "{\n"
" int2 sampleUv;\n"
" int2 uv1 = int2(gl_FragCoord.xy);\n"
);
WRITE(p, " int y_block_position = uv1.y & %d;\n", ~(blkH - 1));
WRITE(p, " int y_offset_in_block = uv1.y & %d;\n", blkH - 1);
WRITE(p, " int x_virtual_position = (uv1.x << %d) + y_offset_in_block * position.z;\n", IntLog2(samples));
WRITE(p, " int x_block_position = (x_virtual_position >> %d) & %d;\n", IntLog2(blkH), ~(blkW - 1));
if (samples == 1)
{
// 32 bit textures (RGBA8 and Z24) are stored in 2 cache line increments
WRITE(p, " bool first = 0 == (x_virtual_position & %d);\n", 8 * samples); // first cache line, used in the encoders
WRITE(p, " x_virtual_position = x_virtual_position << 1;\n");
}
WRITE(p, " int x_offset_in_block = x_virtual_position & %d;\n", blkW - 1);
WRITE(p, " int y_offset = (x_virtual_position >> %d) & %d;\n", IntLog2(blkW), blkH - 1);
WRITE(p, " sampleUv.x = x_offset_in_block + x_block_position;\n");
WRITE(p, " sampleUv.y = y_block_position + y_offset;\n");
WRITE(p, " float2 uv0 = float2(sampleUv);\n"); // sampleUv is the sample position in (int)gx_coords
WRITE(p, " uv0 += float2(0.5, 0.5);\n"); // move to center of pixel
WRITE(p, " uv0 *= float(position.w);\n"); // scale by two if needed (also move to pixel borders so that linear filtering will average adjacent pixel)
WRITE(p, " uv0 += float2(position.xy);\n"); // move to copied rect
WRITE(p, " uv0 /= float2(%d, %d);\n", EFB_WIDTH, EFB_HEIGHT); // normalize to [0:1]
if (ApiType == API_OPENGL) // ogl has to flip up and down
{
WRITE(p, " uv0.y = 1.0-uv0.y;\n");
}
WRITE(p, " float sample_offset = float(position.w) / float(%d);\n", EFB_WIDTH);
}
StreamBuffer::StreamBuffer(u32 type, u32 size)
: m_buffer(GenBuffer()), m_buffertype(type), m_size(ROUND_UP_POW2(size)),
m_bit_per_slot(IntLog2(ROUND_UP_POW2(size) / SYNC_POINTS))
{
m_iterator = 0;
m_used_iterator = 0;
m_free_iterator = 0;
}
TEST(MathUtil, IntLog2)
{
EXPECT_EQ(0, IntLog2(1));
EXPECT_EQ(1, IntLog2(2));
EXPECT_EQ(2, IntLog2(4));
EXPECT_EQ(3, IntLog2(8));
EXPECT_EQ(63, IntLog2(0x8000000000000000ull));
// Rounding behavior.
EXPECT_EQ(3, IntLog2(15));
EXPECT_EQ(63, IntLog2(0xFFFFFFFFFFFFFFFFull));
}
StreamBuffer::StreamBuffer(u32 type, u32 size, u32 align_size, bool need_cpu_buffer)
: m_buffer(GenBuffer()),
m_buffertype(type),
m_size(Common::AlignUpSizePow2(ROUND_UP_POW2(size), align_size)),
m_bit_per_slot(IntLog2(Common::AlignUpSizePow2(ROUND_UP_POW2(size), align_size) / SYNC_POINTS))
{
m_iterator = 0;
m_used_iterator = 0;
m_free_iterator = 0;
for (int i = 0; i < SYNC_POINTS; i++)
{
m_fences[i] = 0;
}
}
std::string FormatSize(u64 bytes)
{
// i18n: The symbol for the unit "bytes"
const char* const unit_symbols[] = {_trans("B"), _trans("KiB"), _trans("MiB"), _trans("GiB"),
_trans("TiB"), _trans("PiB"), _trans("EiB")};
// Find largest power of 2 less than size.
// div 10 to get largest named unit less than size
// 10 == log2(1024) (number of B in a KiB, KiB in a MiB, etc)
// Max value is 63 / 10 = 6
const int unit = IntLog2(std::max<u64>(bytes, 1)) / 10;
// Don't need exact values, only 5 most significant digits
const double unit_size = std::pow(2, unit * 10);
return StringFromFormat("%.2f %s", bytes / unit_size, GetStringT(unit_symbols[unit]).c_str());
}
static wxString NiceSizeFormat(u64 size)
{
// Return a pretty filesize string from byte count.
// e.g. 1134278 -> "1.08 MiB"
const char* const unit_symbols[] = { "B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB" };
// Find largest power of 2 less than size.
// div 10 to get largest named unit less than size
// 10 == log2(1024) (number of B in a KiB, KiB in a MiB, etc)
// Max value is 63 / 10 = 6
const int unit = IntLog2(std::max<u64>(size, 1)) / 10;
// Don't need exact values, only 5 most significant digits
double unit_size = std::pow(2, unit * 10);
return wxString::Format("%.2f %s", size / unit_size, unit_symbols[unit]);
}
static wxString NiceSizeFormat(u64 _size)
{
// Return a pretty filesize string from byte count.
// e.g. 1134278 -> "1.08 MiB"
const char* const unit_symbols[] = {"B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"};
// Find largest power of 2 less than _size.
// div 10 to get largest named unit less than _size
// 10 == log2(1024) (number of B in a KiB, KiB in a MiB, etc)
const u64 unit = IntLog2(std::max<u64>(_size, 1)) / 10;
const u64 unit_size = (1ull << (unit * 10));
// mul 1000 for 3 decimal places, add 5 to round up, div 10 for 2 decimal places
std::string value = std::to_string((_size * 1000 / unit_size + 5) / 10);
// Insert decimal point.
value.insert(value.size() - 2, ".");
return StrToWxStr(StringFromFormat("%s %s", value.c_str(), unit_symbols[unit]));
}
TextureCache::TCacheEntryBase* TextureCache::Load(const u32 stage)
{
const FourTexUnits &tex = bpmem.tex[stage >> 2];
const u32 id = stage & 3;
const u32 address = (tex.texImage3[id].image_base/* & 0x1FFFFF*/) << 5;
u32 width = tex.texImage0[id].width + 1;
u32 height = tex.texImage0[id].height + 1;
const int texformat = tex.texImage0[id].format;
const u32 tlutaddr = tex.texTlut[id].tmem_offset << 9;
const u32 tlutfmt = tex.texTlut[id].tlut_format;
const bool use_mipmaps = (tex.texMode0[id].min_filter & 3) != 0;
u32 tex_levels = use_mipmaps ? ((tex.texMode1[id].max_lod + 0xf) / 0x10 + 1) : 1;
const bool from_tmem = tex.texImage1[id].image_type != 0;
if (0 == address)
return nullptr;
// TexelSizeInNibbles(format) * width * height / 16;
const unsigned int bsw = TexDecoder_GetBlockWidthInTexels(texformat);
const unsigned int bsh = TexDecoder_GetBlockHeightInTexels(texformat);
unsigned int expandedWidth = ROUND_UP(width, bsw);
unsigned int expandedHeight = ROUND_UP(height, bsh);
const unsigned int nativeW = width;
const unsigned int nativeH = height;
// Hash assigned to texcache entry (also used to generate filenames used for texture dumping and custom texture lookup)
u64 base_hash = TEXHASH_INVALID;
u64 full_hash = TEXHASH_INVALID;
u32 full_format = texformat;
const bool isPaletteTexture = (texformat == GX_TF_C4 || texformat == GX_TF_C8 || texformat == GX_TF_C14X2);
// Reject invalid tlut format.
if (isPaletteTexture && tlutfmt > GX_TL_RGB5A3)
return nullptr;
if (isPaletteTexture)
full_format = texformat | (tlutfmt << 16);
const u32 texture_size = TexDecoder_GetTextureSizeInBytes(expandedWidth, expandedHeight, texformat);
u32 additional_mips_size = 0; // not including level 0, which is texture_size
// GPUs don't like when the specified mipmap count would require more than one 1x1-sized LOD in the mipmap chain
// e.g. 64x64 with 7 LODs would have the mipmap chain 64x64,32x32,16x16,8x8,4x4,2x2,1x1,0x0, so we limit the mipmap count to 6 there
tex_levels = std::min<u32>(IntLog2(std::max(width, height)) + 1, tex_levels);
for (u32 level = 1; level != tex_levels; ++level)
{
// We still need to calculate the original size of the mips
const u32 expanded_mip_width = ROUND_UP(CalculateLevelSize(width, level), bsw);
const u32 expanded_mip_height = ROUND_UP(CalculateLevelSize(height, level), bsh);
additional_mips_size += TexDecoder_GetTextureSizeInBytes(expanded_mip_width, expanded_mip_height, texformat);
}
// If we are recording a FifoLog, keep track of what memory we read.
// FifiRecorder does it's own memory modification tracking independant of the texture hashing below.
if (g_bRecordFifoData && !from_tmem)
FifoRecorder::GetInstance().UseMemory(address, texture_size + additional_mips_size, MemoryUpdate::TEXTURE_MAP);
const u8* src_data;
if (from_tmem)
src_data = &texMem[bpmem.tex[stage / 4].texImage1[stage % 4].tmem_even * TMEM_LINE_SIZE];
else
src_data = Memory::GetPointer(address);
// TODO: This doesn't hash GB tiles for preloaded RGBA8 textures (instead, it's hashing more data from the low tmem bank than it should)
base_hash = GetHash64(src_data, texture_size, g_ActiveConfig.iSafeTextureCache_ColorSamples);
u32 palette_size = 0;
if (isPaletteTexture)
{
palette_size = TexDecoder_GetPaletteSize(texformat);
full_hash = base_hash ^ GetHash64(&texMem[tlutaddr], palette_size, g_ActiveConfig.iSafeTextureCache_ColorSamples);
}
else
{
full_hash = base_hash;
}
// Search the texture cache for textures by address
//
// Find all texture cache entries for the current texture address, and decide whether to use one of
// them, or to create a new one
//
// In most cases, the fastest way is to use only one texture cache entry for the same address. Usually,
// when a texture changes, the old version of the texture is unlikely to be used again. If there were
// new cache entries created for normal texture updates, there would be a slowdown due to a huge amount
// of unused cache entries. Also thanks to texture pooling, overwriting an existing cache entry is
// faster than creating a new one from scratch.
//
// Some games use the same address for different textures though. If the same cache entry was used in
// this case, it would be constantly overwritten, and effectively there wouldn't be any caching for
// those textures. Examples for this are Metroid Prime and Castlevania 3. Metroid Prime has multiple
// sets of fonts on each other stored in a single texture and uses the palette to make different
// characters visible or invisible. In Castlevania 3 some textures are used for 2 different things or
// at least in 2 different ways(size 1024x1024 vs 1024x256).
//
// To determine whether to use multiple cache entries or a single entry, use the following heuristic:
// If the same texture address is used several times during the same frame, assume the address is used
// for different purposes and allow creating an additional cache entry. If there's at least one entry
// that hasn't been used for the same frame, then overwrite it, in order to keep the cache as small as
// possible. If the current texture is found in the cache, use that entry.
//
// For efb copies, the entry created in CopyRenderTargetToTexture always has to be used, or else it was
// done in vain.
std::pair<TexCache::iterator, TexCache::iterator> iter_range = textures_by_address.equal_range((u64)address);
TexCache::iterator iter = iter_range.first;
TexCache::iterator oldest_entry = iter;
int temp_frameCount = 0x7fffffff;
TexCache::iterator unconverted_copy = textures_by_address.end();
while (iter != iter_range.second)
{
TCacheEntryBase* entry = iter->second;
// Do not load strided EFB copies, they are not meant to be used directly
if (entry->IsEfbCopy() && entry->native_width == nativeW && entry->native_height == nativeH &&
entry->memory_stride == entry->CacheLinesPerRow() * 32)
{
// EFB copies have slightly different rules as EFB copy formats have different
// meanings from texture formats.
if ((base_hash == entry->hash && (!isPaletteTexture || g_Config.backend_info.bSupportsPaletteConversion)) ||
IsPlayingBackFifologWithBrokenEFBCopies)
{
// TODO: We should check format/width/height/levels for EFB copies. Checking
// format is complicated because EFB copy formats don't exactly match
// texture formats. I'm not sure what effect checking width/height/levels
// would have.
if (!isPaletteTexture || !g_Config.backend_info.bSupportsPaletteConversion)
return ReturnEntry(stage, entry);
// Note that we found an unconverted EFB copy, then continue. We'll
// perform the conversion later. Currently, we only convert EFB copies to
// palette textures; we could do other conversions if it proved to be
// beneficial.
unconverted_copy = iter;
}
else
{
// Aggressively prune EFB copies: if it isn't useful here, it will probably
// never be useful again. It's theoretically possible for a game to do
// something weird where the copy could become useful in the future, but in
// practice it doesn't happen.
iter = FreeTexture(iter);
continue;
}
}
else
{
// For normal textures, all texture parameters need to match
if (entry->hash == full_hash && entry->format == full_format && entry->native_levels >= tex_levels &&
entry->native_width == nativeW && entry->native_height == nativeH)
{
entry = DoPartialTextureUpdates(iter);
return ReturnEntry(stage, entry);
}
}
// Find the texture which hasn't been used for the longest time. Count paletted
// textures as the same texture here, when the texture itself is the same. This
// improves the performance a lot in some games that use paletted textures.
// Example: Sonic the Fighters (inside Sonic Gems Collection)
// Skip EFB copies here, so they can be used for partial texture updates
if (entry->frameCount != FRAMECOUNT_INVALID && entry->frameCount < temp_frameCount &&
!entry->IsEfbCopy() && !(isPaletteTexture && entry->base_hash == base_hash))
{
temp_frameCount = entry->frameCount;
oldest_entry = iter;
}
++iter;
}
if (unconverted_copy != textures_by_address.end())
{
// Perform palette decoding.
TCacheEntryBase *entry = unconverted_copy->second;
TCacheEntryConfig config;
config.rendertarget = true;
config.width = entry->config.width;
config.height = entry->config.height;
config.layers = FramebufferManagerBase::GetEFBLayers();
TCacheEntryBase *decoded_entry = AllocateTexture(config);
decoded_entry->SetGeneralParameters(address, texture_size, full_format);
decoded_entry->SetDimensions(entry->native_width, entry->native_height, 1);
decoded_entry->SetHashes(base_hash, full_hash);
decoded_entry->frameCount = FRAMECOUNT_INVALID;
decoded_entry->is_efb_copy = false;
g_texture_cache->ConvertTexture(decoded_entry, entry, &texMem[tlutaddr], (TlutFormat)tlutfmt);
textures_by_address.emplace((u64)address, decoded_entry);
return ReturnEntry(stage, decoded_entry);
}
// Search the texture cache for normal textures by hash
//
// If the texture was fully hashed, the address does not need to match. Identical duplicate textures cause unnecessary slowdowns
// Example: Tales of Symphonia (GC) uses over 500 small textures in menus, but only around 70 different ones
if (g_ActiveConfig.iSafeTextureCache_ColorSamples == 0 ||
std::max(texture_size, palette_size) <= (u32)g_ActiveConfig.iSafeTextureCache_ColorSamples * 8)
{
iter_range = textures_by_hash.equal_range(full_hash);
iter = iter_range.first;
while (iter != iter_range.second)
{
TCacheEntryBase* entry = iter->second;
// All parameters, except the address, need to match here
if (entry->format == full_format && entry->native_levels >= tex_levels &&
entry->native_width == nativeW && entry->native_height == nativeH)
{
entry = DoPartialTextureUpdates(iter);
return ReturnEntry(stage, entry);
}
++iter;
}
}
// If at least one entry was not used for the same frame, overwrite the oldest one
if (temp_frameCount != 0x7fffffff)
{
// pool this texture and make a new one later
FreeTexture(oldest_entry);
}
std::shared_ptr<HiresTexture> hires_tex;
if (g_ActiveConfig.bHiresTextures)
{
hires_tex = HiresTexture::Search(
src_data, texture_size,
&texMem[tlutaddr], palette_size,
width, height,
texformat, use_mipmaps
);
if (hires_tex)
{
auto& l = hires_tex->m_levels[0];
if (l.width != width || l.height != height)
{
width = l.width;
height = l.height;
}
expandedWidth = l.width;
expandedHeight = l.height;
CheckTempSize(l.data_size);
memcpy(temp, l.data, l.data_size);
}
}
if (!hires_tex)
{
if (!(texformat == GX_TF_RGBA8 && from_tmem))
{
const u8* tlut = &texMem[tlutaddr];
TexDecoder_Decode(temp, src_data, expandedWidth, expandedHeight, texformat, tlut, (TlutFormat)tlutfmt);
}
else
{
u8* src_data_gb = &texMem[bpmem.tex[stage / 4].texImage2[stage % 4].tmem_odd * TMEM_LINE_SIZE];
TexDecoder_DecodeRGBA8FromTmem(temp, src_data, src_data_gb, expandedWidth, expandedHeight);
}
}
// how many levels the allocated texture shall have
const u32 texLevels = hires_tex ? (u32)hires_tex->m_levels.size() : tex_levels;
// create the entry/texture
TCacheEntryConfig config;
config.width = width;
config.height = height;
config.levels = texLevels;
TCacheEntryBase* entry = AllocateTexture(config);
GFX_DEBUGGER_PAUSE_AT(NEXT_NEW_TEXTURE, true);
iter = textures_by_address.emplace((u64)address, entry);
if (g_ActiveConfig.iSafeTextureCache_ColorSamples == 0 ||
std::max(texture_size, palette_size) <= (u32)g_ActiveConfig.iSafeTextureCache_ColorSamples * 8)
{
entry->textures_by_hash_iter = textures_by_hash.emplace(full_hash, entry);
}
entry->SetGeneralParameters(address, texture_size, full_format);
entry->SetDimensions(nativeW, nativeH, tex_levels);
entry->SetHashes(base_hash, full_hash);
entry->is_efb_copy = false;
entry->is_custom_tex = hires_tex != nullptr;
// load texture
entry->Load(width, height, expandedWidth, 0);
std::string basename = "";
if (g_ActiveConfig.bDumpTextures && !hires_tex)
{
basename = HiresTexture::GenBaseName(
src_data, texture_size,
&texMem[tlutaddr], palette_size,
width, height,
texformat, use_mipmaps,
true
);
DumpTexture(entry, basename, 0);
}
if (hires_tex)
{
for (u32 level = 1; level != texLevels; ++level)
{
auto& l = hires_tex->m_levels[level];
CheckTempSize(l.data_size);
memcpy(temp, l.data, l.data_size);
entry->Load(l.width, l.height, l.width, level);
}
}
else
{
// load mips - TODO: Loading mipmaps from tmem is untested!
src_data += texture_size;
const u8* ptr_even = nullptr;
const u8* ptr_odd = nullptr;
if (from_tmem)
{
ptr_even = &texMem[bpmem.tex[stage / 4].texImage1[stage % 4].tmem_even * TMEM_LINE_SIZE + texture_size];
ptr_odd = &texMem[bpmem.tex[stage / 4].texImage2[stage % 4].tmem_odd * TMEM_LINE_SIZE];
}
for (u32 level = 1; level != texLevels; ++level)
{
const u32 mip_width = CalculateLevelSize(width, level);
const u32 mip_height = CalculateLevelSize(height, level);
const u32 expanded_mip_width = ROUND_UP(mip_width, bsw);
const u32 expanded_mip_height = ROUND_UP(mip_height, bsh);
const u8*& mip_src_data = from_tmem
? ((level % 2) ? ptr_odd : ptr_even)
: src_data;
const u8* tlut = &texMem[tlutaddr];
TexDecoder_Decode(temp, mip_src_data, expanded_mip_width, expanded_mip_height, texformat, tlut, (TlutFormat)tlutfmt);
mip_src_data += TexDecoder_GetTextureSizeInBytes(expanded_mip_width, expanded_mip_height, texformat);
entry->Load(mip_width, mip_height, expanded_mip_width, level);
if (g_ActiveConfig.bDumpTextures)
DumpTexture(entry, basename, level);
}
}
INCSTAT(stats.numTexturesUploaded);
SETSTAT(stats.numTexturesAlive, textures_by_address.size());
entry = DoPartialTextureUpdates(iter);
return ReturnEntry(stage, entry);
}
// block dimensions : widthStride, heightStride
// texture dims : width, height, x offset, y offset
static void WriteSwizzler(char*& p, u32 format, APIType ApiType)
{
// left, top, of source rectangle within source texture
// width of the destination rectangle, scale_factor (1 or 2)
WRITE(p, "uniform int4 position;\n");
int blkW = TexDecoder_GetBlockWidthInTexels(format);
int blkH = TexDecoder_GetBlockHeightInTexels(format);
int samples = GetEncodedSampleCount(format);
if (ApiType == APIType::OpenGL)
{
WRITE(p, "#define samp0 samp9\n");
WRITE(p, "SAMPLER_BINDING(9) uniform sampler2DArray samp0;\n");
WRITE(p, " out vec4 ocol0;\n");
WRITE(p, "void main()\n");
WRITE(p, "{\n"
" int2 sampleUv;\n"
" int2 uv1 = int2(gl_FragCoord.xy);\n");
}
else // D3D
{
WRITE(p, "sampler samp0 : register(s0);\n");
WRITE(p, "Texture2DArray Tex0 : register(t0);\n");
WRITE(p, "void main(\n");
WRITE(p, " out float4 ocol0 : SV_Target, in float4 rawpos : SV_Position)\n");
WRITE(p, "{\n"
" int2 sampleUv;\n"
" int2 uv1 = int2(rawpos.xy);\n");
}
WRITE(p, " int x_block_position = (uv1.x >> %d) << %d;\n", IntLog2(blkH * blkW / samples),
IntLog2(blkW));
WRITE(p, " int y_block_position = uv1.y << %d;\n", IntLog2(blkH));
if (samples == 1)
{
// With samples == 1, we write out pairs of blocks; one A8R8, one G8B8.
WRITE(p, " bool first = (uv1.x & %d) == 0;\n", blkH * blkW / 2);
samples = 2;
}
WRITE(p, " int offset_in_block = uv1.x & %d;\n", (blkH * blkW / samples) - 1);
WRITE(p, " int y_offset_in_block = offset_in_block >> %d;\n", IntLog2(blkW / samples));
WRITE(p, " int x_offset_in_block = (offset_in_block & %d) << %d;\n", (blkW / samples) - 1,
IntLog2(samples));
WRITE(p, " sampleUv.x = x_block_position + x_offset_in_block;\n");
WRITE(p, " sampleUv.y = y_block_position + y_offset_in_block;\n");
WRITE(p,
" float2 uv0 = float2(sampleUv);\n"); // sampleUv is the sample position in (int)gx_coords
WRITE(p, " uv0 += float2(0.5, 0.5);\n"); // move to center of pixel
WRITE(p, " uv0 *= float(position.w);\n"); // scale by two if needed (also move to pixel borders
// so that linear filtering will average adjacent
// pixel)
WRITE(p, " uv0 += float2(position.xy);\n"); // move to copied rect
WRITE(p, " uv0 /= float2(%d, %d);\n", EFB_WIDTH, EFB_HEIGHT); // normalize to [0:1]
if (ApiType == APIType::OpenGL) // ogl has to flip up and down
{
WRITE(p, " uv0.y = 1.0-uv0.y;\n");
}
WRITE(p, " float sample_offset = float(position.w) / float(%d);\n", EFB_WIDTH);
}
// block dimensions : widthStride, heightStride
// texture dims : width, height, x offset, y offset
static void WriteSwizzler(char*& p, EFBCopyFormat format, APIType ApiType)
{
// left, top, of source rectangle within source texture
// width of the destination rectangle, scale_factor (1 or 2)
if (ApiType == APIType::Vulkan)
WRITE(p, "layout(std140, push_constant) uniform PCBlock { int4 position; } PC;\n");
else
WRITE(p, "uniform int4 position;\n");
// Alpha channel in the copy is set to 1 the EFB format does not have an alpha channel.
WRITE(p, "float4 RGBA8ToRGB8(float4 src)\n");
WRITE(p, "{\n");
WRITE(p, " return float4(src.xyz, 1.0);\n");
WRITE(p, "}\n");
WRITE(p, "float4 RGBA8ToRGBA6(float4 src)\n");
WRITE(p, "{\n");
WRITE(p, " int4 val = int4(src * 255.0) >> 2;\n");
WRITE(p, " return float4(val) / 63.0;\n");
WRITE(p, "}\n");
WRITE(p, "float4 RGBA8ToRGB565(float4 src)\n");
WRITE(p, "{\n");
WRITE(p, " int4 val = int4(src * 255.0);\n");
WRITE(p, " val = int4(val.r >> 3, val.g >> 2, val.b >> 3, 1);\n");
WRITE(p, " return float4(val) / float4(31.0, 63.0, 31.0, 1.0);\n");
WRITE(p, "}\n");
int blkW = TexDecoder_GetEFBCopyBlockWidthInTexels(format);
int blkH = TexDecoder_GetEFBCopyBlockHeightInTexels(format);
int samples = GetEncodedSampleCount(format);
if (ApiType == APIType::OpenGL)
{
WRITE(p, "#define samp0 samp9\n");
WRITE(p, "SAMPLER_BINDING(9) uniform sampler2DArray samp0;\n");
WRITE(p, "FRAGMENT_OUTPUT_LOCATION(0) out vec4 ocol0;\n");
WRITE(p, "void main()\n");
WRITE(p, "{\n"
" int2 sampleUv;\n"
" int2 uv1 = int2(gl_FragCoord.xy);\n");
}
else if (ApiType == APIType::Vulkan)
{
WRITE(p, "SAMPLER_BINDING(0) uniform sampler2DArray samp0;\n");
WRITE(p, "FRAGMENT_OUTPUT_LOCATION(0) out vec4 ocol0;\n");
WRITE(p, "void main()\n");
WRITE(p, "{\n"
" int2 sampleUv;\n"
" int2 uv1 = int2(gl_FragCoord.xy);\n"
" int4 position = PC.position;\n");
}
else // D3D
{
WRITE(p, "sampler samp0 : register(s0);\n");
WRITE(p, "Texture2DArray Tex0 : register(t0);\n");
WRITE(p, "void main(\n");
WRITE(p, " out float4 ocol0 : SV_Target, in float4 rawpos : SV_Position)\n");
WRITE(p, "{\n"
" int2 sampleUv;\n"
" int2 uv1 = int2(rawpos.xy);\n");
}
WRITE(p, " int x_block_position = (uv1.x >> %d) << %d;\n", IntLog2(blkH * blkW / samples),
IntLog2(blkW));
WRITE(p, " int y_block_position = uv1.y << %d;\n", IntLog2(blkH));
if (samples == 1)
{
// With samples == 1, we write out pairs of blocks; one A8R8, one G8B8.
WRITE(p, " bool first = (uv1.x & %d) == 0;\n", blkH * blkW / 2);
samples = 2;
}
WRITE(p, " int offset_in_block = uv1.x & %d;\n", (blkH * blkW / samples) - 1);
WRITE(p, " int y_offset_in_block = offset_in_block >> %d;\n", IntLog2(blkW / samples));
WRITE(p, " int x_offset_in_block = (offset_in_block & %d) << %d;\n", (blkW / samples) - 1,
IntLog2(samples));
WRITE(p, " sampleUv.x = x_block_position + x_offset_in_block;\n");
WRITE(p, " sampleUv.y = y_block_position + y_offset_in_block;\n");
WRITE(p,
" float2 uv0 = float2(sampleUv);\n"); // sampleUv is the sample position in (int)gx_coords
WRITE(p, " uv0 += float2(0.5, 0.5);\n"); // move to center of pixel
WRITE(p, " uv0 *= float(position.w);\n"); // scale by two if needed (also move to pixel borders
// so that linear filtering will average adjacent
// pixel)
WRITE(p, " uv0 += float2(position.xy);\n"); // move to copied rect
WRITE(p, " uv0 /= float2(%d, %d);\n", EFB_WIDTH, EFB_HEIGHT); // normalize to [0:1]
if (ApiType == APIType::OpenGL) // ogl has to flip up and down
{
WRITE(p, " uv0.y = 1.0-uv0.y;\n");
}
WRITE(p, " float sample_offset = float(position.w) / float(%d);\n", EFB_WIDTH);
}
void TDataProviderBuilder::Finish() {
CB_ENSURE(!IsDone, "Error: can't finish more than once");
DataProvider.Features.reserve(FeatureValues.size());
DataProvider.Order.resize(DataProvider.Targets.size());
std::iota(DataProvider.Order.begin(),
DataProvider.Order.end(), 0);
if (!AreEqualTo<ui64>(DataProvider.Timestamp, 0)) {
ShuffleFlag = false;
DataProvider.Order = CreateOrderByKey(DataProvider.Timestamp);
}
bool hasQueryIds = HasQueryIds(DataProvider.QueryIds);
if (!hasQueryIds) {
DataProvider.QueryIds.resize(0);
}
//TODO(noxoomo): it's not safe here, if we change order with shuffle everything'll go wrong
if (Pairs.size()) {
//they are local, so we don't need shuffle
CB_ENSURE(hasQueryIds, "Error: for GPU pairwise learning you should provide query id column. Query ids will be used to split data between devices and for dynamic boosting learning scheme.");
DataProvider.FillQueryPairs(Pairs);
}
if (ShuffleFlag) {
if (hasQueryIds) {
//should not change order inside query for pairs consistency
QueryConsistentShuffle(Seed, 1, DataProvider.QueryIds, &DataProvider.Order);
} else {
Shuffle(Seed, 1, DataProvider.Targets.size(), &DataProvider.Order);
}
DataProvider.SetShuffleSeed(Seed);
}
if (ShuffleFlag || !DataProvider.Timestamp.empty()) {
DataProvider.ApplyOrderToMetaColumns();
}
TVector<TString> featureNames;
featureNames.resize(FeatureValues.size());
TAdaptiveLock lock;
NPar::TLocalExecutor executor;
executor.RunAdditionalThreads(BuildThreads - 1);
TVector<TFeatureColumnPtr> featureColumns(FeatureValues.size());
if (!IsTest) {
RegisterFeaturesInFeatureManager(featureColumns);
}
TVector<TVector<float>> grid;
grid.resize(FeatureValues.size());
NPar::ParallelFor(executor, 0, FeatureValues.size(), [&](ui32 featureId) {
auto featureName = GetFeatureName(featureId);
featureNames[featureId] = featureName;
if (FeatureValues[featureId].size() == 0) {
return;
}
TVector<float> line(DataProvider.Order.size());
for (ui32 i = 0; i < DataProvider.Order.size(); ++i) {
line[i] = FeatureValues[featureId][DataProvider.Order[i]];
}
if (CatFeatureIds.has(featureId)) {
static_assert(sizeof(float) == sizeof(ui32), "Error: float size should be equal to ui32 size");
const bool shouldSkip = IsTest && (CatFeaturesPerfectHashHelper.GetUniqueValues(featureId) == 0);
if (!shouldSkip) {
auto data = CatFeaturesPerfectHashHelper.UpdatePerfectHashAndBinarize(featureId,
~line,
line.size());
const ui32 uniqueValues = CatFeaturesPerfectHashHelper.GetUniqueValues(featureId);
if (uniqueValues > 1) {
auto compressedData = CompressVector<ui64>(~data, line.size(), IntLog2(uniqueValues));
featureColumns[featureId] = MakeHolder<TCatFeatureValuesHolder>(featureId,
line.size(),
std::move(compressedData),
uniqueValues,
featureName);
}
}
} else {
auto floatFeature = MakeHolder<TFloatValuesHolder>(featureId,
std::move(line),
featureName);
TVector<float>& borders = grid[featureId];
ENanMode nanMode = ENanMode::Forbidden;
{
TGuard<TAdaptiveLock> guard(lock);
nanMode = FeaturesManager.GetOrCreateNanMode(*floatFeature);
}
if (FeaturesManager.HasFloatFeatureBorders(*floatFeature)) {
borders = FeaturesManager.GetFloatFeatureBorders(*floatFeature);
}
if (borders.empty() && !IsTest) {
const auto& floatValues = floatFeature->GetValues();
NCatboostOptions::TBinarizationOptions config = FeaturesManager.GetFloatFeatureBinarization();
config.NanMode = nanMode;
borders = BuildBorders(floatValues, floatFeature->GetId(), config);
}
if (borders.ysize() == 0) {
MATRIXNET_DEBUG_LOG << "Float Feature #" << featureId << " is empty" << Endl;
return;
}
auto binarizedData = BinarizeLine(floatFeature->GetValues().data(),
floatFeature->GetValues().size(),
nanMode,
borders);
const int binCount = static_cast<const int>(borders.size() + 1 + (ENanMode::Forbidden != nanMode));
auto compressedLine = CompressVector<ui64>(binarizedData, IntLog2(binCount));
featureColumns[featureId] = MakeHolder<TBinarizedFloatValuesHolder>(featureId,
floatFeature->GetValues().size(),
nanMode,
borders,
std::move(compressedLine),
featureName);
}
//Free memory
{
auto emptyVec = TVector<float>();
FeatureValues[featureId].swap(emptyVec);
}
});
for (ui32 featureId = 0; featureId < featureColumns.size(); ++featureId) {
if (CatFeatureIds.has(featureId)) {
if (featureColumns[featureId] == nullptr && (!IsTest)) {
MATRIXNET_DEBUG_LOG << "Cat Feature #" << featureId << " is empty" << Endl;
}
} else if (featureColumns[featureId] != nullptr) {
if (!FeaturesManager.HasFloatFeatureBordersForDataProviderFeature(featureId)) {
FeaturesManager.SetFloatFeatureBordersForDataProviderId(featureId,
std::move(grid[featureId]));
}
}
if (featureColumns[featureId] != nullptr) {
DataProvider.Features.push_back(std::move(featureColumns[featureId]));
}
}
DataProvider.BuildIndicesRemap();
if (!IsTest) {
TOnCpuGridBuilderFactory gridBuilderFactory;
FeaturesManager.SetTargetBorders(TBordersBuilder(gridBuilderFactory,
DataProvider.GetTargets())(FeaturesManager.GetTargetBinarizationDescription()));
}
DataProvider.FeatureNames = featureNames;
DataProvider.CatFeatureIds = CatFeatureIds;
if (ClassesWeights.size()) {
Reweight(DataProvider.Targets, ClassesWeights, &DataProvider.Weights);
}
IsDone = true;
}
// block dimensions : widthStride, heightStride
// texture dims : width, height, x offset, y offset
static void WriteSwizzler(char*& p, const EFBCopyParams& params, EFBCopyFormat format,
APIType ApiType)
{
WriteHeader(p, ApiType);
WriteSampleFunction(p, params, ApiType);
if (ApiType == APIType::OpenGL || ApiType == APIType::Vulkan)
{
WRITE(p, "void main()\n");
WRITE(p, "{\n"
" int2 sampleUv;\n"
" int2 uv1 = int2(gl_FragCoord.xy);\n");
}
else // D3D
{
WRITE(p, "void main(\n");
WRITE(p, " in float3 v_tex0 : TEXCOORD0,\n");
WRITE(p, " in float4 rawpos : SV_Position,\n");
WRITE(p, " out float4 ocol0 : SV_Target)\n");
WRITE(p, "{\n"
" int2 sampleUv;\n"
" int2 uv1 = int2(rawpos.xy);\n");
}
int blkW = TexDecoder_GetEFBCopyBlockWidthInTexels(format);
int blkH = TexDecoder_GetEFBCopyBlockHeightInTexels(format);
int samples = GetEncodedSampleCount(format);
WRITE(p, " int x_block_position = (uv1.x >> %d) << %d;\n", IntLog2(blkH * blkW / samples),
IntLog2(blkW));
WRITE(p, " int y_block_position = uv1.y << %d;\n", IntLog2(blkH));
if (samples == 1)
{
// With samples == 1, we write out pairs of blocks; one A8R8, one G8B8.
WRITE(p, " bool first = (uv1.x & %d) == 0;\n", blkH * blkW / 2);
samples = 2;
}
WRITE(p, " int offset_in_block = uv1.x & %d;\n", (blkH * blkW / samples) - 1);
WRITE(p, " int y_offset_in_block = offset_in_block >> %d;\n", IntLog2(blkW / samples));
WRITE(p, " int x_offset_in_block = (offset_in_block & %d) << %d;\n", (blkW / samples) - 1,
IntLog2(samples));
WRITE(p, " sampleUv.x = x_block_position + x_offset_in_block;\n");
WRITE(p, " sampleUv.y = y_block_position + y_offset_in_block;\n");
WRITE(p,
" float2 uv0 = float2(sampleUv);\n"); // sampleUv is the sample position in (int)gx_coords
WRITE(p, " uv0 += float2(0.5, 0.5);\n"); // move to center of pixel
WRITE(p, " uv0 *= float(position.w);\n"); // scale by two if needed (also move to pixel borders
// so that linear filtering will average adjacent
// pixel)
WRITE(p, " uv0 += float2(position.xy);\n"); // move to copied rect
WRITE(p, " uv0 /= float2(%d, %d);\n", EFB_WIDTH, EFB_HEIGHT); // normalize to [0:1]
WRITE(p, " uv0 /= float2(1, y_scale);\n"); // apply the y scaling
if (ApiType == APIType::OpenGL) // ogl has to flip up and down
{
WRITE(p, " uv0.y = 1.0-uv0.y;\n");
}
WRITE(p, " float2 pixel_size = float2(position.w, position.w) / float2(%d, %d);\n", EFB_WIDTH,
EFB_HEIGHT);
}