void HiresTexture::Shutdown()
{
if (s_prefetcher.joinable())
{
s_textureCacheAbortLoading.Set();
s_prefetcher.join();
}
s_textureMap.clear();
s_textureCache.clear();
}
HiresTexture* HiresTexture::Load(const std::string& basename,
std::function<u8*(size_t)> request_buffer_delegate,
bool cacheresult)
{
if (s_textureMap.size() == 0)
{
return nullptr;
}
HiresTextureCache::iterator iter = s_textureMap.find(basename);
if (iter == s_textureMap.end())
{
return nullptr;
}
HiresTextureCacheItem& current = iter->second;
if (current.maps[MapType::color].size() == 0)
{
return nullptr;
}
// First level is mandatory
if (current.maps[MapType::color][0].path.size() == 0)
{
return nullptr;
}
HiresTexture* ret = nullptr;
u8* buffer_pointer;
u32 maxwidth = 0;
u32 maxheight = 0;
bool last_level_is_dds = false;
bool allocated_data = false;
bool mipmapsize_included = false;
size_t material_mat_index =
current.maps[MapType::color].size() == current.maps[MapType::normal].size() &&
current.maps[MapType::normal].size() != current.maps[MapType::material].size() ?
MapType::normal :
MapType::material;
size_t emissive_index = MapType::emissive;
bool nrm_posible =
current.maps[MapType::color].size() == current.maps[material_mat_index].size() &&
g_ActiveConfig.HiresMaterialMapsEnabled();
bool emissive_posible =
current.maps[MapType::color].size() == current.maps[emissive_index].size() &&
g_ActiveConfig.HiresMaterialMapsEnabled();
size_t remaining_buffer_size = 0;
size_t total_buffer_size = 0;
std::function<u8*(size_t, bool)> first_level_function = [&](size_t requiredsize,
bool mipmapsincluded) {
// Allocate double side buffer if we are going to load normal maps
allocated_data = true;
mipmapsize_included = mipmapsincluded;
// Pre allocate space for the textures and potetially all the posible mip levels
if (current.maps[MapType::color].size() > 1 && !mipmapsincluded)
{
requiredsize = (requiredsize * 4) / 3;
}
requiredsize *=
(nrm_posible && emissive_posible ? 3 : (nrm_posible || emissive_posible ? 2 : 1));
total_buffer_size = requiredsize;
remaining_buffer_size = total_buffer_size;
return request_buffer_delegate(requiredsize);
};
std::function<u8*(size_t, bool)> allocation_function = [&](size_t requiredsize,
bool mipmapsincluded) {
// is required size is biguer that the remaining size on the packed buffer just reject
if (requiredsize > remaining_buffer_size)
{
return static_cast<u8*>(nullptr);
}
// just return the pointer to pack the textures in a single buffer.
return buffer_pointer;
};
for (size_t level = 0; level < current.maps[MapType::color].size(); level++)
{
const hires_mip_level& item = current.maps[MapType::color][level];
ImageLoaderParams imgInfo =
LoadMipLevel(item, level == 0 ? first_level_function : allocation_function, cacheresult);
imgInfo.releaseresourcesonerror = cacheresult;
nrm_posible = nrm_posible && current.maps[material_mat_index][level].path.size() > 0;
emissive_posible = emissive_posible && current.maps[emissive_index][level].path.size() > 0;
bool ddsfile = item.is_compressed && TexDecoder::IsCompressed(imgInfo.resultTex);
if ((level > 0 && ddsfile != last_level_is_dds) || imgInfo.dst == nullptr ||
imgInfo.resultTex == PC_TEX_FMT_NONE)
{
// don't give support to mixed formats
if (allocated_data && cacheresult && imgInfo.dst != nullptr)
{
delete[] imgInfo.dst;
}
break;
}
last_level_is_dds = ddsfile;
if (level == 0)
{
ret = new HiresTexture();
ret->has_arbitrary_mips = current.has_arbitrary_mips;
ret->m_format = imgInfo.resultTex;
ret->m_width = maxwidth = imgInfo.Width;
ret->m_height = maxheight = imgInfo.Height;
if (cacheresult)
{
ret->m_cached_data.reset(imgInfo.dst);
ret->m_cached_data_size = total_buffer_size;
}
}
else
{
if (!ValidateImage(imgInfo, level, maxwidth, maxheight, ret->m_format, "color"))
break;
}
ret->m_levels++;
if (ddsfile)
{
u32 requiredsize = TextureUtil::GetTextureSizeInBytes(maxwidth, maxheight, imgInfo.resultTex);
buffer_pointer = imgInfo.dst + requiredsize;
remaining_buffer_size -= requiredsize;
}
else
{
buffer_pointer = imgInfo.dst + imgInfo.data_size;
remaining_buffer_size -= imgInfo.data_size;
}
if (imgInfo.nummipmaps > 0)
{
// Give priority to load dds with packed levels
ret->m_levels = imgInfo.nummipmaps + 1;
if (nrm_posible || emissive_posible)
{
SkipMipData(ret->m_levels, imgInfo.Width, imgInfo.Height, ret->m_format, buffer_pointer,
remaining_buffer_size);
}
break;
}
// no more mipmaps available
if (maxwidth == 1 && maxheight == 1)
break;
maxwidth = std::max(maxwidth >> 1, 1u);
maxheight = std::max(maxheight >> 1, 1u);
}
if (nrm_posible)
{
for (size_t level = 0; level < current.maps[material_mat_index].size(); level++)
{
const hires_mip_level& item = current.maps[material_mat_index][level];
ImageLoaderParams imgInfo = LoadMipLevel(item, allocation_function, cacheresult);
bool ddsfile = item.is_compressed && TexDecoder::IsCompressed(imgInfo.resultTex);
if ((level > 0 && ddsfile != last_level_is_dds) || imgInfo.dst == nullptr ||
imgInfo.resultTex == PC_TEX_FMT_NONE)
{
// don't give support to mixed formats
break;
}
if (g_ActiveConfig.bHiresMaterialMapsBuild && imgInfo.resultTex == PC_TEX_FMT_RGBA32)
{
BuildMaterial(current, imgInfo, level);
}
if (level == 0)
{
maxwidth = imgInfo.Width;
maxheight = imgInfo.Height;
if (!ValidateImage(imgInfo, level, ret->m_width, ret->m_height, ret->m_format, "normal"))
break;
}
else
{
if (!ValidateImage(imgInfo, level, maxwidth, maxheight, ret->m_format, "normal"))
break;
}
ret->m_nrm_levels++;
if (ddsfile)
{
u32 requiredsize =
TextureUtil::GetTextureSizeInBytes(maxwidth, maxheight, imgInfo.resultTex);
buffer_pointer = imgInfo.dst + requiredsize;
remaining_buffer_size -= requiredsize;
}
else
{
buffer_pointer = imgInfo.dst + imgInfo.data_size;
remaining_buffer_size -= imgInfo.data_size;
}
if (imgInfo.nummipmaps > 0)
{
// Give priority to load dds with packed levels
ret->m_nrm_levels = imgInfo.nummipmaps + 1;
if (emissive_posible)
{
SkipMipData(ret->m_nrm_levels, imgInfo.Width, imgInfo.Height, ret->m_format,
buffer_pointer, remaining_buffer_size);
}
break;
}
// no more mipmaps available
if (maxwidth == 1 && maxheight == 1)
break;
maxwidth = std::max(maxwidth >> 1, 1u);
maxheight = std::max(maxheight >> 1, 1u);
}
}
if (emissive_posible)
{
for (size_t level = 0; level < current.maps[emissive_index].size(); level++)
{
const hires_mip_level& item = current.maps[emissive_index][level];
ImageLoaderParams imgInfo = LoadMipLevel(item, allocation_function, cacheresult);
bool ddsfile = item.is_compressed && TexDecoder::IsCompressed(imgInfo.resultTex);
if ((level > 0 && ddsfile != last_level_is_dds) || imgInfo.dst == nullptr ||
imgInfo.resultTex == PC_TEX_FMT_NONE)
{
// don't give support to mixed formats
break;
}
if (level == 0)
{
maxwidth = imgInfo.Width;
maxheight = imgInfo.Height;
if (!ValidateImage(imgInfo, level, ret->m_width, ret->m_height, ret->m_format, "normal"))
break;
}
else
{
if (!ValidateImage(imgInfo, level, maxwidth, maxheight, ret->m_format, "normal"))
break;
}
ret->m_lum_levels++;
if (ddsfile)
{
u32 requiredsize =
TextureUtil::GetTextureSizeInBytes(maxwidth, maxheight, imgInfo.resultTex);
buffer_pointer = imgInfo.dst + requiredsize;
remaining_buffer_size -= requiredsize;
}
else
{
buffer_pointer = imgInfo.dst + imgInfo.data_size;
remaining_buffer_size -= imgInfo.data_size;
}
if (imgInfo.nummipmaps > 0)
{
// Give priority to load dds with packed levels
ret->m_lum_levels = imgInfo.nummipmaps + 1;
break;
}
// no more mipmaps available
if (maxwidth == 1 && maxheight == 1)
break;
maxwidth = std::max(maxwidth >> 1, 1u);
maxheight = std::max(maxheight >> 1, 1u);
}
}
if (ret != nullptr)
{
u32 levels = ret->m_levels;
// Disable normal map if the size or levels are different
if (ret->m_nrm_levels != levels)
{
ret->m_nrm_levels = 0;
}
if (ret->m_lum_levels != levels)
{
ret->m_lum_levels = 0;
}
}
return ret;
}
void HiresTexture::Prefetch()
{
Common::SetCurrentThreadName("Prefetcher");
const size_t total = s_textureMap.size();
size_t count = 0;
size_t notification = 10;
u32 starttime = Common::Timer::GetTimeMs();
for (const auto& entry : s_textureMap)
{
const std::string& base_filename = entry.first;
std::unique_lock<std::mutex> lk(s_textureCacheMutex);
auto iter = s_textureCache.find(base_filename);
if (iter == s_textureCache.end())
{
lk.unlock();
HiresTexture* ptr =
Load(base_filename, [](size_t requested_size) { return new u8[requested_size]; }, true);
lk.lock();
if (ptr != nullptr)
{
size_sum.fetch_add(ptr->m_cached_data_size);
iter = s_textureCache.insert(
iter, std::make_pair(base_filename, std::shared_ptr<HiresTexture>(ptr)));
}
}
if (s_textureCacheAbortLoading.IsSet())
{
if (g_ActiveConfig.bWaitForCacheHiresTextures)
{
Host_UpdateProgressDialog("", -1, -1);
}
return;
}
if (size_sum.load() > max_mem)
{
Config::SetCurrent(Config::GFX_HIRES_TEXTURES, false);
OSD::AddMessage(
StringFromFormat(
"Custom Textures prefetching after %.1f MB aborted, not enough RAM available",
size_sum / (1024.0 * 1024.0)),
10000);
if (g_ActiveConfig.bWaitForCacheHiresTextures)
{
Host_UpdateProgressDialog("", -1, -1);
}
return;
}
count++;
size_t percent = (count * 100) / total;
if (percent >= notification)
{
if (g_ActiveConfig.bWaitForCacheHiresTextures)
{
Host_UpdateProgressDialog(GetStringT("Prefetching Custom Textures...").c_str(),
static_cast<int>(count), static_cast<int>(total));
}
else
{
OSD::AddMessage(StringFromFormat("Custom Textures prefetching %.1f MB %zu %% finished",
size_sum / (1024.0 * 1024.0), percent),
2000);
}
notification += 10;
}
}
if (g_ActiveConfig.bWaitForCacheHiresTextures)
{
Host_UpdateProgressDialog("", -1, -1);
}
u32 stoptime = Common::Timer::GetTimeMs();
OSD::AddMessage(StringFromFormat("Custom Textures loaded, %.1f MB in %.1f s",
size_sum / (1024.0 * 1024.0), (stoptime - starttime) / 1000.0),
10000);
}
void HiresTexture::Update()
{
bool BuildMaterialMaps = g_ActiveConfig.bHiresMaterialMapsBuild;
if (s_prefetcher.joinable())
{
s_textureCacheAbortLoading.Set();
s_prefetcher.join();
}
if (!g_ActiveConfig.bHiresTextures)
{
s_textureMap.clear();
s_textureCache.clear();
size_sum.store(0);
return;
}
if (!g_ActiveConfig.bCacheHiresTextures)
{
s_textureCache.clear();
size_sum.store(0);
}
s_textureMap.clear();
const std::string& game_id = SConfig::GetInstance().GetGameID();
const std::string texture_directory = GetTextureDirectory(game_id);
std::string ddscode(".dds");
std::string cddscode(".DDS");
std::vector<std::string> Extensions;
Extensions.push_back(".png");
if (!BuildMaterialMaps)
{
Extensions.push_back(".dds");
}
std::vector<std::string> filenames =
Common::DoFileSearch({texture_directory}, Extensions, /*recursive*/ true);
const std::string miptag = "mip";
for (const std::string& fileitem : filenames)
{
std::string filename;
std::string Extension;
SplitPath(fileitem, nullptr, &filename, &Extension);
if (filename.substr(0, s_format_prefix.length()) != s_format_prefix)
{
// Discard wrong files
continue;
}
size_t map_index = 0;
size_t max_type = BuildMaterialMaps ? MapType::specular : MapType::normal;
bool arbitrary_mips = false;
for (size_t tag = 1; tag <= MapType::specular; tag++)
{
if (StringEndsWith(filename, s_maps_tags[tag]))
{
map_index = tag;
filename = filename.substr(0, filename.size() - s_maps_tags[tag].size());
break;
}
}
if (map_index > max_type)
{
continue;
}
if (BuildMaterialMaps && map_index == MapType::material)
{
continue;
}
else if (!BuildMaterialMaps && map_index == MapType::color)
{
const size_t arb_index = filename.rfind("_arb");
arbitrary_mips = arb_index != std::string::npos;
if (arbitrary_mips)
filename.erase(arb_index, 4);
}
const bool is_compressed = Extension.compare(ddscode) == 0 || Extension.compare(cddscode) == 0;
hires_mip_level mip_level_detail(fileitem, Extension, is_compressed);
u32 level = 0;
size_t idx = filename.find_last_of('_');
std::string miplevel = filename.substr(idx + 1, std::string::npos);
if (miplevel.substr(0, miptag.length()) == miptag)
{
sscanf(miplevel.substr(3, std::string::npos).c_str(), "%i", &level);
filename = filename.substr(0, idx);
}
HiresTextureCache::iterator iter = s_textureMap.find(filename);
u32 min_item_size = level + 1;
if (iter == s_textureMap.end())
{
HiresTextureCacheItem item(min_item_size);
if (arbitrary_mips)
{
item.has_arbitrary_mips = true;
}
item.maps[map_index].resize(min_item_size);
std::vector<hires_mip_level>& dst = item.maps[map_index];
dst[level] = mip_level_detail;
s_textureMap.emplace(filename, item);
}
else
{
std::vector<hires_mip_level>& dst = iter->second.maps[map_index];
if (arbitrary_mips)
{
iter->second.has_arbitrary_mips = true;
}
if (dst.size() < min_item_size)
{
dst.resize(min_item_size);
}
dst[level] = mip_level_detail;
}
}
if (g_ActiveConfig.bCacheHiresTextures && s_textureMap.size() > 0)
{
// remove cached but deleted textures
auto iter = s_textureCache.begin();
while (iter != s_textureCache.end())
{
if (s_textureMap.find(iter->first) == s_textureMap.end())
{
size_sum.fetch_sub(iter->second->m_cached_data_size);
iter = s_textureCache.erase(iter);
}
else
{
iter++;
}
}
s_textureCacheAbortLoading.Clear();
s_prefetcher = std::thread(Prefetch);
if (g_ActiveConfig.bWaitForCacheHiresTextures && s_prefetcher.joinable())
{
s_prefetcher.join();
}
}
}
void HiresTexture::Update()
{
s_check_native_format = false;
s_check_new_format = false;
if (s_prefetcher.joinable())
{
s_textureCacheAbortLoading.Set();
s_prefetcher.join();
}
if (!g_ActiveConfig.bHiresTextures)
{
s_textureMap.clear();
s_textureCache.clear();
size_sum.store(0);
return;
}
if (!g_ActiveConfig.bCacheHiresTextures)
{
s_textureCache.clear();
size_sum.store(0);
}
s_textureMap.clear();
const std::string& gameCode = SConfig::GetInstance().m_strUniqueID;
std::string szDir = StringFromFormat("%s%s", File::GetUserPath(D_HIRESTEXTURES_IDX).c_str(), gameCode.c_str());
std::string ddscode(".dds");
std::string cddscode(".DDS");
std::vector<std::string> Extensions = {
".png",
".dds"
};
auto rFilenames = DoFileSearch(Extensions, { szDir }, /*recursive*/ true);
const std::string code = StringFromFormat("%s_", gameCode.c_str());
const std::string miptag = "mip";
const std::string normaltag = ".nrm";
for (u32 i = 0; i < rFilenames.size(); i++)
{
std::string FileName;
std::string Extension;
SplitPath(rFilenames[i], nullptr, &FileName, &Extension);
if (FileName.substr(0, code.length()) == code)
{
s_check_native_format = true;
}
else if (FileName.substr(0, s_format_prefix.length()) == s_format_prefix)
{
s_check_new_format = true;
}
else
{
// Discard wrong files
continue;
}
const bool is_compressed = Extension.compare(ddscode) == 0 || Extension.compare(cddscode) == 0;
const bool is_normal_map = hasEnding(FileName, normaltag);
if (is_normal_map)
{
FileName = FileName.substr(0, FileName.size() - normaltag.size());
}
hires_mip_level mip_level_detail(rFilenames[i], Extension, is_compressed);
u32 level = 0;
size_t idx = FileName.find_last_of('_');
std::string miplevel = FileName.substr(idx + 1, std::string::npos);
if (miplevel.substr(0, miptag.length()) == miptag)
{
sscanf(miplevel.substr(3, std::string::npos).c_str(), "%i", &level);
FileName = FileName.substr(0, idx);
}
HiresTextureCache::iterator iter = s_textureMap.find(FileName);
u32 min_item_size = level + 1;
if (iter == s_textureMap.end())
{
HiresTextureCacheItem item(min_item_size);
if (is_normal_map)
{
item.normal_map.resize(min_item_size);
}
std::vector<hires_mip_level> &dst = is_normal_map ? item.normal_map : item.color_map;
dst[level] = mip_level_detail;
s_textureMap.emplace(FileName, item);
}
else
{
std::vector<hires_mip_level> &dst = is_normal_map ? iter->second.normal_map : iter->second.color_map;
if (dst.size() < min_item_size)
{
dst.resize(min_item_size);
}
dst[level] = mip_level_detail;
}
}
if (g_ActiveConfig.bCacheHiresTextures && s_textureMap.size() > 0)
{
// remove cached but deleted textures
auto iter = s_textureCache.begin();
while (iter != s_textureCache.end())
{
if (s_textureMap.find(iter->first) == s_textureMap.end())
{
size_sum.fetch_sub(iter->second->m_cached_data_size);
iter = s_textureCache.erase(iter);
}
else
{
iter++;
}
}
s_textureCacheAbortLoading.Clear();
s_prefetcher = std::thread(Prefetch);
}
}
std::string HiresTexture::GenBaseName(
const u8* texture, size_t texture_size,
const u8* tlut, size_t tlut_size,
u32 width, u32 height,
int format,
bool has_mipmaps,
bool dump)
{
std::string name = "";
bool convert = false;
HiresTextureCache::iterator convert_iter;
if ((!dump || convert) && s_check_native_format)
{
// try to load the old format first
u64 tex_hash = GetHashHiresTexture(texture, (int)texture_size, g_ActiveConfig.iSafeTextureCache_ColorSamples);
u64 tlut_hash = 0;
if(tlut_size)
tlut_hash = GetHashHiresTexture(tlut, (int)tlut_size, g_ActiveConfig.iSafeTextureCache_ColorSamples);
name = StringFromFormat("%s_%08x_%i", SConfig::GetInstance().m_strUniqueID.c_str(), (u32)(tex_hash ^ tlut_hash), (u16)format);
convert_iter = s_textureMap.find(name);
if (convert_iter != s_textureMap.end())
{
if (g_ActiveConfig.bConvertHiresTextures)
convert = true;
else
return name;
}
}
if (dump || s_check_new_format || convert)
{
// checking for min/max on paletted textures
u32 min = 0xffff;
u32 max = 0;
switch (tlut_size)
{
case 0: break;
case 16 * 2:
for (size_t i = 0; i < texture_size; i++)
{
min = std::min<u32>(min, texture[i] & 0xf);
min = std::min<u32>(min, texture[i] >> 4);
max = std::max<u32>(max, texture[i] & 0xf);
max = std::max<u32>(max, texture[i] >> 4);
}
break;
case 256 * 2:
for (size_t i = 0; i < texture_size; i++)
{
min = std::min<u32>(min, texture[i]);
max = std::max<u32>(max, texture[i]);
}
break;
case 16384 * 2:
for (size_t i = 0; i < texture_size / 2; i++)
{
min = std::min<u32>(min, Common::swap16(((u16*)texture)[i]) & 0x3fff);
max = std::max<u32>(max, Common::swap16(((u16*)texture)[i]) & 0x3fff);
}
break;
}
if (tlut_size > 0)
{
tlut_size = 2 * (max + 1 - min);
tlut += 2 * min;
}
u64 tex_hash = XXH64(texture, texture_size);
u64 tlut_hash = 0;
if(tlut_size)
tlut_hash = XXH64(tlut, tlut_size);
std::string basename = s_format_prefix + StringFromFormat("%dx%d%s_%0016" PRIx64, width, height, has_mipmaps ? "_m" : "", tex_hash);
std::string tlutname = tlut_size ? StringFromFormat("_%0016" PRIx64, tlut_hash) : "";
std::string formatname = StringFromFormat("_%d", format);
std::string fullname = basename + tlutname + formatname;
if (convert)
{
// new texture
if (s_textureMap.find(fullname) == s_textureMap.end())
{
HiresTextureCacheItem newitem(convert_iter->second.color_map.size());
for (size_t level = 0; level < convert_iter->second.color_map.size(); level++)
{
std::string newname = fullname;
if (level)
newname += StringFromFormat("_mip%d", level);
newname += convert_iter->second.color_map[level].extension;
std::string &src = convert_iter->second.color_map[level].path;
size_t postfix = src.find(name);
std::string dst = src.substr(0, postfix) + newname;
if (File::Rename(src, dst))
{
s_check_new_format = true;
OSD::AddMessage(StringFromFormat("Rename custom texture %s to %s", src.c_str(), dst.c_str()), 5000);
}
else
{
ERROR_LOG(VIDEO, "rename failed");
}
newitem.color_map[level] = hires_mip_level(dst, convert_iter->second.color_map[level].extension, convert_iter->second.color_map[level].is_compressed);
}
s_textureMap.emplace(fullname, newitem);
}
else
{
for (size_t level = 0; level < convert_iter->second.color_map.size(); level++)
{
if (File::Delete(convert_iter->second.color_map[level].path))
{
OSD::AddMessage(StringFromFormat("Delete double old custom texture %s", convert_iter->second.color_map[level].path.c_str()), 5000);
}
else
{
ERROR_LOG(VIDEO, "delete failed");
}
}
}
s_textureMap.erase(name);
}
return fullname;
}
