HiresTexture* HiresTexture::Load(const std::string& basename, std::function<u8*(size_t)> request_buffer_delegate, bool cacheresult) { if (s_textureMap.size() == 0) { return nullptr; } HiresTextureCache::iterator iter = s_textureMap.find(basename); if (iter == s_textureMap.end()) { return nullptr; } HiresTextureCacheItem& current = iter->second; if (current.maps[MapType::color].size() == 0) { return nullptr; } // First level is mandatory if (current.maps[MapType::color][0].path.size() == 0) { return nullptr; } HiresTexture* ret = nullptr; u8* buffer_pointer; u32 maxwidth = 0; u32 maxheight = 0; bool last_level_is_dds = false; bool allocated_data = false; bool mipmapsize_included = false; size_t material_mat_index = current.maps[MapType::color].size() == current.maps[MapType::normal].size() && current.maps[MapType::normal].size() != current.maps[MapType::material].size() ? MapType::normal : MapType::material; size_t emissive_index = MapType::emissive; bool nrm_posible = current.maps[MapType::color].size() == current.maps[material_mat_index].size() && g_ActiveConfig.HiresMaterialMapsEnabled(); bool emissive_posible = current.maps[MapType::color].size() == current.maps[emissive_index].size() && g_ActiveConfig.HiresMaterialMapsEnabled(); size_t remaining_buffer_size = 0; size_t total_buffer_size = 0; std::function<u8*(size_t, bool)> first_level_function = [&](size_t requiredsize, bool mipmapsincluded) { // Allocate double side buffer if we are going to load normal maps allocated_data = true; mipmapsize_included = mipmapsincluded; // Pre allocate space for the textures and potetially all the posible mip levels if (current.maps[MapType::color].size() > 1 && !mipmapsincluded) { requiredsize = (requiredsize * 4) / 3; } requiredsize *= (nrm_posible && emissive_posible ? 3 : (nrm_posible || emissive_posible ? 2 : 1)); total_buffer_size = requiredsize; remaining_buffer_size = total_buffer_size; return request_buffer_delegate(requiredsize); }; std::function<u8*(size_t, bool)> allocation_function = [&](size_t requiredsize, bool mipmapsincluded) { // is required size is biguer that the remaining size on the packed buffer just reject if (requiredsize > remaining_buffer_size) { return static_cast<u8*>(nullptr); } // just return the pointer to pack the textures in a single buffer. return buffer_pointer; }; for (size_t level = 0; level < current.maps[MapType::color].size(); level++) { const hires_mip_level& item = current.maps[MapType::color][level]; ImageLoaderParams imgInfo = LoadMipLevel(item, level == 0 ? first_level_function : allocation_function, cacheresult); imgInfo.releaseresourcesonerror = cacheresult; nrm_posible = nrm_posible && current.maps[material_mat_index][level].path.size() > 0; emissive_posible = emissive_posible && current.maps[emissive_index][level].path.size() > 0; bool ddsfile = item.is_compressed && TexDecoder::IsCompressed(imgInfo.resultTex); if ((level > 0 && ddsfile != last_level_is_dds) || imgInfo.dst == nullptr || imgInfo.resultTex == PC_TEX_FMT_NONE) { // don't give support to mixed formats if (allocated_data && cacheresult && imgInfo.dst != nullptr) { delete[] imgInfo.dst; } break; } last_level_is_dds = ddsfile; if (level == 0) { ret = new HiresTexture(); ret->has_arbitrary_mips = current.has_arbitrary_mips; ret->m_format = imgInfo.resultTex; ret->m_width = maxwidth = imgInfo.Width; ret->m_height = maxheight = imgInfo.Height; if (cacheresult) { ret->m_cached_data.reset(imgInfo.dst); ret->m_cached_data_size = total_buffer_size; } } else { if (!ValidateImage(imgInfo, level, maxwidth, maxheight, ret->m_format, "color")) break; } ret->m_levels++; if (ddsfile) { u32 requiredsize = TextureUtil::GetTextureSizeInBytes(maxwidth, maxheight, imgInfo.resultTex); buffer_pointer = imgInfo.dst + requiredsize; remaining_buffer_size -= requiredsize; } else { buffer_pointer = imgInfo.dst + imgInfo.data_size; remaining_buffer_size -= imgInfo.data_size; } if (imgInfo.nummipmaps > 0) { // Give priority to load dds with packed levels ret->m_levels = imgInfo.nummipmaps + 1; if (nrm_posible || emissive_posible) { SkipMipData(ret->m_levels, imgInfo.Width, imgInfo.Height, ret->m_format, buffer_pointer, remaining_buffer_size); } break; } // no more mipmaps available if (maxwidth == 1 && maxheight == 1) break; maxwidth = std::max(maxwidth >> 1, 1u); maxheight = std::max(maxheight >> 1, 1u); } if (nrm_posible) { for (size_t level = 0; level < current.maps[material_mat_index].size(); level++) { const hires_mip_level& item = current.maps[material_mat_index][level]; ImageLoaderParams imgInfo = LoadMipLevel(item, allocation_function, cacheresult); bool ddsfile = item.is_compressed && TexDecoder::IsCompressed(imgInfo.resultTex); if ((level > 0 && ddsfile != last_level_is_dds) || imgInfo.dst == nullptr || imgInfo.resultTex == PC_TEX_FMT_NONE) { // don't give support to mixed formats break; } if (g_ActiveConfig.bHiresMaterialMapsBuild && imgInfo.resultTex == PC_TEX_FMT_RGBA32) { BuildMaterial(current, imgInfo, level); } if (level == 0) { maxwidth = imgInfo.Width; maxheight = imgInfo.Height; if (!ValidateImage(imgInfo, level, ret->m_width, ret->m_height, ret->m_format, "normal")) break; } else { if (!ValidateImage(imgInfo, level, maxwidth, maxheight, ret->m_format, "normal")) break; } ret->m_nrm_levels++; if (ddsfile) { u32 requiredsize = TextureUtil::GetTextureSizeInBytes(maxwidth, maxheight, imgInfo.resultTex); buffer_pointer = imgInfo.dst + requiredsize; remaining_buffer_size -= requiredsize; } else { buffer_pointer = imgInfo.dst + imgInfo.data_size; remaining_buffer_size -= imgInfo.data_size; } if (imgInfo.nummipmaps > 0) { // Give priority to load dds with packed levels ret->m_nrm_levels = imgInfo.nummipmaps + 1; if (emissive_posible) { SkipMipData(ret->m_nrm_levels, imgInfo.Width, imgInfo.Height, ret->m_format, buffer_pointer, remaining_buffer_size); } break; } // no more mipmaps available if (maxwidth == 1 && maxheight == 1) break; maxwidth = std::max(maxwidth >> 1, 1u); maxheight = std::max(maxheight >> 1, 1u); } } if (emissive_posible) { for (size_t level = 0; level < current.maps[emissive_index].size(); level++) { const hires_mip_level& item = current.maps[emissive_index][level]; ImageLoaderParams imgInfo = LoadMipLevel(item, allocation_function, cacheresult); bool ddsfile = item.is_compressed && TexDecoder::IsCompressed(imgInfo.resultTex); if ((level > 0 && ddsfile != last_level_is_dds) || imgInfo.dst == nullptr || imgInfo.resultTex == PC_TEX_FMT_NONE) { // don't give support to mixed formats break; } if (level == 0) { maxwidth = imgInfo.Width; maxheight = imgInfo.Height; if (!ValidateImage(imgInfo, level, ret->m_width, ret->m_height, ret->m_format, "normal")) break; } else { if (!ValidateImage(imgInfo, level, maxwidth, maxheight, ret->m_format, "normal")) break; } ret->m_lum_levels++; if (ddsfile) { u32 requiredsize = TextureUtil::GetTextureSizeInBytes(maxwidth, maxheight, imgInfo.resultTex); buffer_pointer = imgInfo.dst + requiredsize; remaining_buffer_size -= requiredsize; } else { buffer_pointer = imgInfo.dst + imgInfo.data_size; remaining_buffer_size -= imgInfo.data_size; } if (imgInfo.nummipmaps > 0) { // Give priority to load dds with packed levels ret->m_lum_levels = imgInfo.nummipmaps + 1; break; } // no more mipmaps available if (maxwidth == 1 && maxheight == 1) break; maxwidth = std::max(maxwidth >> 1, 1u); maxheight = std::max(maxheight >> 1, 1u); } } if (ret != nullptr) { u32 levels = ret->m_levels; // Disable normal map if the size or levels are different if (ret->m_nrm_levels != levels) { ret->m_nrm_levels = 0; } if (ret->m_lum_levels != levels) { ret->m_lum_levels = 0; } } return ret; }
void HiresTexture::Update() { bool BuildMaterialMaps = g_ActiveConfig.bHiresMaterialMapsBuild; if (s_prefetcher.joinable()) { s_textureCacheAbortLoading.Set(); s_prefetcher.join(); } if (!g_ActiveConfig.bHiresTextures) { s_textureMap.clear(); s_textureCache.clear(); size_sum.store(0); return; } if (!g_ActiveConfig.bCacheHiresTextures) { s_textureCache.clear(); size_sum.store(0); } s_textureMap.clear(); const std::string& game_id = SConfig::GetInstance().GetGameID(); const std::string texture_directory = GetTextureDirectory(game_id); std::string ddscode(".dds"); std::string cddscode(".DDS"); std::vector<std::string> Extensions; Extensions.push_back(".png"); if (!BuildMaterialMaps) { Extensions.push_back(".dds"); } std::vector<std::string> filenames = Common::DoFileSearch({texture_directory}, Extensions, /*recursive*/ true); const std::string miptag = "mip"; for (const std::string& fileitem : filenames) { std::string filename; std::string Extension; SplitPath(fileitem, nullptr, &filename, &Extension); if (filename.substr(0, s_format_prefix.length()) != s_format_prefix) { // Discard wrong files continue; } size_t map_index = 0; size_t max_type = BuildMaterialMaps ? MapType::specular : MapType::normal; bool arbitrary_mips = false; for (size_t tag = 1; tag <= MapType::specular; tag++) { if (StringEndsWith(filename, s_maps_tags[tag])) { map_index = tag; filename = filename.substr(0, filename.size() - s_maps_tags[tag].size()); break; } } if (map_index > max_type) { continue; } if (BuildMaterialMaps && map_index == MapType::material) { continue; } else if (!BuildMaterialMaps && map_index == MapType::color) { const size_t arb_index = filename.rfind("_arb"); arbitrary_mips = arb_index != std::string::npos; if (arbitrary_mips) filename.erase(arb_index, 4); } const bool is_compressed = Extension.compare(ddscode) == 0 || Extension.compare(cddscode) == 0; hires_mip_level mip_level_detail(fileitem, Extension, is_compressed); u32 level = 0; size_t idx = filename.find_last_of('_'); std::string miplevel = filename.substr(idx + 1, std::string::npos); if (miplevel.substr(0, miptag.length()) == miptag) { sscanf(miplevel.substr(3, std::string::npos).c_str(), "%i", &level); filename = filename.substr(0, idx); } HiresTextureCache::iterator iter = s_textureMap.find(filename); u32 min_item_size = level + 1; if (iter == s_textureMap.end()) { HiresTextureCacheItem item(min_item_size); if (arbitrary_mips) { item.has_arbitrary_mips = true; } item.maps[map_index].resize(min_item_size); std::vector<hires_mip_level>& dst = item.maps[map_index]; dst[level] = mip_level_detail; s_textureMap.emplace(filename, item); } else { std::vector<hires_mip_level>& dst = iter->second.maps[map_index]; if (arbitrary_mips) { iter->second.has_arbitrary_mips = true; } if (dst.size() < min_item_size) { dst.resize(min_item_size); } dst[level] = mip_level_detail; } } if (g_ActiveConfig.bCacheHiresTextures && s_textureMap.size() > 0) { // remove cached but deleted textures auto iter = s_textureCache.begin(); while (iter != s_textureCache.end()) { if (s_textureMap.find(iter->first) == s_textureMap.end()) { size_sum.fetch_sub(iter->second->m_cached_data_size); iter = s_textureCache.erase(iter); } else { iter++; } } s_textureCacheAbortLoading.Clear(); s_prefetcher = std::thread(Prefetch); if (g_ActiveConfig.bWaitForCacheHiresTextures && s_prefetcher.joinable()) { s_prefetcher.join(); } } }
void HiresTexture::Prefetch() { Common::SetCurrentThreadName("Prefetcher"); const size_t total = s_textureMap.size(); size_t count = 0; size_t notification = 10; u32 starttime = Common::Timer::GetTimeMs(); for (const auto& entry : s_textureMap) { const std::string& base_filename = entry.first; std::unique_lock<std::mutex> lk(s_textureCacheMutex); auto iter = s_textureCache.find(base_filename); if (iter == s_textureCache.end()) { lk.unlock(); HiresTexture* ptr = Load(base_filename, [](size_t requested_size) { return new u8[requested_size]; }, true); lk.lock(); if (ptr != nullptr) { size_sum.fetch_add(ptr->m_cached_data_size); iter = s_textureCache.insert( iter, std::make_pair(base_filename, std::shared_ptr<HiresTexture>(ptr))); } } if (s_textureCacheAbortLoading.IsSet()) { if (g_ActiveConfig.bWaitForCacheHiresTextures) { Host_UpdateProgressDialog("", -1, -1); } return; } if (size_sum.load() > max_mem) { Config::SetCurrent(Config::GFX_HIRES_TEXTURES, false); OSD::AddMessage( StringFromFormat( "Custom Textures prefetching after %.1f MB aborted, not enough RAM available", size_sum / (1024.0 * 1024.0)), 10000); if (g_ActiveConfig.bWaitForCacheHiresTextures) { Host_UpdateProgressDialog("", -1, -1); } return; } count++; size_t percent = (count * 100) / total; if (percent >= notification) { if (g_ActiveConfig.bWaitForCacheHiresTextures) { Host_UpdateProgressDialog(GetStringT("Prefetching Custom Textures...").c_str(), static_cast<int>(count), static_cast<int>(total)); } else { OSD::AddMessage(StringFromFormat("Custom Textures prefetching %.1f MB %zu %% finished", size_sum / (1024.0 * 1024.0), percent), 2000); } notification += 10; } } if (g_ActiveConfig.bWaitForCacheHiresTextures) { Host_UpdateProgressDialog("", -1, -1); } u32 stoptime = Common::Timer::GetTimeMs(); OSD::AddMessage(StringFromFormat("Custom Textures loaded, %.1f MB in %.1f s", size_sum / (1024.0 * 1024.0), (stoptime - starttime) / 1000.0), 10000); }
void HiresTexture::Update() { s_check_native_format = false; s_check_new_format = false; if (s_prefetcher.joinable()) { s_textureCacheAbortLoading.Set(); s_prefetcher.join(); } if (!g_ActiveConfig.bHiresTextures) { s_textureMap.clear(); s_textureCache.clear(); size_sum.store(0); return; } if (!g_ActiveConfig.bCacheHiresTextures) { s_textureCache.clear(); size_sum.store(0); } s_textureMap.clear(); const std::string& gameCode = SConfig::GetInstance().m_strUniqueID; std::string szDir = StringFromFormat("%s%s", File::GetUserPath(D_HIRESTEXTURES_IDX).c_str(), gameCode.c_str()); std::string ddscode(".dds"); std::string cddscode(".DDS"); std::vector<std::string> Extensions = { ".png", ".dds" }; auto rFilenames = DoFileSearch(Extensions, { szDir }, /*recursive*/ true); const std::string code = StringFromFormat("%s_", gameCode.c_str()); const std::string miptag = "mip"; const std::string normaltag = ".nrm"; for (u32 i = 0; i < rFilenames.size(); i++) { std::string FileName; std::string Extension; SplitPath(rFilenames[i], nullptr, &FileName, &Extension); if (FileName.substr(0, code.length()) == code) { s_check_native_format = true; } else if (FileName.substr(0, s_format_prefix.length()) == s_format_prefix) { s_check_new_format = true; } else { // Discard wrong files continue; } const bool is_compressed = Extension.compare(ddscode) == 0 || Extension.compare(cddscode) == 0; const bool is_normal_map = hasEnding(FileName, normaltag); if (is_normal_map) { FileName = FileName.substr(0, FileName.size() - normaltag.size()); } hires_mip_level mip_level_detail(rFilenames[i], Extension, is_compressed); u32 level = 0; size_t idx = FileName.find_last_of('_'); std::string miplevel = FileName.substr(idx + 1, std::string::npos); if (miplevel.substr(0, miptag.length()) == miptag) { sscanf(miplevel.substr(3, std::string::npos).c_str(), "%i", &level); FileName = FileName.substr(0, idx); } HiresTextureCache::iterator iter = s_textureMap.find(FileName); u32 min_item_size = level + 1; if (iter == s_textureMap.end()) { HiresTextureCacheItem item(min_item_size); if (is_normal_map) { item.normal_map.resize(min_item_size); } std::vector<hires_mip_level> &dst = is_normal_map ? item.normal_map : item.color_map; dst[level] = mip_level_detail; s_textureMap.emplace(FileName, item); } else { std::vector<hires_mip_level> &dst = is_normal_map ? iter->second.normal_map : iter->second.color_map; if (dst.size() < min_item_size) { dst.resize(min_item_size); } dst[level] = mip_level_detail; } } if (g_ActiveConfig.bCacheHiresTextures && s_textureMap.size() > 0) { // remove cached but deleted textures auto iter = s_textureCache.begin(); while (iter != s_textureCache.end()) { if (s_textureMap.find(iter->first) == s_textureMap.end()) { size_sum.fetch_sub(iter->second->m_cached_data_size); iter = s_textureCache.erase(iter); } else { iter++; } } s_textureCacheAbortLoading.Clear(); s_prefetcher = std::thread(Prefetch); } }