void CArchiveScanner::ScanArchive(const std::string& fullName, bool doChecksum)
{
struct stat info;
stat(fullName.c_str(), &info);
const std::string fn = filesystem.GetFilename(fullName);
const std::string fpath = filesystem.GetDirectory(fullName);
const std::string lcfn = StringToLower(fn);
//! Determine whether this archive has earlier be found to be broken
std::map<std::string, BrokenArchive>::iterator bai = brokenArchives.find(lcfn);
if (bai != brokenArchives.end()) {
if ((unsigned)info.st_mtime == bai->second.modified && fpath == bai->second.path) {
bai->second.updated = true;
return;
}
}
//! Determine whether to rely on the cached info or not
bool cached = false;
std::map<std::string, ArchiveInfo>::iterator aii = archiveInfo.find(lcfn);
if (aii != archiveInfo.end()) {
//! This archive may have been obsoleted, do not process it if so
if (aii->second.replaced.length() > 0) {
return;
}
if ((unsigned)info.st_mtime == aii->second.modified && fpath == aii->second.path) {
cached = true;
aii->second.updated = true;
}
//! If we are here, we could have invalid info in the cache
//! Force a reread if it's a directory archive, as st_mtime only
//! reflects changes to the directory itself, not the contents.
if (!cached) {
archiveInfo.erase(aii);
}
}
//! Time to parse the info we are interested in
if (cached) {
//! If cached is true, aii will point to the archive
if (doChecksum && (aii->second.checksum == 0))
aii->second.checksum = GetCRC(fullName);
} else {
CArchiveBase* ar = CArchiveFactory::OpenArchive(fullName);
if (!ar || !ar->IsOpen()) {
logOutput.Print("Unable to open archive: %s", fullName.c_str());
return;
}
ArchiveInfo ai;
std::string error = "";
std::string mapfile;
bool hasModinfo = ar->FileExists("modinfo.lua");
bool hasMapinfo = ar->FileExists("mapinfo.lua");
//! check for smf/sm3 and if the uncompression of important files is too costy
for (unsigned fid = 0; fid != ar->NumFiles(); ++fid)
{
std::string name;
int size;
ar->FileInfo(fid, name, size);
const std::string lowerName = StringToLower(name);
const std::string ext = filesystem.GetExtension(lowerName);
if ((ext == "smf") || (ext == "sm3")) {
mapfile = name;
}
const unsigned char metaFileClass = GetMetaFileClass(lowerName);
if ((metaFileClass != 0) && !(ar->HasLowReadingCost(fid))) {
//! is a meta-file and not cheap to read
if (metaFileClass == 1) {
//! 1st class
error = "Unpacking/reading cost for meta file " + name
+ " is too high, please repack the archive (make sure to use a non-solid algorithm, if applicable)";
break;
} else if (metaFileClass == 2) {
//! 2nd class
logOutput.Print(LOG_ARCHIVESCANNER,
"Warning: Archive %s: The cost for reading a 2nd class meta-file is too high: %s",
fullName.c_str(), name.c_str());
}
}
}
if (!error.empty()) {
//! we already have an error, no further evaluation required
} if (hasMapinfo || !mapfile.empty()) {
//! it is a map
if (hasMapinfo) {
ScanArchiveLua(ar, "mapinfo.lua", ai, error);
} else if (hasModinfo) {
//! backwards-compat for modinfo.lua in maps
ScanArchiveLua(ar, "modinfo.lua", ai, error);
}
if (ai.archiveData.GetName().empty()) {
//! FIXME The name will never be empty, if version is set (see HACK in ArchiveData)
ai.archiveData.SetInfoItemValueString("name", filesystem.GetBasename(mapfile));
}
if (ai.archiveData.GetMapFile().empty()) {
ai.archiveData.SetInfoItemValueString("mapfile", mapfile);
}
AddDependency(ai.archiveData.GetDependencies(), "Map Helper v1");
ai.archiveData.SetInfoItemValueInteger("modType", modtype::map);
logOutput.Print(LOG_ARCHIVESCANNER, "Found new map: %s", ai.archiveData.GetName().c_str());
} else if (hasModinfo) {
//! it is a mod
ScanArchiveLua(ar, "modinfo.lua", ai, error);
if (ai.archiveData.GetModType() == modtype::primary) {
AddDependency(ai.archiveData.GetDependencies(), "Spring content v1");
}
logOutput.Print(LOG_ARCHIVESCANNER, "Found new game: %s", ai.archiveData.GetName().c_str());
} else {
//! neither a map nor a mod: error
error = "missing modinfo.lua/mapinfo.lua";
}
delete ar;
if (!error.empty()) {
//! for some reason, the archive is marked as broken
logOutput.Print("Failed to scan %s (%s)", fullName.c_str(), error.c_str());
//! record it as broken, so we don't need to look inside everytime
BrokenArchive ba;
ba.path = fpath;
ba.modified = info.st_mtime;
ba.updated = true;
ba.problem = error;
brokenArchives[lcfn] = ba;
return;
}
ai.path = fpath;
ai.modified = info.st_mtime;
ai.origName = fn;
ai.updated = true;
//! Optionally calculate a checksum for the file
//! To prevent reading all files in all directory (.sdd) archives
//! every time this function is called, directory archive checksums
//! are calculated on the fly.
if (doChecksum) {
ai.checksum = GetCRC(fullName);
} else {
ai.checksum = 0;
}
archiveInfo[lcfn] = ai;
}
}
void CArchiveScanner::ScanArchive(const std::string& fullName, bool doChecksum)
{
const std::string fn = FileSystem::GetFilename(fullName);
const std::string fpath = FileSystem::GetDirectory(fullName);
const std::string lcfn = StringToLower(fn);
// Stat file
struct stat info = {0};
int statfailed = stat(fullName.c_str(), &info);
// If stat fails, assume the archive is not broken nor cached
if (!statfailed) {
// Determine whether this archive has earlier be found to be broken
std::map<std::string, BrokenArchive>::iterator bai = brokenArchives.find(lcfn);
if (bai != brokenArchives.end()) {
if ((unsigned)info.st_mtime == bai->second.modified && fpath == bai->second.path) {
bai->second.updated = true;
return;
}
}
// Determine whether to rely on the cached info or not
std::map<std::string, ArchiveInfo>::iterator aii = archiveInfos.find(lcfn);
if (aii != archiveInfos.end()) {
// This archive may have been obsoleted, do not process it if so
if (!aii->second.replaced.empty()) {
return;
}
if ((unsigned)info.st_mtime == aii->second.modified && fpath == aii->second.path) {
// cache found update checksum if wanted
aii->second.updated = true;
if (doChecksum && (aii->second.checksum == 0)) {
aii->second.checksum = GetCRC(fullName);
}
return;
} else {
if (aii->second.updated) {
const std::string filename = aii->first;
LOG_L(L_ERROR, "Found a \"%s\" already in \"%s\", ignoring.", fullName.c_str(), (aii->second.path + aii->second.origName).c_str());
if (IsBaseContent(filename)) {
throw user_error(std::string("duplicate base content detected:\n\t") + aii->second.path + std::string("\n\t") + fpath
+ std::string("\nPlease fix your configuration/installation as this can cause desyncs!"));
}
return;
}
// If we are here, we could have invalid info in the cache
// Force a reread if it is a directory archive (.sdd), as
// st_mtime only reflects changes to the directory itself,
// not the contents.
archiveInfos.erase(aii);
}
}
}
boost::scoped_ptr<IArchive> ar(archiveLoader.OpenArchive(fullName));
if (!ar || !ar->IsOpen()) {
LOG_L(L_WARNING, "Unable to open archive: %s", fullName.c_str());
// record it as broken, so we don't need to look inside everytime
BrokenArchive& ba = brokenArchives[lcfn];
ba.path = fpath;
ba.modified = info.st_mtime;
ba.updated = true;
ba.problem = "Unable to open archive";
return;
}
std::string error;
std::string mapfile;
const bool hasModinfo = ar->FileExists("modinfo.lua");
const bool hasMapinfo = ar->FileExists("mapinfo.lua");
ArchiveInfo ai;
auto& ad = ai.archiveData;
if (hasMapinfo) {
ScanArchiveLua(ar.get(), "mapinfo.lua", ai, error);
if (ad.GetMapFile().empty()) {
LOG_L(L_WARNING, "%s: mapfile isn't set in mapinfo.lua, please set it for faster loading!", fullName.c_str());
mapfile = SearchMapFile(ar.get(), error);
}
} else if (hasModinfo) {
ScanArchiveLua(ar.get(), "modinfo.lua", ai, error);
} else {
mapfile = SearchMapFile(ar.get(), error);
}
CheckCompression(ar.get(), fullName, error);
if (!error.empty()) {
// for some reason, the archive is marked as broken
LOG_L(L_WARNING, "Failed to scan %s (%s)", fullName.c_str(), error.c_str());
// record it as broken, so we don't need to look inside everytime
BrokenArchive& ba = brokenArchives[lcfn];
ba.path = fpath;
ba.modified = info.st_mtime;
ba.updated = true;
ba.problem = error;
return;
}
if (hasMapinfo || !mapfile.empty()) {
// it is a map
if (ad.GetName().empty()) {
// FIXME The name will never be empty, if version is set (see HACK in ArchiveData)
ad.SetInfoItemValueString("name_pure", FileSystem::GetBasename(mapfile));
ad.SetInfoItemValueString("name", FileSystem::GetBasename(mapfile));
}
if (ad.GetMapFile().empty()) {
ad.SetInfoItemValueString("mapfile", mapfile);
}
AddDependency(ad.GetDependencies(), "Map Helper v1");
ad.SetInfoItemValueInteger("modType", modtype::map);
LOG_S(LOG_SECTION_ARCHIVESCANNER, "Found new map: %s", ad.GetNameVersioned().c_str());
} else if (hasModinfo) {
// it is a game
if (ad.GetModType() == modtype::primary) {
AddDependency(ad.GetDependencies(), "Spring content v1");
}
LOG_S(LOG_SECTION_ARCHIVESCANNER, "Found new game: %s", ad.GetNameVersioned().c_str());
} else {
// neither a map nor a mod: error
error = "missing modinfo.lua/mapinfo.lua";
}
ai.path = fpath;
ai.modified = info.st_mtime;
ai.origName = fn;
ai.updated = true;
ai.checksum = (doChecksum) ? GetCRC(fullName) : 0;
archiveInfos[lcfn] = ai;
}
void CArchiveScanner::ScanArchive(const std::string& fullName, bool doChecksum)
{
struct stat info;
stat(fullName.c_str(), &info);
const std::string fn = filesystem.GetFilename(fullName);
const std::string fpath = filesystem.GetDirectory(fullName);
const std::string lcfn = StringToLower(fn);
// Determine whether this archive has earlier be found to be broken
std::map<std::string, BrokenArchive>::iterator bai = brokenArchives.find(lcfn);
if (bai != brokenArchives.end()) {
if ((unsigned)info.st_mtime == bai->second.modified && fpath == bai->second.path) {
bai->second.updated = true;
return;
}
}
// Determine whether to rely on the cached info or not
bool cached = false;
std::map<std::string, ArchiveInfo>::iterator aii = archiveInfo.find(lcfn);
if (aii != archiveInfo.end()) {
// This archive may have been obsoleted, do not process it if so
if (aii->second.replaced.length() > 0) {
return;
}
if ((unsigned)info.st_mtime == aii->second.modified && fpath == aii->second.path) {
cached = true;
aii->second.updated = true;
}
// If we are here, we could have invalid info in the cache
// Force a reread if it's a directory archive, as st_mtime only
// reflects changes to the directory itself, not the contents.
if (!cached) {
archiveInfo.erase(aii);
}
}
// Time to parse the info we are interested in
if (cached) {
// If cached is true, aii will point to the archive
if (doChecksum && (aii->second.checksum == 0))
aii->second.checksum = GetCRC(fullName);
} else {
CArchiveBase* ar = CArchiveFactory::OpenArchive(fullName);
if (ar && ar->IsOpen()) {
ArchiveInfo ai;
std::string mapfile;
bool hasModinfo = false;
bool hasMapinfo = false;
for (unsigned fid = 0; fid != ar->NumFiles(); ++fid)
{
std::string name;
int size;
ar->FileInfo(fid, name, size);
const std::string lowerName = StringToLower(name);
const std::string ext = filesystem.GetExtension(lowerName);
if ((ext == "smf") || (ext == "sm3")) {
mapfile = name;
} else if (lowerName == "modinfo.lua") {
hasModinfo = true;
} else if (lowerName == "mapinfo.lua") {
hasMapinfo = true;
}
}
if (hasMapinfo || !mapfile.empty()) {
// it is a map
if (hasMapinfo) {
ScanArchiveLua(ar, "mapinfo.lua", ai);
} else if (hasModinfo) {
// backwards-compat for modinfo.lua in maps
ScanArchiveLua(ar, "modinfo.lua", ai);
}
if (ai.archiveData.name.empty()) {
ai.archiveData.name = filesystem.GetBasename(mapfile);
}
if (ai.archiveData.mapfile.empty()) {
ai.archiveData.mapfile = mapfile;
}
AddDependency(ai.archiveData.dependencies, "Map Helper v1");
ai.archiveData.modType = modtype::map;
} else if (hasModinfo) {
// it is a mod
ScanArchiveLua(ar, "modinfo.lua", ai);
if (ai.archiveData.modType == modtype::primary)
AddDependency(ai.archiveData.dependencies, "Spring content v1");
} else {
// neither a map nor a mod: error
logOutput.Print(LOG_ARCHIVESCANNER, "Failed to scan %s (missing modinfo.lua/mapinfo.lua)", fullName.c_str());
delete ar;
// record it as broken, so we don't need to look inside everytime
BrokenArchive ba;
ba.path = fpath;
ba.modified = info.st_mtime;
ba.updated = true;
brokenArchives[lcfn] = ba;
return;
}
ai.path = fpath;
ai.modified = info.st_mtime;
ai.origName = fn;
ai.updated = true;
delete ar;
// Optionally calculate a checksum for the file
// To prevent reading all files in all directory (.sdd) archives
// every time this function is called, directory archive checksums
// are calculated on the fly.
if (doChecksum) {
ai.checksum = GetCRC(fullName);
} else {
ai.checksum = 0;
}
archiveInfo[lcfn] = ai;
} else {
logOutput.Print(LOG_ARCHIVESCANNER, "Unable to open archive: %s", fullName.c_str());
}
}
}
void CArchiveScanner::ScanArchive(const std::string& fullName, bool doChecksum)
{
unsigned modifiedTime = 0;
if (CheckCachedData(fullName, &modifiedTime, doChecksum))
return;
isDirty = true;
const std::string& fn = FileSystem::GetFilename(fullName);
const std::string& fpath = FileSystem::GetDirectory(fullName);
const std::string& lcfn = StringToLower(fn);
std::unique_ptr<IArchive> ar(archiveLoader.OpenArchive(fullName));
if (ar == nullptr || !ar->IsOpen()) {
LOG_L(L_WARNING, "[AS::%s] unable to open archive \"%s\"", __func__, fullName.c_str());
// record it as broken, so we don't need to look inside everytime
BrokenArchive& ba = brokenArchives[lcfn];
ba.path = fpath;
ba.modified = modifiedTime;
ba.updated = true;
ba.problem = "Unable to open archive";
// does not count as a scan
// numScannedArchives += 1;
return;
}
std::string error;
std::string arMapFile; // file in archive with "smf" or "sm3" extension
std::string miMapFile; // value for the 'mapfile' key parsed from mapinfo
const bool hasModinfo = ar->FileExists("modinfo.lua");
const bool hasMapinfo = ar->FileExists("mapinfo.lua");
ArchiveInfo ai;
ArchiveData& ad = ai.archiveData;
// execute the respective .lua, otherwise assume this archive is a map
if (hasMapinfo) {
ScanArchiveLua(ar.get(), "mapinfo.lua", ai, error);
if ((miMapFile = ad.GetMapFile()).empty()) {
LOG_L(L_WARNING, "[AS::%s] set the 'mapfile' key in mapinfo.lua of archive \"%s\" for faster loading!", __func__, fullName.c_str());
arMapFile = SearchMapFile(ar.get(), error);
}
} else if (hasModinfo) {
ScanArchiveLua(ar.get(), "modinfo.lua", ai, error);
} else {
arMapFile = SearchMapFile(ar.get(), error);
}
if (!CheckCompression(ar.get(), fullName, error)) {
// for some reason, the archive is marked as broken
LOG_L(L_WARNING, "[AS::%s] failed to scan \"%s\" (%s)", __func__, fullName.c_str(), error.c_str());
// record it as broken, so we don't need to look inside everytime
BrokenArchive& ba = brokenArchives[lcfn];
ba.path = fpath;
ba.modified = modifiedTime;
ba.updated = true;
ba.problem = error;
// does count as a scan
numScannedArchives += 1;
return;
}
if (hasMapinfo || !arMapFile.empty()) {
// map archive
if ((ad.GetName()).empty()) {
// FIXME The name will never be empty, if version is set (see HACK in ArchiveData)
ad.SetInfoItemValueString("name_pure", FileSystem::GetBasename(arMapFile));
ad.SetInfoItemValueString("name", FileSystem::GetBasename(arMapFile));
}
if (miMapFile.empty())
ad.SetInfoItemValueString("mapfile", arMapFile);
AddDependency(ad.GetDependencies(), GetMapHelperContentName());
ad.SetInfoItemValueInteger("modType", modtype::map);
LOG_S(LOG_SECTION_ARCHIVESCANNER, "Found new map: %s", ad.GetNameVersioned().c_str());
} else if (hasModinfo) {
// game or base-type (cursors, bitmaps, ...) archive
// babysitting like this is really no longer required
if (ad.IsGame() || ad.IsMenu())
AddDependency(ad.GetDependencies(), GetSpringBaseContentName());
LOG_S(LOG_SECTION_ARCHIVESCANNER, "Found new game: %s", ad.GetNameVersioned().c_str());
} else {
// neither a map nor a mod: error
error = "missing modinfo.lua/mapinfo.lua";
}
ai.path = fpath;
ai.modified = modifiedTime;
ai.origName = fn;
ai.updated = true;
ai.checksum = (doChecksum) ? GetCRC(fullName) : 0;
archiveInfos[lcfn] = ai;
numScannedArchives += 1;
}
