/**
* Get CRC of the data in the specified archive.
* Returns 0 if file could not be opened.
*/
unsigned int CArchiveScanner::GetCRC(const std::string& arcName)
{
CRC crc;
IArchive* ar;
std::list<std::string> files;
//! Try to open an archive
ar = archiveLoader.OpenArchive(arcName);
if (!ar) {
return 0; // It wasn't an archive
}
//! Load ignore list.
IFileFilter* ignore = CreateIgnoreFilter(ar);
for (unsigned fid = 0; fid != ar->NumFiles(); ++fid) {
std::string name;
int size;
ar->FileInfo(fid, name, size);
if (ignore->Match(name)) {
continue;
}
StringToLowerInPlace(name); //! case insensitive hash
files.push_back(name);
}
files.sort();
//! Add all files in sorted order
for (std::list<std::string>::iterator i = files.begin(); i != files.end(); ++i) {
const unsigned int nameCRC = CRC().Update(i->data(), i->size()).GetDigest();
const unsigned fid = ar->FindFile(*i);
const unsigned int dataCRC = ar->GetCrc32(fid);
crc.Update(nameCRC);
crc.Update(dataCRC);
}
delete ignore;
delete ar;
unsigned int digest = crc.GetDigest();
//! A value of 0 is used to indicate no crc.. so never return that
//! Shouldn't happen all that often
if (digest == 0) {
return 4711;
} else {
return digest;
}
}
bool CFileSystem::extract(const std::string& filename, const std::string& dstdir)
{
#ifdef ARCHIVE_SUPPORT
LOG_INFO("Extracting %s to %s", filename.c_str(), dstdir.c_str());
const int len = filename.length();
IArchive* archive;
if ((len>4) && (filename.compare(len-3, 3,".7z") == 0 ) ) {
archive = new CSevenZipArchive(filename);
} else {
archive = new CZipArchive(filename);
}
const unsigned int num = archive->NumFiles();
for (unsigned int i=0; i<num; i++) {
std::vector<unsigned char> buf;
std::string name;
int size, mode;
archive->FileInfo(i,name, size, mode);
if (!archive->GetFile(i, buf)) {
LOG_ERROR("Error extracting %s from %s", name.c_str(), filename.c_str());
delete archive;
return false;
}
#ifdef WIN32
for(unsigned int i=0; i<name.length(); i++) { //replace / with \ on win32
if (name[i] == '/')
name[i]=PATH_DELIMITER;
}
#endif
std::string tmp = dstdir + PATH_DELIMITER;
tmp += name.c_str(); //FIXME: concating UTF-16
createSubdirs(tmp);
if (fileSystem->fileExists(tmp)) {
LOG_ERROR("File already exists: %s", tmp.c_str());
continue;
}
LOG_INFO("extracting (%s)", tmp.c_str());
FILE* f=fopen(tmp.c_str(), "wb+");
if (f == NULL) {
LOG_ERROR("Error creating %s", tmp.c_str());
delete archive;
return false;
}
int res=1;
if (!buf.empty())
res = fwrite(&buf[0], buf.size(), 1,f);
#ifndef WIN32
fchmod(fileno(f), mode);
#endif
if (res<=0) {
const int err=ferror(f);
LOG_ERROR("fwrite(%s): %d %s",name.c_str(), err, strerror(err));
fclose(f);
delete archive;
return false;
}
fclose(f);
}
delete archive;
LOG_INFO("done");
return true;
#else
LOG_ERROR("no archive support!");
return false;
#endif
}
void CArchiveScanner::ScanArchive(const std::string& fullName, bool doChecksum)
{
struct stat info;
stat(fullName.c_str(), &info);
const std::string fn = filesystem.GetFilename(fullName);
const std::string fpath = filesystem.GetDirectory(fullName);
const std::string lcfn = StringToLower(fn);
//! Determine whether this archive has earlier be found to be broken
std::map<std::string, BrokenArchive>::iterator bai = brokenArchives.find(lcfn);
if (bai != brokenArchives.end()) {
if ((unsigned)info.st_mtime == bai->second.modified && fpath == bai->second.path) {
bai->second.updated = true;
return;
}
}
//! Determine whether to rely on the cached info or not
bool cached = false;
std::map<std::string, ArchiveInfo>::iterator aii = archiveInfo.find(lcfn);
if (aii != archiveInfo.end()) {
//! This archive may have been obsoleted, do not process it if so
if (aii->second.replaced.length() > 0) {
return;
}
if ((unsigned)info.st_mtime == aii->second.modified && fpath == aii->second.path) {
cached = true;
aii->second.updated = true;
}
//! If we are here, we could have invalid info in the cache
//! Force a reread if it's a directory archive, as st_mtime only
//! reflects changes to the directory itself, not the contents.
if (!cached) {
archiveInfo.erase(aii);
}
}
//! Time to parse the info we are interested in
if (cached) {
//! If cached is true, aii will point to the archive
if (doChecksum && (aii->second.checksum == 0))
aii->second.checksum = GetCRC(fullName);
} else {
IArchive* ar = archiveLoader.OpenArchive(fullName);
if (!ar || !ar->IsOpen()) {
logOutput.Print("Unable to open archive: %s", fullName.c_str());
return;
}
ArchiveInfo ai;
std::string error = "";
std::string mapfile;
bool hasModinfo = ar->FileExists("modinfo.lua");
bool hasMapinfo = ar->FileExists("mapinfo.lua");
//! check for smf/sm3 and if the uncompression of important files is too costy
for (unsigned fid = 0; fid != ar->NumFiles(); ++fid)
{
std::string name;
int size;
ar->FileInfo(fid, name, size);
const std::string lowerName = StringToLower(name);
const std::string ext = filesystem.GetExtension(lowerName);
if ((ext == "smf") || (ext == "sm3")) {
mapfile = name;
}
const unsigned char metaFileClass = GetMetaFileClass(lowerName);
if ((metaFileClass != 0) && !(ar->HasLowReadingCost(fid))) {
//! is a meta-file and not cheap to read
if (metaFileClass == 1) {
//! 1st class
error = "Unpacking/reading cost for meta file " + name
+ " is too high, please repack the archive (make sure to use a non-solid algorithm, if applicable)";
break;
} else if (metaFileClass == 2) {
//! 2nd class
logOutput.Print(LOG_ARCHIVESCANNER,
"Warning: Archive %s: The cost for reading a 2nd class meta-file is too high: %s",
fullName.c_str(), name.c_str());
}
}
}
if (!error.empty()) {
//! we already have an error, no further evaluation required
} if (hasMapinfo || !mapfile.empty()) {
//! it is a map
if (hasMapinfo) {
ScanArchiveLua(ar, "mapinfo.lua", ai, error);
} else if (hasModinfo) {
//! backwards-compat for modinfo.lua in maps
ScanArchiveLua(ar, "modinfo.lua", ai, error);
}
if (ai.archiveData.GetName().empty()) {
//! FIXME The name will never be empty, if version is set (see HACK in ArchiveData)
ai.archiveData.SetInfoItemValueString("name", filesystem.GetBasename(mapfile));
}
if (ai.archiveData.GetMapFile().empty()) {
ai.archiveData.SetInfoItemValueString("mapfile", mapfile);
}
AddDependency(ai.archiveData.GetDependencies(), "Map Helper v1");
ai.archiveData.SetInfoItemValueInteger("modType", modtype::map);
logOutput.Print(LOG_ARCHIVESCANNER, "Found new map: %s",
ai.archiveData.GetName().c_str());
} else if (hasModinfo) {
//! it is a mod
ScanArchiveLua(ar, "modinfo.lua", ai, error);
if (ai.archiveData.GetModType() == modtype::primary) {
AddDependency(ai.archiveData.GetDependencies(), "Spring content v1");
}
logOutput.Print(LOG_ARCHIVESCANNER, "Found new game: %s",
ai.archiveData.GetName().c_str());
} else {
//! neither a map nor a mod: error
error = "missing modinfo.lua/mapinfo.lua";
}
delete ar;
if (!error.empty()) {
//! for some reason, the archive is marked as broken
logOutput.Print("Failed to scan %s (%s)",
fullName.c_str(), error.c_str());
//! record it as broken, so we don't need to look inside everytime
BrokenArchive ba;
ba.path = fpath;
ba.modified = info.st_mtime;
ba.updated = true;
ba.problem = error;
brokenArchives[lcfn] = ba;
return;
}
ai.path = fpath;
ai.modified = info.st_mtime;
ai.origName = fn;
ai.updated = true;
//! Optionally calculate a checksum for the file
//! To prevent reading all files in all directory (.sdd) archives
//! every time this function is called, directory archive checksums
//! are calculated on the fly.
if (doChecksum) {
ai.checksum = GetCRC(fullName);
} else {
ai.checksum = 0;
}
archiveInfo[lcfn] = ai;
}
}
/**
* Get CRC of the data in the specified archive.
* Returns 0 if file could not be opened.
*/
unsigned int CArchiveScanner::GetCRC(const std::string& arcName)
{
CRC crc;
IArchive* ar;
std::list<std::string> files;
// Try to open an archive
ar = archiveLoader.OpenArchive(arcName);
if (!ar) {
return 0; // It wasn't an archive
}
// Load ignore list.
IFileFilter* ignore = CreateIgnoreFilter(ar);
// Insert all files to check in lowercase format
for (unsigned fid = 0; fid != ar->NumFiles(); ++fid) {
std::string name;
int size;
ar->FileInfo(fid, name, size);
if (ignore->Match(name)) {
continue;
}
StringToLowerInPlace(name); // case insensitive hash
files.push_back(name);
}
// Sort by FileName
files.sort();
// Push the filenames into a std::vector, cause OMP can better iterate over those
std::vector<CRCPair> crcs;
crcs.reserve(files.size());
CRCPair crcp;
for (std::list<std::string>::iterator it = files.begin(); it != files.end(); ++it) {
crcp.filename = &(*it);
crcs.push_back(crcp);
}
// Compute CRCs of the files
// Hint: Multithreading only speedups `.sdd` loading. For those the CRC generation is extremely slow -
// it has to load the full file to calc it! For the other formats (sd7, sdz, sdp) the CRC is saved
// in the metainformation of the container and so the loading is much faster. Neither does any of our
// current (2011) packing libraries support multithreading :/
for_mt(0, crcs.size(), [&](const int i) {
CRCPair& crcp = crcs[i];
const unsigned int nameCRC = CRC().Update(crcp.filename->data(), crcp.filename->size()).GetDigest();
const unsigned fid = ar->FindFile(*crcp.filename);
const unsigned int dataCRC = ar->GetCrc32(fid);
crcp.nameCRC = nameCRC;
crcp.dataCRC = dataCRC;
#if !defined(DEDICATED) && !defined(UNITSYNC)
Watchdog::ClearTimer(WDT_MAIN);
#endif
});
// Add file CRCs to the main archive CRC
for (std::vector<CRCPair>::iterator it = crcs.begin(); it != crcs.end(); ++it) {
crc.Update(it->nameCRC);
crc.Update(it->dataCRC);
#if !defined(DEDICATED) && !defined(UNITSYNC)
Watchdog::ClearTimer();
#endif
}
delete ignore;
delete ar;
unsigned int digest = crc.GetDigest();
// A value of 0 is used to indicate no crc.. so never return that
// Shouldn't happen all that often
if (digest == 0) {
return 4711;
} else {
return digest;
}
}