/** Get CRC of the data in the specified archive.
Returns 0 if file could not be opened. */
unsigned int CArchiveScanner::GetCRC(const string& arcName)
{
CRC crc;
CArchiveBase* ar;
std::list<string> files;
// Try to open an archive
ar = CArchiveFactory::OpenArchive(arcName);
if (!ar) {
return 0; // It wasn't an archive
}
// Load ignore list.
IFileFilter* ignore = CreateIgnoreFilter(ar);
string name;
int size;
// Sort all file paths for deterministic behaviour
for (int cur = 0; (cur = ar->FindFiles(cur, &name, &size)); /* no-op */) {
if (ignore->Match(name)) {
continue;
}
const string lower = StringToLower(name); // case insensitive hash
files.push_back(lower);
}
files.sort();
// Add all files in sorted order
for (std::list<string>::iterator i = files.begin(); i != files.end(); i++ ) {
const unsigned int nameCRC = CRC().Update(i->data(), i->size()).GetDigest();
const unsigned int dataCRC = ar->GetCrc32(*i);
crc.Update(nameCRC);
crc.Update(dataCRC);
}
delete ignore;
delete ar;
unsigned int digest = crc.GetDigest();
// A value of 0 is used to indicate no crc.. so never return that
// Shouldn't happen all that often
if (digest == 0) {
return 4711;
} else {
return digest;
}
}
void CArchiveScanner::Scan(const string& curPath, bool checksum)
{
InitCrcTable();
isDirty = true;
const int flags = (FileSystem::INCLUDE_DIRS | FileSystem::RECURSE);
std::vector<std::string> found = filesystem.FindFiles(curPath, "*", flags);
for (std::vector<std::string>::iterator it = found.begin(); it != found.end(); ++it) {
string fullName = *it;
// Strip
const char lastFullChar = fullName[fullName.size() - 1];
if ((lastFullChar == '/') || (lastFullChar == '\\')) {
fullName = fullName.substr(0, fullName.size() - 1);
}
const string fn = filesystem.GetFilename(fullName);
const string fpath = filesystem.GetDirectory(fullName);
const string lcfn = StringToLower(fn);
const string lcfpath = StringToLower(fpath);
// Exclude archivefiles found inside directory archives (.sdd)
if (lcfpath.find(".sdd") != string::npos) {
continue;
}
// Exclude archivefiles found inside hidden directories
if ((lcfpath.find("/hidden/") != string::npos) ||
(lcfpath.find("\\hidden\\") != string::npos)) {
continue;
}
// Is this an archive we should look into?
if (CArchiveFactory::IsArchive(fullName)) {
struct stat info;
stat(fullName.c_str(), &info);
// Determine whether to rely on the cached info or not
bool cached = false;
map<string, ArchiveInfo>::iterator aii = archiveInfo.find(lcfn);
if (aii != archiveInfo.end()) {
// This archive may have been obsoleted, do not process it if so
if (aii->second.replaced.length() > 0)
continue;
/*
For truely correct updating of .sdd archives, this code should
be enabled. Unfortunately it has as side effect that all files
in all .sdd's always need to be stat()'ed, which really slows
down program startup.
An update can be forced anyway by removing ArchiveCacheV*.txt
or renaming the archive.
*/
/*if (S_ISDIR(info.st_mode)) {
struct stat info2;
std::vector<std::string> sddfiles = filesystem.FindFiles(fpath, "*", FileSystem::RECURSE | FileSystem::INCLUDE_DIRS);
for (std::vector<std::string>::iterator sddit = found.begin(); sddit != found.end(); ++sddit) {
stat(sddit->c_str(), &info2);
if (info.st_mtime < info2.st_mtime) {
info.st_mtime = info2.st_mtime;
}
}
}*/
if ((unsigned)info.st_mtime == aii->second.modified && fpath == aii->second.path) {
cached = true;
aii->second.updated = true;
}
// If we are here, we could have invalid info in the cache
// Force a reread if it's a directory archive, as st_mtime only
// reflects changes to the directory itself, not the contents.
if (!cached) {
archiveInfo.erase(aii);
}
}
// Time to parse the info we are interested in
if (!cached) {
//printf("scanning archive: %s\n", fullName.c_str());
CArchiveBase* ar = CArchiveFactory::OpenArchive(fullName);
if (ar) {
int cur;
string name;
int size;
ArchiveInfo ai;
cur = ar->FindFiles(0, &name, &size);
while (cur != 0) {
//printf("found %s %d\n", name.c_str(), size);
string ext = StringToLower(name.substr(name.find_last_of('.') + 1));
// only accept new format maps
if (ext == "smf" || ext == "sm3") {
MapData md;
if (name.find_last_of('\\') == string::npos && name.find_last_of('/') == string::npos) {
md.name = name;
md.virtualPath = "/";
}
else {
if (name.find_last_of('\\') == string::npos) {
md.name = name.substr(name.find_last_of('/') + 1);
md.virtualPath = name.substr(0, name.find_last_of('/') + 1); // include the backslash
} else {
md.name = name.substr(name.find_last_of('\\') + 1);
md.virtualPath = name.substr(0, name.find_last_of('\\') + 1); // include the backslash
}
//md.name = md.name.substr(0, md.name.find_last_of('.'));
}
ai.mapData.push_back(md);
}
if (name == "modinfo.tdf") {
int fh = ar->OpenFile(name);
if (fh) {
int fsize = ar->FileSize(fh);
char* buf = SAFE_NEW char[fsize];
ar->ReadFile(fh, buf, fsize);
ar->CloseFile(fh);
try {
TdfParser p( buf, fsize );
ai.modData = GetModData(&p, "mod");
} catch (const TdfParser::parse_error&) {
// Silently ignore mods with parse errors
}
delete [] buf;
}
}
cur = ar->FindFiles(cur, &name, &size);
}
ai.path = fpath;
ai.modified = info.st_mtime;
ai.origName = fn;
ai.updated = true;
delete ar;
// Optionally calculate a checksum for the file
// To prevent reading all files in all directory (.sdd) archives every time this function
// is called, directory archive checksums are calculated on the fly.
if (checksum) {
ai.checksum = GetCRC(fullName);
}
else {
ai.checksum = 0;
}
archiveInfo[lcfn] = ai;
}
}
else {
// If cached is true, aii will point to the archive
if ((checksum) && (aii->second.checksum == 0)) {
void CArchiveScanner::Scan(const string& curPath, bool checksum)
{
isDirty = true;
std::vector<std::string> found = filesystem.FindFiles(curPath, "*", FileSystem::RECURSE | FileSystem::INCLUDE_DIRS);
struct stat info;
for (std::vector<std::string>::iterator it = found.begin(); it != found.end(); ++it) {
stat(it->c_str(),&info);
string fullName = *it;
string fn = filesystem.GetFilename(fullName);
string fpath = filesystem.GetDirectory(fullName);
string lcfn = StringToLower(fn);
string lcfpath = StringToLower(fpath);
// Exclude archivefiles found inside directory (.sdd) archives.
string::size_type sdd = lcfpath.find(".sdd");
if (sdd != string::npos)
continue;
// Is this an archive we should look into?
if (CArchiveFactory::IsArchive(fullName)) {
// Determine whether to rely on the cached info or not
bool cached = false;
map<string, ArchiveInfo>::iterator aii = archiveInfo.find(lcfn);
if (aii != archiveInfo.end()) {
// This archive may have been obsoleted, do not process it if so
if (aii->second.replaced.length() > 0)
continue;
if (S_ISDIR(info.st_mode)) {
struct stat info2;
std::vector<std::string> sddfiles = filesystem.FindFiles(fpath, "*", FileSystem::RECURSE | FileSystem::INCLUDE_DIRS);
for (std::vector<std::string>::iterator sddit = found.begin(); sddit != found.end(); ++sddit) {
stat(sddit->c_str(), &info2);
if (info.st_mtime < info2.st_mtime) {
info.st_mtime = info2.st_mtime;
}
}
}
if (info.st_mtime == aii->second.modified && fpath == aii->second.path) {
cached = true;
aii->second.updated = true;
}
// If we are here, we could have invalid info in the cache
// Force a reread if it's a directory archive, as st_mtime only
// reflects changes to the directory itself, not the contents.
if (!cached) {
archiveInfo.erase(aii);
}
}
// Time to parse the info we are interested in
if (!cached) {
CArchiveBase* ar = CArchiveFactory::OpenArchive(fullName);
if (ar) {
int cur;
string name;
int size;
ArchiveInfo ai;
cur = ar->FindFiles(0, &name, &size);
while (cur != 0) {
//printf("found %s %d\n", name.c_str(), size);
string ext = StringToLower(name.substr(name.find_last_of('.') + 1));
// only accept new format maps
if (ext == "smf" || ext == "sm3") {
MapData md;
if (name.find_last_of('\\') == string::npos && name.find_last_of('/') == string::npos) {
md.name = name;
md.virtualPath = "/";
}
else {
if (name.find_last_of('\\') == string::npos) {
md.name = name.substr(name.find_last_of('/') + 1);
md.virtualPath = name.substr(0, name.find_last_of('/') + 1); // include the backslash
} else {
md.name = name.substr(name.find_last_of('\\') + 1);
md.virtualPath = name.substr(0, name.find_last_of('\\') + 1); // include the backslash
}
//md.name = md.name.substr(0, md.name.find_last_of('.'));
}
ai.mapData.push_back(md);
}
if (name == "modinfo.tdf") {
int fh = ar->OpenFile(name);
if (fh) {
int fsize = ar->FileSize(fh);
void* buf = malloc(fsize);
ar->ReadFile(fh, buf, fsize);
ar->CloseFile(fh);
try {
TdfParser p( reinterpret_cast<char*>(buf), fsize );
ai.modData = GetModData(&p, "mod");
} catch (const TdfParser::parse_error& e) {
// Silently ignore mods with parse errors
}
free(buf);
}
}
cur = ar->FindFiles(cur, &name, &size);
}
ai.path = fpath;
ai.modified = info.st_mtime;
ai.origName = fn;
ai.updated = true;
delete ar;
// Optionally calculate a checksum for the file
// To prevent reading all files in all directory (.sdd) archives every time this function
// is called, directory archive checksums are calculated on the fly.
if (checksum) {
ai.checksum = GetCRC(fullName);
}
else {
ai.checksum = 0;
}
archiveInfo[lcfn] = ai;
}
}
else {
// If cached is true, aii will point to the archive
if ((checksum) && (aii->second.checksum == 0)) {
aii->second.checksum = GetCRC(fullName);
}
}
}
}
// Now we'll have to parse the replaces-stuff found in the mods
for (map<string, ArchiveInfo>::iterator aii = archiveInfo.begin(); aii != archiveInfo.end(); ++aii) {
for (vector<string>::iterator i = aii->second.modData.replaces.begin(); i != aii->second.modData.replaces.end(); ++i) {
string lcname = StringToLower(*i);
map<string, ArchiveInfo>::iterator ar = archiveInfo.find(lcname);
// If it's not there, we will create a new entry
if (ar == archiveInfo.end()) {
ArchiveInfo tmp;
archiveInfo[lcname] = tmp;
ar = archiveInfo.find(lcname);
}
// Overwrite the info for this archive with a replaced pointer
ar->second.path = "";
ar->second.origName = lcname;
ar->second.modified = 1;
ar->second.mapData.clear();
ar->second.modData.name = "";
ar->second.modData.replaces.clear();
ar->second.updated = true;
ar->second.replaced = aii->first;
}
}
}
void CArchiveScanner::ScanArchive(const string& fullName, bool doChecksum)
{
struct stat info;
stat(fullName.c_str(), &info);
const string fn = filesystem.GetFilename(fullName);
const string fpath = filesystem.GetDirectory(fullName);
const string lcfn = StringToLower(fn);
const string lcfpath = StringToLower(fpath);
// Determine whether to rely on the cached info or not
bool cached = false;
std::map<string, ArchiveInfo>::iterator aii = archiveInfo.find(lcfn);
if (aii != archiveInfo.end()) {
// This archive may have been obsoleted, do not process it if so
if (aii->second.replaced.length() > 0)
return;
if ((unsigned)info.st_mtime == aii->second.modified && fpath == aii->second.path) {
cached = true;
aii->second.updated = true;
}
// If we are here, we could have invalid info in the cache
// Force a reread if it's a directory archive, as st_mtime only
// reflects changes to the directory itself, not the contents.
if (!cached) {
archiveInfo.erase(aii);
}
}
// Time to parse the info we are interested in
if (cached)
{
// If cached is true, aii will point to the archive
if (doChecksum && (aii->second.checksum == 0))
aii->second.checksum = GetCRC(fullName);
}
else
{
CArchiveBase* ar = CArchiveFactory::OpenArchive(fullName);
if (ar)
{
ArchiveInfo ai;
string name;
int size;
std::string mapfile;
bool hasModinfo = false;
bool hasMapinfo = false;
for (int cur = 0; (cur = ar->FindFiles(cur, &name, &size)); /* no-op */)
{
const string lowerName = StringToLower(name);
const string ext = lowerName.substr(lowerName.find_last_of('.') + 1);
if ((ext == "smf") || (ext == "sm3"))
{
mapfile = name;
}
else if (lowerName == "modinfo.lua")
{
hasModinfo = true;
}
else if (lowerName == "mapinfo.lua")
{
hasMapinfo = true;
}
}
if (hasMapinfo || !mapfile.empty())
{ // its a map
if (hasMapinfo)
{
ScanArchiveLua(ar, "mapinfo.lua", ai);
}
else if (hasModinfo) // backwards-compat for modinfo.lua in maps
{
ScanArchiveLua(ar, "modinfo.lua", ai);
}
if (ai.archiveData.name.empty())
ai.archiveData.name = filesystem.GetFilename(mapfile);
if (ai.archiveData.mapfile.empty())
ai.archiveData.mapfile = mapfile;
AddDependency(ai.archiveData.dependencies, "maphelper.sdz");
ai.archiveData.modType = modtype::map;
}
else if (hasModinfo)
{ // mod
ScanArchiveLua(ar, "modinfo.lua", ai);
if (ai.archiveData.modType == modtype::primary)
AddDependency(ai.archiveData.dependencies, "Spring content v1");
}
else
{ // error
LogObject() << "Failed to read archive, files missing: " << fullName;
delete ar;
return;
}
ai.path = fpath;
ai.modified = info.st_mtime;
ai.origName = fn;
ai.updated = true;
delete ar;
// Optionally calculate a checksum for the file
// To prevent reading all files in all directory (.sdd) archives
// every time this function is called, directory archive checksums
// are calculated on the fly.
if (doChecksum) {
ai.checksum = GetCRC(fullName);
} else {
ai.checksum = 0;
}
archiveInfo[lcfn] = ai;
}
}
}