/** Get CRC of the data in the specified archive. Returns 0 if file could not be opened. */ unsigned int CArchiveScanner::GetCRC(const string& arcName) { CRC crc; CArchiveBase* ar; std::list<string> files; // Try to open an archive ar = CArchiveFactory::OpenArchive(arcName); if (!ar) { return 0; // It wasn't an archive } // Load ignore list. IFileFilter* ignore = CreateIgnoreFilter(ar); string name; int size; // Sort all file paths for deterministic behaviour for (int cur = 0; (cur = ar->FindFiles(cur, &name, &size)); /* no-op */) { if (ignore->Match(name)) { continue; } const string lower = StringToLower(name); // case insensitive hash files.push_back(lower); } files.sort(); // Add all files in sorted order for (std::list<string>::iterator i = files.begin(); i != files.end(); i++ ) { const unsigned int nameCRC = CRC().Update(i->data(), i->size()).GetDigest(); const unsigned int dataCRC = ar->GetCrc32(*i); crc.Update(nameCRC); crc.Update(dataCRC); } delete ignore; delete ar; unsigned int digest = crc.GetDigest(); // A value of 0 is used to indicate no crc.. so never return that // Shouldn't happen all that often if (digest == 0) { return 4711; } else { return digest; } }
void CArchiveScanner::Scan(const string& curPath, bool checksum) { InitCrcTable(); isDirty = true; const int flags = (FileSystem::INCLUDE_DIRS | FileSystem::RECURSE); std::vector<std::string> found = filesystem.FindFiles(curPath, "*", flags); for (std::vector<std::string>::iterator it = found.begin(); it != found.end(); ++it) { string fullName = *it; // Strip const char lastFullChar = fullName[fullName.size() - 1]; if ((lastFullChar == '/') || (lastFullChar == '\\')) { fullName = fullName.substr(0, fullName.size() - 1); } const string fn = filesystem.GetFilename(fullName); const string fpath = filesystem.GetDirectory(fullName); const string lcfn = StringToLower(fn); const string lcfpath = StringToLower(fpath); // Exclude archivefiles found inside directory archives (.sdd) if (lcfpath.find(".sdd") != string::npos) { continue; } // Exclude archivefiles found inside hidden directories if ((lcfpath.find("/hidden/") != string::npos) || (lcfpath.find("\\hidden\\") != string::npos)) { continue; } // Is this an archive we should look into? if (CArchiveFactory::IsArchive(fullName)) { struct stat info; stat(fullName.c_str(), &info); // Determine whether to rely on the cached info or not bool cached = false; map<string, ArchiveInfo>::iterator aii = archiveInfo.find(lcfn); if (aii != archiveInfo.end()) { // This archive may have been obsoleted, do not process it if so if (aii->second.replaced.length() > 0) continue; /* For truely correct updating of .sdd archives, this code should be enabled. Unfortunately it has as side effect that all files in all .sdd's always need to be stat()'ed, which really slows down program startup. An update can be forced anyway by removing ArchiveCacheV*.txt or renaming the archive. */ /*if (S_ISDIR(info.st_mode)) { struct stat info2; std::vector<std::string> sddfiles = filesystem.FindFiles(fpath, "*", FileSystem::RECURSE | FileSystem::INCLUDE_DIRS); for (std::vector<std::string>::iterator sddit = found.begin(); sddit != found.end(); ++sddit) { stat(sddit->c_str(), &info2); if (info.st_mtime < info2.st_mtime) { info.st_mtime = info2.st_mtime; } } }*/ if ((unsigned)info.st_mtime == aii->second.modified && fpath == aii->second.path) { cached = true; aii->second.updated = true; } // If we are here, we could have invalid info in the cache // Force a reread if it's a directory archive, as st_mtime only // reflects changes to the directory itself, not the contents. if (!cached) { archiveInfo.erase(aii); } } // Time to parse the info we are interested in if (!cached) { //printf("scanning archive: %s\n", fullName.c_str()); CArchiveBase* ar = CArchiveFactory::OpenArchive(fullName); if (ar) { int cur; string name; int size; ArchiveInfo ai; cur = ar->FindFiles(0, &name, &size); while (cur != 0) { //printf("found %s %d\n", name.c_str(), size); string ext = StringToLower(name.substr(name.find_last_of('.') + 1)); // only accept new format maps if (ext == "smf" || ext == "sm3") { MapData md; if (name.find_last_of('\\') == string::npos && name.find_last_of('/') == string::npos) { md.name = name; md.virtualPath = "/"; } else { if (name.find_last_of('\\') == string::npos) { md.name = name.substr(name.find_last_of('/') + 1); md.virtualPath = name.substr(0, name.find_last_of('/') + 1); // include the backslash } else { md.name = name.substr(name.find_last_of('\\') + 1); md.virtualPath = name.substr(0, name.find_last_of('\\') + 1); // include the backslash } //md.name = md.name.substr(0, md.name.find_last_of('.')); } ai.mapData.push_back(md); } if (name == "modinfo.tdf") { int fh = ar->OpenFile(name); if (fh) { int fsize = ar->FileSize(fh); char* buf = SAFE_NEW char[fsize]; ar->ReadFile(fh, buf, fsize); ar->CloseFile(fh); try { TdfParser p( buf, fsize ); ai.modData = GetModData(&p, "mod"); } catch (const TdfParser::parse_error&) { // Silently ignore mods with parse errors } delete [] buf; } } cur = ar->FindFiles(cur, &name, &size); } ai.path = fpath; ai.modified = info.st_mtime; ai.origName = fn; ai.updated = true; delete ar; // Optionally calculate a checksum for the file // To prevent reading all files in all directory (.sdd) archives every time this function // is called, directory archive checksums are calculated on the fly. if (checksum) { ai.checksum = GetCRC(fullName); } else { ai.checksum = 0; } archiveInfo[lcfn] = ai; } } else { // If cached is true, aii will point to the archive if ((checksum) && (aii->second.checksum == 0)) {
void CArchiveScanner::Scan(const string& curPath, bool checksum) { isDirty = true; std::vector<std::string> found = filesystem.FindFiles(curPath, "*", FileSystem::RECURSE | FileSystem::INCLUDE_DIRS); struct stat info; for (std::vector<std::string>::iterator it = found.begin(); it != found.end(); ++it) { stat(it->c_str(),&info); string fullName = *it; string fn = filesystem.GetFilename(fullName); string fpath = filesystem.GetDirectory(fullName); string lcfn = StringToLower(fn); string lcfpath = StringToLower(fpath); // Exclude archivefiles found inside directory (.sdd) archives. string::size_type sdd = lcfpath.find(".sdd"); if (sdd != string::npos) continue; // Is this an archive we should look into? if (CArchiveFactory::IsArchive(fullName)) { // Determine whether to rely on the cached info or not bool cached = false; map<string, ArchiveInfo>::iterator aii = archiveInfo.find(lcfn); if (aii != archiveInfo.end()) { // This archive may have been obsoleted, do not process it if so if (aii->second.replaced.length() > 0) continue; if (S_ISDIR(info.st_mode)) { struct stat info2; std::vector<std::string> sddfiles = filesystem.FindFiles(fpath, "*", FileSystem::RECURSE | FileSystem::INCLUDE_DIRS); for (std::vector<std::string>::iterator sddit = found.begin(); sddit != found.end(); ++sddit) { stat(sddit->c_str(), &info2); if (info.st_mtime < info2.st_mtime) { info.st_mtime = info2.st_mtime; } } } if (info.st_mtime == aii->second.modified && fpath == aii->second.path) { cached = true; aii->second.updated = true; } // If we are here, we could have invalid info in the cache // Force a reread if it's a directory archive, as st_mtime only // reflects changes to the directory itself, not the contents. if (!cached) { archiveInfo.erase(aii); } } // Time to parse the info we are interested in if (!cached) { CArchiveBase* ar = CArchiveFactory::OpenArchive(fullName); if (ar) { int cur; string name; int size; ArchiveInfo ai; cur = ar->FindFiles(0, &name, &size); while (cur != 0) { //printf("found %s %d\n", name.c_str(), size); string ext = StringToLower(name.substr(name.find_last_of('.') + 1)); // only accept new format maps if (ext == "smf" || ext == "sm3") { MapData md; if (name.find_last_of('\\') == string::npos && name.find_last_of('/') == string::npos) { md.name = name; md.virtualPath = "/"; } else { if (name.find_last_of('\\') == string::npos) { md.name = name.substr(name.find_last_of('/') + 1); md.virtualPath = name.substr(0, name.find_last_of('/') + 1); // include the backslash } else { md.name = name.substr(name.find_last_of('\\') + 1); md.virtualPath = name.substr(0, name.find_last_of('\\') + 1); // include the backslash } //md.name = md.name.substr(0, md.name.find_last_of('.')); } ai.mapData.push_back(md); } if (name == "modinfo.tdf") { int fh = ar->OpenFile(name); if (fh) { int fsize = ar->FileSize(fh); void* buf = malloc(fsize); ar->ReadFile(fh, buf, fsize); ar->CloseFile(fh); try { TdfParser p( reinterpret_cast<char*>(buf), fsize ); ai.modData = GetModData(&p, "mod"); } catch (const TdfParser::parse_error& e) { // Silently ignore mods with parse errors } free(buf); } } cur = ar->FindFiles(cur, &name, &size); } ai.path = fpath; ai.modified = info.st_mtime; ai.origName = fn; ai.updated = true; delete ar; // Optionally calculate a checksum for the file // To prevent reading all files in all directory (.sdd) archives every time this function // is called, directory archive checksums are calculated on the fly. if (checksum) { ai.checksum = GetCRC(fullName); } else { ai.checksum = 0; } archiveInfo[lcfn] = ai; } } else { // If cached is true, aii will point to the archive if ((checksum) && (aii->second.checksum == 0)) { aii->second.checksum = GetCRC(fullName); } } } } // Now we'll have to parse the replaces-stuff found in the mods for (map<string, ArchiveInfo>::iterator aii = archiveInfo.begin(); aii != archiveInfo.end(); ++aii) { for (vector<string>::iterator i = aii->second.modData.replaces.begin(); i != aii->second.modData.replaces.end(); ++i) { string lcname = StringToLower(*i); map<string, ArchiveInfo>::iterator ar = archiveInfo.find(lcname); // If it's not there, we will create a new entry if (ar == archiveInfo.end()) { ArchiveInfo tmp; archiveInfo[lcname] = tmp; ar = archiveInfo.find(lcname); } // Overwrite the info for this archive with a replaced pointer ar->second.path = ""; ar->second.origName = lcname; ar->second.modified = 1; ar->second.mapData.clear(); ar->second.modData.name = ""; ar->second.modData.replaces.clear(); ar->second.updated = true; ar->second.replaced = aii->first; } } }
void CArchiveScanner::ScanArchive(const string& fullName, bool doChecksum) { struct stat info; stat(fullName.c_str(), &info); const string fn = filesystem.GetFilename(fullName); const string fpath = filesystem.GetDirectory(fullName); const string lcfn = StringToLower(fn); const string lcfpath = StringToLower(fpath); // Determine whether to rely on the cached info or not bool cached = false; std::map<string, ArchiveInfo>::iterator aii = archiveInfo.find(lcfn); if (aii != archiveInfo.end()) { // This archive may have been obsoleted, do not process it if so if (aii->second.replaced.length() > 0) return; if ((unsigned)info.st_mtime == aii->second.modified && fpath == aii->second.path) { cached = true; aii->second.updated = true; } // If we are here, we could have invalid info in the cache // Force a reread if it's a directory archive, as st_mtime only // reflects changes to the directory itself, not the contents. if (!cached) { archiveInfo.erase(aii); } } // Time to parse the info we are interested in if (cached) { // If cached is true, aii will point to the archive if (doChecksum && (aii->second.checksum == 0)) aii->second.checksum = GetCRC(fullName); } else { CArchiveBase* ar = CArchiveFactory::OpenArchive(fullName); if (ar) { ArchiveInfo ai; string name; int size; std::string mapfile; bool hasModinfo = false; bool hasMapinfo = false; for (int cur = 0; (cur = ar->FindFiles(cur, &name, &size)); /* no-op */) { const string lowerName = StringToLower(name); const string ext = lowerName.substr(lowerName.find_last_of('.') + 1); if ((ext == "smf") || (ext == "sm3")) { mapfile = name; } else if (lowerName == "modinfo.lua") { hasModinfo = true; } else if (lowerName == "mapinfo.lua") { hasMapinfo = true; } } if (hasMapinfo || !mapfile.empty()) { // its a map if (hasMapinfo) { ScanArchiveLua(ar, "mapinfo.lua", ai); } else if (hasModinfo) // backwards-compat for modinfo.lua in maps { ScanArchiveLua(ar, "modinfo.lua", ai); } if (ai.archiveData.name.empty()) ai.archiveData.name = filesystem.GetFilename(mapfile); if (ai.archiveData.mapfile.empty()) ai.archiveData.mapfile = mapfile; AddDependency(ai.archiveData.dependencies, "maphelper.sdz"); ai.archiveData.modType = modtype::map; } else if (hasModinfo) { // mod ScanArchiveLua(ar, "modinfo.lua", ai); if (ai.archiveData.modType == modtype::primary) AddDependency(ai.archiveData.dependencies, "Spring content v1"); } else { // error LogObject() << "Failed to read archive, files missing: " << fullName; delete ar; return; } ai.path = fpath; ai.modified = info.st_mtime; ai.origName = fn; ai.updated = true; delete ar; // Optionally calculate a checksum for the file // To prevent reading all files in all directory (.sdd) archives // every time this function is called, directory archive checksums // are calculated on the fly. if (doChecksum) { ai.checksum = GetCRC(fullName); } else { ai.checksum = 0; } archiveInfo[lcfn] = ai; } } }