IFileFilter* CArchiveScanner::CreateIgnoreFilter(IArchive* ar)
{
IFileFilter* ignore = IFileFilter::Create();
std::vector<boost::uint8_t> buf;
if (ar->GetFile("springignore.txt", buf) && !buf.empty()) {
// this automatically splits lines
ignore->AddRule(std::string((char*)(&buf[0]), buf.size()));
}
return ignore;
}
/**
* Get CRC of the data in the specified archive.
* Returns 0 if file could not be opened.
*/
unsigned int CArchiveScanner::GetCRC(const std::string& arcName)
{
CRC crc;
CArchiveBase* ar;
std::list<std::string> files;
//! Try to open an archive
ar = CArchiveFactory::OpenArchive(arcName);
if (!ar) {
return 0; // It wasn't an archive
}
//! Load ignore list.
IFileFilter* ignore = CreateIgnoreFilter(ar);
for (unsigned fid = 0; fid != ar->NumFiles(); ++fid) {
std::string name;
int size;
ar->FileInfo(fid, name, size);
if (ignore->Match(name)) {
continue;
}
StringToLowerInPlace(name); //! case insensitive hash
files.push_back(name);
}
files.sort();
//! Add all files in sorted order
for (std::list<std::string>::iterator i = files.begin(); i != files.end(); ++i) {
const unsigned int nameCRC = CRC().Update(i->data(), i->size()).GetDigest();
const unsigned fid = ar->FindFile(*i);
const unsigned int dataCRC = ar->GetCrc32(fid);
crc.Update(nameCRC);
crc.Update(dataCRC);
}
delete ignore;
delete ar;
unsigned int digest = crc.GetDigest();
//! A value of 0 is used to indicate no crc.. so never return that
//! Shouldn't happen all that often
if (digest == 0) {
return 4711;
} else {
return digest;
}
}
IFileFilter* CArchiveScanner::CreateIgnoreFilter(CArchiveBase* ar)
{
IFileFilter* ignore = IFileFilter::Create();
std::vector<boost::uint8_t> buf;
if (ar->GetFile("springignore.txt", buf)) {
//! this automatically splits lines
if (!buf.empty()) {
const std::string cleanbuf((char*)(&buf[0]), buf.size());
ignore->AddRule(cleanbuf);
}
}
return ignore;
}
/** Get CRC of the data in the specified archive.
Returns 0 if file could not be opened. */
unsigned int CArchiveScanner::GetCRC(const string& arcName)
{
CRC crc;
CArchiveBase* ar;
std::list<string> files;
// Try to open an archive
ar = CArchiveFactory::OpenArchive(arcName);
if (!ar) {
return 0; // It wasn't an archive
}
// Load ignore list.
IFileFilter* ignore = CreateIgnoreFilter(ar);
string name;
int size;
// Sort all file paths for deterministic behaviour
for (int cur = 0; (cur = ar->FindFiles(cur, &name, &size)); /* no-op */) {
if (ignore->Match(name)) {
continue;
}
const string lower = StringToLower(name); // case insensitive hash
files.push_back(lower);
}
files.sort();
// Add all files in sorted order
for (std::list<string>::iterator i = files.begin(); i != files.end(); i++ ) {
const unsigned int nameCRC = CRC().Update(i->data(), i->size()).GetDigest();
const unsigned int dataCRC = ar->GetCrc32(*i);
crc.Update(nameCRC);
crc.Update(dataCRC);
}
delete ignore;
delete ar;
unsigned int digest = crc.GetDigest();
// A value of 0 is used to indicate no crc.. so never return that
// Shouldn't happen all that often
if (digest == 0) {
return 4711;
} else {
return digest;
}
}
IFileFilter* CArchiveScanner::CreateIgnoreFilter(CArchiveBase* ar)
{
IFileFilter* ignore = IFileFilter::Create();
int fh = ar->OpenFile("springignore.txt");
if (fh) {
const int fsize = ar->FileSize(fh);
char* buf = new char[fsize];
const int read = ar->ReadFile(fh, buf, fsize);
ar->CloseFile(fh);
// this automatically splits lines
if (read > 0)
ignore->AddRule(string(buf, read));
//TODO: figure out why read != fsize sometimes
delete[] buf;
}
return ignore;
}
/**
* Get CRC of the data in the specified archive.
* Returns 0 if file could not be opened.
*/
unsigned int CArchiveScanner::GetCRC(const std::string& arcName)
{
CRC crc;
IArchive* ar;
std::list<std::string> files;
// Try to open an archive
ar = archiveLoader.OpenArchive(arcName);
if (!ar) {
return 0; // It wasn't an archive
}
// Load ignore list.
IFileFilter* ignore = CreateIgnoreFilter(ar);
// Insert all files to check in lowercase format
for (unsigned fid = 0; fid != ar->NumFiles(); ++fid) {
std::string name;
int size;
ar->FileInfo(fid, name, size);
if (ignore->Match(name)) {
continue;
}
StringToLowerInPlace(name); // case insensitive hash
files.push_back(name);
}
// Sort by FileName
files.sort();
// Push the filenames into a std::vector, cause OMP can better iterate over those
std::vector<CRCPair> crcs;
crcs.reserve(files.size());
CRCPair crcp;
for (std::list<std::string>::iterator it = files.begin(); it != files.end(); ++it) {
crcp.filename = &(*it);
crcs.push_back(crcp);
}
// Compute CRCs of the files
// Hint: Multithreading only speedups `.sdd` loading. For those the CRC generation is extremely slow -
// it has to load the full file to calc it! For the other formats (sd7, sdz, sdp) the CRC is saved
// in the metainformation of the container and so the loading is much faster. Neither does any of our
// current (2011) packing libraries support multithreading :/
for_mt(0, crcs.size(), [&](const int i) {
CRCPair& crcp = crcs[i];
const unsigned int nameCRC = CRC().Update(crcp.filename->data(), crcp.filename->size()).GetDigest();
const unsigned fid = ar->FindFile(*crcp.filename);
const unsigned int dataCRC = ar->GetCrc32(fid);
crcp.nameCRC = nameCRC;
crcp.dataCRC = dataCRC;
#if !defined(DEDICATED) && !defined(UNITSYNC)
Watchdog::ClearTimer(WDT_MAIN);
#endif
});
// Add file CRCs to the main archive CRC
for (std::vector<CRCPair>::iterator it = crcs.begin(); it != crcs.end(); ++it) {
crc.Update(it->nameCRC);
crc.Update(it->dataCRC);
#if !defined(DEDICATED) && !defined(UNITSYNC)
Watchdog::ClearTimer();
#endif
}
delete ignore;
delete ar;
unsigned int digest = crc.GetDigest();
// A value of 0 is used to indicate no crc.. so never return that
// Shouldn't happen all that often
if (digest == 0) {
return 4711;
} else {
return digest;
}
}
