/** * Get CRC of the data in the specified archive. * Returns 0 if file could not be opened. */ unsigned int CArchiveScanner::GetCRC(const std::string& arcName) { CRC crc; CArchiveBase* ar; std::list<std::string> files; //! Try to open an archive ar = CArchiveFactory::OpenArchive(arcName); if (!ar) { return 0; // It wasn't an archive } //! Load ignore list. IFileFilter* ignore = CreateIgnoreFilter(ar); for (unsigned fid = 0; fid != ar->NumFiles(); ++fid) { std::string name; int size; ar->FileInfo(fid, name, size); if (ignore->Match(name)) { continue; } StringToLowerInPlace(name); //! case insensitive hash files.push_back(name); } files.sort(); //! Add all files in sorted order for (std::list<std::string>::iterator i = files.begin(); i != files.end(); ++i) { const unsigned int nameCRC = CRC().Update(i->data(), i->size()).GetDigest(); const unsigned fid = ar->FindFile(*i); const unsigned int dataCRC = ar->GetCrc32(fid); crc.Update(nameCRC); crc.Update(dataCRC); } delete ignore; delete ar; unsigned int digest = crc.GetDigest(); //! A value of 0 is used to indicate no crc.. so never return that //! Shouldn't happen all that often if (digest == 0) { return 4711; } else { return digest; } }
/** * Get CRC of the data in the specified archive. * Returns 0 if file could not be opened. */ unsigned int CArchiveScanner::GetCRC(const std::string& arcName) { CRC crc; struct CRCPair { std::string* filename; unsigned int nameCRC; unsigned int dataCRC; }; // try to open an archive std::unique_ptr<IArchive> ar(archiveLoader.OpenArchive(arcName)); if (ar == nullptr) return 0; // load ignore list, and insert all files to check in lowercase format std::unique_ptr<IFileFilter> ignore(CreateIgnoreFilter(ar.get())); std::vector<std::string> files; std::vector<CRCPair> crcs; files.reserve(ar->NumFiles()); crcs.reserve(ar->NumFiles()); for (unsigned fid = 0; fid != ar->NumFiles(); ++fid) { const std::pair<std::string, int>& info = ar->FileInfo(fid); if (ignore->Match(info.first)) continue; // create case-insensitive hashes files.push_back(StringToLower(info.first)); } // sort by filename std::stable_sort(files.begin(), files.end()); for (std::string& f: files) { crcs.push_back(CRCPair{&f, 0, 0}); } // compute CRCs of the files // Hint: Multithreading only speedups `.sdd` loading. For those the CRC generation is extremely slow - // it has to load the full file to calc it! For the other formats (sd7, sdz, sdp) the CRC is saved // in the metainformation of the container and so the loading is much faster. Neither does any of our // current (2011) packing libraries support multithreading :/ for_mt(0, crcs.size(), [&](const int i) { CRCPair& crcp = crcs[i]; assert(crcp.filename == &files[i]); const unsigned int nameCRC = CRC::GetCRC(crcp.filename->data(), crcp.filename->size()); const unsigned fid = ar->FindFile(*crcp.filename); const unsigned int dataCRC = ar->GetCrc32(fid); crcp.nameCRC = nameCRC; crcp.dataCRC = dataCRC; #if !defined(DEDICATED) && !defined(UNITSYNC) Watchdog::ClearTimer(WDT_MAIN); #endif }); // Add file CRCs to the main archive CRC for (const CRCPair& crcp: crcs) { crc.Update(crcp.nameCRC); crc.Update(crcp.dataCRC); #if !defined(DEDICATED) && !defined(UNITSYNC) Watchdog::ClearTimer(); #endif } // A value of 0 is used to indicate no crc.. so never return that // Shouldn't happen all that often const unsigned int digest = crc.GetDigest(); return (digest == 0)? 4711: digest; }
/** * Get CRC of the data in the specified archive. * Returns 0 if file could not be opened. */ unsigned int CArchiveScanner::GetCRC(const std::string& arcName) { CRC crc; std::list<std::string> files; // Try to open an archive boost::scoped_ptr<IArchive> ar(archiveLoader.OpenArchive(arcName)); if (!ar) { return 0; // It wasn't an archive } // Load ignore list. boost::scoped_ptr<IFileFilter> ignore(CreateIgnoreFilter(ar.get())); // Insert all files to check in lowercase format for (unsigned fid = 0; fid != ar->NumFiles(); ++fid) { std::string name; int size; ar->FileInfo(fid, name, size); if (ignore->Match(name)) { continue; } StringToLowerInPlace(name); // case insensitive hash files.push_back(name); } // Sort by FileName files.sort(); // Push the filenames into a std::vector, cause OMP can better iterate over those std::vector<CRCPair> crcs; crcs.reserve(files.size()); CRCPair crcp; for (std::string& f: files) { crcp.filename = &f; crcs.push_back(crcp); } // Compute CRCs of the files // Hint: Multithreading only speedups `.sdd` loading. For those the CRC generation is extremely slow - // it has to load the full file to calc it! For the other formats (sd7, sdz, sdp) the CRC is saved // in the metainformation of the container and so the loading is much faster. Neither does any of our // current (2011) packing libraries support multithreading :/ for_mt(0, crcs.size(), [&](const int i) { CRCPair& crcp = crcs[i]; const unsigned int nameCRC = CRC::GetCRC(crcp.filename->data(), crcp.filename->size()); const unsigned fid = ar->FindFile(*crcp.filename); const unsigned int dataCRC = ar->GetCrc32(fid); crcp.nameCRC = nameCRC; crcp.dataCRC = dataCRC; #if !defined(DEDICATED) && !defined(UNITSYNC) Watchdog::ClearTimer(WDT_MAIN); #endif }); // Add file CRCs to the main archive CRC for (CRCPair& crcp: crcs) { crc.Update(crcp.nameCRC); crc.Update(crcp.dataCRC); #if !defined(DEDICATED) && !defined(UNITSYNC) Watchdog::ClearTimer(); #endif } // A value of 0 is used to indicate no crc.. so never return that // Shouldn't happen all that often unsigned int digest = crc.GetDigest(); if (digest == 0) digest = 4711; return digest; }
/** * Get CRC of the data in the specified archive. * Returns 0 if file could not be opened. */ unsigned int CArchiveScanner::GetCRC(const std::string& arcName) { CRC crc; IArchive* ar; std::list<std::string> files; //! Try to open an archive ar = archiveLoader.OpenArchive(arcName); if (!ar) { return 0; // It wasn't an archive } //! Load ignore list. IFileFilter* ignore = CreateIgnoreFilter(ar); //! Insert all files to check in lowercase format for (unsigned fid = 0; fid != ar->NumFiles(); ++fid) { std::string name; int size; ar->FileInfo(fid, name, size); if (ignore->Match(name)) { continue; } StringToLowerInPlace(name); //! case insensitive hash files.push_back(name); } //! Sort by FileName files.sort(); //! Push the filenames into a std::vector, cause OMP can better iterate over those std::vector<CRCPair> crcs; crcs.reserve(files.size()); CRCPair crcp; for (std::list<std::string>::iterator it = files.begin(); it != files.end(); ++it) { crcp.filename = &(*it); crcs.push_back(crcp); } //! Compute CRCs of the files //! Hint: Multithreading only speedups `.sdd` loading. For those the CRC generation is extremely slow - //! it has to load the full file to calc it! For the other formats (sd7, sdz, sdp) the CRC is saved //! in the metainformation of the container and so the loading is much faster. Neither does any of our //! current (2011) packing libraries support multithreading :/ int i; #pragma omp parallel for private(i) for (i=0; i<crcs.size(); ++i) { CRCPair& crcp = crcs[i]; const unsigned int nameCRC = CRC().Update(crcp.filename->data(), crcp.filename->size()).GetDigest(); const unsigned fid = ar->FindFile(*crcp.filename); const unsigned int dataCRC = ar->GetCrc32(fid); crcp.nameCRC = nameCRC; crcp.dataCRC = dataCRC; #if !defined(DEDICATED) && !defined(UNITSYNC) Watchdog::ClearTimer(WDT_MAIN); #endif } //! Add file CRCs to the main archive CRC for (std::vector<CRCPair>::iterator it = crcs.begin(); it != crcs.end(); ++it) { crc.Update(it->nameCRC); crc.Update(it->dataCRC); #if !defined(DEDICATED) && !defined(UNITSYNC) Watchdog::ClearTimer(); #endif } delete ignore; delete ar; unsigned int digest = crc.GetDigest(); //! A value of 0 is used to indicate no crc.. so never return that //! Shouldn't happen all that often if (digest == 0) { return 4711; } else { return digest; } }