// this function is only allowed to be called right after successfully calculating the hashset (!)
// will delete the hashset, after saving to free the memory
bool CAICHRecoveryHashSet::SaveHashSet(){
if (m_eStatus != AICH_HASHSETCOMPLETE){
ASSERT( false );
return false;
}
if ( !m_pHashTree.m_bHashValid || m_pHashTree.m_nDataSize != m_pOwner->GetFileSize()){
ASSERT( false );
return false;
}
CSingleLock lockKnown2Met(&m_mutKnown2File, false);
if (!lockKnown2Met.Lock(5000)){
return false;
}
CString fullpath = thePrefs.GetMuleDirectory(EMULE_CONFIGDIR);
fullpath.Append(KNOWN2_MET_FILENAME);
CSafeFile file;
CFileException fexp;
if (!file.Open(fullpath,CFile::modeCreate|CFile::modeReadWrite|CFile::modeNoTruncate|CFile::osSequentialScan|CFile::typeBinary|CFile::shareDenyNone, &fexp)){
if (fexp.m_cause != CFileException::fileNotFound){
CString strError(_T("Failed to load ") KNOWN2_MET_FILENAME _T(" file"));
TCHAR szError[MAX_CFEXP_ERRORMSG];
if (fexp.GetErrorMessage(szError, ARRSIZE(szError))){
strError += _T(" - ");
strError += szError;
}
theApp.QueueLogLine(true, _T("%s"), strError);
}
return false;
}
try {
//setvbuf(file.m_pStream, NULL, _IOFBF, 16384);
uint8 header = file.ReadUInt8();
if (header != KNOWN2_MET_VERSION){
AfxThrowFileException(CFileException::endOfFile, 0, file.GetFileName());
}
// first we check if the hashset we want to write is already stored
if (m_liAICHHashsStored.Find(m_pHashTree.m_Hash) != NULL)
{
theApp.QueueDebugLogLine(false, _T("AICH Hashset to write should be already present in known2.met - %s"), m_pHashTree.m_Hash.GetString());
// this hashset if already available, no need to save it again
return true;
}
/*CAICHHash CurrentHash;
while (file.GetPosition() < nExistingSize){
CurrentHash.Read(&file);
if (m_pHashTree.m_Hash == CurrentHash){
// this hashset if already available, no need to save it again
return true;
}
nHashCount = file.ReadUInt32();
if (file.GetPosition() + nHashCount*HASHSIZE > nExistingSize){
AfxThrowFileException(CFileException::endOfFile, 0, file.GetFileName());
}
// skip the rest of this hashset
file.Seek(nHashCount*HASHSIZE, CFile::current);
}*/
// write hashset
uint32 nExistingSize = (UINT)file.GetLength();
file.SeekToEnd();
m_pHashTree.m_Hash.Write(&file);
uint32 nHashCount = (uint32)((PARTSIZE/EMBLOCKSIZE + ((PARTSIZE % EMBLOCKSIZE != 0)? 1 : 0)) * (m_pHashTree.m_nDataSize/PARTSIZE));
if (m_pHashTree.m_nDataSize % PARTSIZE != 0)
nHashCount += (uint32)((m_pHashTree.m_nDataSize % PARTSIZE)/EMBLOCKSIZE + (((m_pHashTree.m_nDataSize % PARTSIZE) % EMBLOCKSIZE != 0)? 1 : 0));
file.WriteUInt32(nHashCount);
if (!m_pHashTree.WriteLowestLevelHashs(&file, 0, true, true)){
// thats bad... really
file.SetLength(nExistingSize);
theApp.QueueDebugLogLine(true, _T("Failed to save HashSet: WriteLowestLevelHashs() failed!"));
return false;
}
if (file.GetLength() != nExistingSize + (nHashCount+1)*HASHSIZE + 4){
// thats even worse
file.SetLength(nExistingSize);
theApp.QueueDebugLogLine(true, _T("Failed to save HashSet: Calculated and real size of hashset differ!"));
return false;
}
CAICHRecoveryHashSet::AddStoredAICHHash(m_pHashTree.m_Hash);
theApp.QueueDebugLogLine(false, _T("Successfully saved eMuleAC Hashset, %u Hashs + 1 Masterhash written"), nHashCount);
file.Flush();
file.Close();
}
catch(CFileException* error){
if (error->m_cause == CFileException::endOfFile)
theApp.QueueLogLine(true, GetResString(IDS_ERR_MET_BAD), KNOWN2_MET_FILENAME);
else{
TCHAR buffer[MAX_CFEXP_ERRORMSG];
error->GetErrorMessage(buffer, ARRSIZE(buffer));
theApp.QueueLogLine(true,GetResString(IDS_ERR_SERVERMET_UNKNOWN),buffer);
}
error->Delete();
return false;
}
FreeHashSet();
return true;
}
bool CAICHRecoveryHashSet::LoadHashSet(){
if (m_eStatus != AICH_HASHSETCOMPLETE){
ASSERT( false );
return false;
}
if ( !m_pHashTree.m_bHashValid || m_pHashTree.m_nDataSize != m_pOwner->GetFileSize() || m_pHashTree.m_nDataSize == 0){
ASSERT( false );
return false;
}
CString fullpath = thePrefs.GetMuleDirectory(EMULE_CONFIGDIR);
fullpath.Append(KNOWN2_MET_FILENAME);
CSafeFile file;
CFileException fexp;
if (!file.Open(fullpath,CFile::modeCreate|CFile::modeRead|CFile::modeNoTruncate|CFile::osSequentialScan|CFile::typeBinary|CFile::shareDenyNone, &fexp)){
if (fexp.m_cause != CFileException::fileNotFound){
CString strError(_T("Failed to load ") KNOWN2_MET_FILENAME _T(" file"));
TCHAR szError[MAX_CFEXP_ERRORMSG];
if (fexp.GetErrorMessage(szError, ARRSIZE(szError))){
strError += _T(" - ");
strError += szError;
}
theApp.QueueLogLine(true, _T("%s"), strError);
}
return false;
}
try {
//setvbuf(file.m_pStream, NULL, _IOFBF, 16384);
uint8 header = file.ReadUInt8();
if (header != KNOWN2_MET_VERSION){
AfxThrowFileException(CFileException::endOfFile, 0, file.GetFileName());
}
CAICHHash CurrentHash;
uint32 nExistingSize = (UINT)file.GetLength();
uint32 nHashCount;
while (file.GetPosition() < nExistingSize){
CurrentHash.Read(&file);
if (m_pHashTree.m_Hash == CurrentHash){
// found Hashset
uint32 nExpectedCount = (uint32)((PARTSIZE/EMBLOCKSIZE + ((PARTSIZE % EMBLOCKSIZE != 0)? 1 : 0)) * (m_pHashTree.m_nDataSize/PARTSIZE));
if (m_pHashTree.m_nDataSize % PARTSIZE != 0)
nExpectedCount += (uint32)((m_pHashTree.m_nDataSize % PARTSIZE)/EMBLOCKSIZE + (((m_pHashTree.m_nDataSize % PARTSIZE) % EMBLOCKSIZE != 0)? 1 : 0));
nHashCount = file.ReadUInt32();
if (nHashCount != nExpectedCount){
theApp.QueueDebugLogLine(true, _T("Failed to load HashSet: Available Hashs and expected hashcount differ!"));
return false;
}
//uint32 dbgPos = file.GetPosition();
if (!m_pHashTree.LoadLowestLevelHashs(&file)){
theApp.QueueDebugLogLine(true, _T("Failed to load HashSet: LoadLowestLevelHashs failed!"));
return false;
}
//uint32 dbgHashRead = (file.GetPosition()-dbgPos)/HASHSIZE;
if (!ReCalculateHash(false)){
theApp.QueueDebugLogLine(true, _T("Failed to load HashSet: Calculating loaded hashs failed!"));
return false;
}
if (CurrentHash != m_pHashTree.m_Hash){
theApp.QueueDebugLogLine(true, _T("Failed to load HashSet: Calculated Masterhash differs from given Masterhash - hashset corrupt!"));
return false;
}
return true;
}
nHashCount = file.ReadUInt32();
if (file.GetPosition() + nHashCount*HASHSIZE > nExistingSize){
AfxThrowFileException(CFileException::endOfFile, 0, file.GetFileName());
}
// skip the rest of this hashset
file.Seek(nHashCount*HASHSIZE, CFile::current);
}
theApp.QueueDebugLogLine(true, _T("Failed to load HashSet: HashSet not found!"));
}
catch(CFileException* error){
if (error->m_cause == CFileException::endOfFile)
theApp.QueueLogLine(true, GetResString(IDS_ERR_MET_BAD), KNOWN2_MET_FILENAME);
else{
TCHAR buffer[MAX_CFEXP_ERRORMSG];
error->GetErrorMessage(buffer, ARRSIZE(buffer));
theApp.QueueLogLine(true,GetResString(IDS_ERR_SERVERMET_UNKNOWN),buffer);
}
error->Delete();
}
return false;
}
bool CCollection::InitCollectionFromFile(const CString& sFilePath, CString sFileName)
{
uint32 uValid = sFileName.Replace(COLLECTION_FILEEXTENSION, _T(""));
ASSERT(uValid);
bool bCollectionLoaded = false;
CSafeFile data;
if(data.Open(sFilePath, CFile::modeRead | CFile::shareDenyWrite | CFile::typeBinary))
{
try
{
if(data.ReadUInt32()==1)
{
uint32 headerTagCount = data.ReadUInt32();
while(headerTagCount)
{
CTag tag(&data, true);
switch(tag.GetNameID())
{
case FT_FILENAME:
{
if(tag.IsStr())
m_sCollectionName = tag.GetStr();
break;
}
case FT_COLLECTIONAUTHOR:
{
if(tag.IsStr())
m_sCollectionAuthorName = tag.GetStr();
break;
}
case FT_COLLECTIONAUTHORKEY:
{
if(tag.IsBlob())
{
SetCollectionAuthorKey(tag.GetBlob(), tag.GetBlobSize());
}
break;
}
}
headerTagCount--;
}
uint32 fileCount = data.ReadUInt32();
while(fileCount)
{
CCollectionFile* pCollectionFile = new CCollectionFile(&data);
if(pCollectionFile)
AddFileToCollection(pCollectionFile, false);
fileCount--;
}
bCollectionLoaded = true;
}
if (m_pabyCollectionAuthorKey != NULL){
bool bResult = false;
if (data.GetLength() > data.GetPosition()){
using namespace CryptoPP;
uint32 nPos = (uint32)data.GetPosition();
data.SeekToBegin();
BYTE* pMessage = new BYTE[nPos];
VERIFY( data.Read(pMessage, nPos) == nPos);
StringSource ss_Pubkey(m_pabyCollectionAuthorKey, m_nKeySize, true, 0);
RSASSA_PKCS1v15_SHA_Verifier pubkey(ss_Pubkey);
int nSignLen = data.GetLength() - data.GetPosition();
BYTE* pSignature = new BYTE[nSignLen ];
VERIFY( data.Read(pSignature, nSignLen) == nSignLen);
bResult = pubkey.VerifyMessage(pMessage, nPos, pSignature, nSignLen);
delete[] pMessage;
delete[] pSignature;
}
if (!bResult){
DebugLogWarning(_T("Collection %s: Verifying of public key failed!"), m_sCollectionName);
delete[] m_pabyCollectionAuthorKey;
m_pabyCollectionAuthorKey = NULL;
m_nKeySize = 0;
m_sCollectionAuthorName = _T("");
}
else
DebugLog(_T("Collection %s: Public key verified"), m_sCollectionName);
}
else
m_sCollectionAuthorName = _T("");
data.Close();
}
catch(CFileException* error)
{
error->Delete();
return false;
}
catch(...)
{
ASSERT( false );
data.Close();
return false;
}
}
else
return false;
if(!bCollectionLoaded)
{
CStdioFile data;
if(data.Open(sFilePath, CFile::modeRead | CFile::shareDenyWrite | CFile::typeText))
{
try
{
CString sLink;
while(data.ReadString(sLink))
{
//Ignore all lines that start with #.
//These lines can be used for future features..
if(sLink.Find(_T("#")) != 0)
{
try
{
CCollectionFile* pCollectionFile = new CCollectionFile();
if (pCollectionFile->InitFromLink(sLink))
AddFileToCollection(pCollectionFile, false);
else
delete pCollectionFile;
}
catch(...)
{
ASSERT( false );
data.Close();
return false;
}
}
}
data.Close();
m_sCollectionName = sFileName;
bCollectionLoaded = true;
m_bTextFormat = true;
}
catch(CFileException* error)
{
error->Delete();
return false;
}
catch(...)
{
ASSERT( false );
data.Close();
return false;
}
}
}
return bCollectionLoaded;
}
int CAICHSyncThread::Run()
{
if ( !theApp.emuledlg->IsRunning() )
return 0;
// we collect all masterhashs which we find in the known2.met and store them in a list
CList<CAICHHash> liKnown2Hashs;
CString fullpath=thePrefs.GetConfigDir();
fullpath.Append(KNOWN2_MET_FILENAME);
CSafeFile file;
CFileException fexp;
uint32 nLastVerifiedPos = 0;
if (!file.Open(fullpath,CFile::modeCreate|CFile::modeReadWrite|CFile::modeNoTruncate|CFile::osSequentialScan|CFile::typeBinary|CFile::shareDenyNone, &fexp)){
if (fexp.m_cause != CFileException::fileNotFound){
CString strError(_T("Failed to load ") KNOWN2_MET_FILENAME _T(" file"));
TCHAR szError[MAX_CFEXP_ERRORMSG];
if (fexp.GetErrorMessage(szError, ARRSIZE(szError))){
strError += _T(" - ");
strError += szError;
}
LogError(LOG_STATUSBAR, _T("%s"), strError);
}
return false;
}
try {
//setvbuf(file.m_pStream, NULL, _IOFBF, 16384);
uint32 nExistingSize = file.GetLength();
uint16 nHashCount;
while (file.GetPosition() < nExistingSize){
liKnown2Hashs.AddTail(CAICHHash(&file));
nHashCount = file.ReadUInt16();
if (file.GetPosition() + nHashCount*HASHSIZE > nExistingSize){
AfxThrowFileException(CFileException::endOfFile, 0, file.GetFileName());
}
// skip the rest of this hashset
file.Seek(nHashCount*HASHSIZE, CFile::current);
nLastVerifiedPos = file.GetPosition();
}
}
catch(CFileException* error){
if (error->m_cause == CFileException::endOfFile){
LogError(LOG_STATUSBAR,GetResString(IDS_ERR_SERVERMET_BAD));
// truncate the file to the size to the last verified valid pos
try{
file.SetLength(nLastVerifiedPos);
}
catch(CFileException* error2){
error2->Delete();
}
}
else{
TCHAR buffer[MAX_CFEXP_ERRORMSG];
error->GetErrorMessage(buffer, ARRSIZE(buffer));
LogError(LOG_STATUSBAR,GetResString(IDS_ERR_SERVERMET_UNKNOWN),buffer);
}
error->Delete();
return false;
}
file.Close();
// now we check that all files which are in the sharedfilelist have a corresponding hash in out list
// those how don'T are added to the hashinglist
for (uint32 i = 0; i < theApp.sharedfiles->GetCount(); i++){
CKnownFile* pCurFile = theApp.sharedfiles->GetFileByIndex(i);
if (pCurFile != NULL && !pCurFile->IsPartFile() ){
if (theApp.emuledlg==NULL || !theApp.emuledlg->IsRunning()) // in case of shutdown while still hashing
return 0;
if (pCurFile->GetAICHHashset()->GetStatus() == AICH_HASHSETCOMPLETE){
bool bFound = false;
for (POSITION pos = liKnown2Hashs.GetHeadPosition();pos != 0;)
{
if (liKnown2Hashs.GetNext(pos) == pCurFile->GetAICHHashset()->GetMasterHash()){
bFound = true;
#ifdef _DEBUG
// in debugmode we load and verify all hashsets
ASSERT( pCurFile->GetAICHHashset()->LoadHashSet() );
// pCurFile->GetAICHHashset()->DbgTest();
pCurFile->GetAICHHashset()->FreeHashSet();
#endif
break;
}
}
if (bFound) // hashset is available, everything fine with this file
continue;
}
pCurFile->GetAICHHashset()->SetStatus(AICH_ERROR);
m_liToHash.AddTail(pCurFile);
}
}
// warn the user if he just upgraded
if (thePrefs.IsFirstStart() && !m_liToHash.IsEmpty()){
LogWarning(GetResString(IDS_AICH_WARNUSER));
}
if (!m_liToHash.IsEmpty()){
AddLogLine(true, GetResString(IDS_AICH_SYNCTOTAL), m_liToHash.GetCount() ); // I18n: ThreadSafeLog
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.SetAICHHashing(m_liToHash.GetCount());
// let first all normal hashing be done before starting out synchashing
CSingleLock sLock1(&theApp.hashing_mut); // only one filehash at a time
while (theApp.sharedfiles->GetHashingCount() != 0){
Sleep(100);
}
sLock1.Lock();
uint32 cDone = 0;
for (POSITION pos = m_liToHash.GetHeadPosition();pos != 0; cDone++)
{
if (theApp.emuledlg==NULL || !theApp.emuledlg->IsRunning()){ // in case of shutdown while still hashing
return 0;
}
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.SetAICHHashing(m_liToHash.GetCount()-cDone);
if (theApp.emuledlg->sharedfileswnd->sharedfilesctrl.m_hWnd != NULL)
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.ShowFilesCount();
CKnownFile* pCurFile = m_liToHash.GetNext(pos);
// just to be sure that the file hasnt been deleted lately
if (!(theApp.knownfiles->IsKnownFile(pCurFile) && theApp.sharedfiles->GetFileByID(pCurFile->GetFileHash())) )
continue;
AddLogLine(false, GetResString(IDS_AICH_CALCFILE), pCurFile->GetFileName()); // I18n: ThreadSafeLog
if(!pCurFile->CreateAICHHashSetOnly())
// [TPT] - WebCache
if(thePrefs.GetLogICHEvents()) //JP log ICH events
AddDebugLogLine(false, _T("Failed to create AICH Hashset while sync. for file %s"), pCurFile->GetFileName()); // I18n: ThreadSafeLog
}
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.SetAICHHashing(0);
if (theApp.emuledlg->sharedfileswnd->sharedfilesctrl.m_hWnd != NULL)
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.ShowFilesCount();
sLock1.Unlock();
}
// [TPT] - WebCache
if(thePrefs.GetLogICHEvents()) //JP log ICH events
AddDebugLogLine(false, _T("AICHSyncThread finished")); // I18n: ThreadSafeLog
return 0;
}
int CAICHSyncThread::Run()
{
if ( !theApp.emuledlg->IsRunning() )
return 0;
// we need to keep a lock on this file while the thread is running
CSingleLock lockKnown2Met(&CAICHHashSet::m_mutKnown2File);
lockKnown2Met.Lock();
// we collect all masterhashs which we find in the known2.met and store them in a list
CList<CAICHHash> liKnown2Hashs;
CString fullpath=thePrefs.GetConfigDir();
fullpath.Append(KNOWN2_MET_FILENAME);
CSafeFile file;
CFileException fexp;
uint32 nLastVerifiedPos = 0;
if (!file.Open(fullpath,CFile::modeCreate|CFile::modeReadWrite|CFile::modeNoTruncate|CFile::osSequentialScan|CFile::typeBinary|CFile::shareDenyNone, &fexp)){
if (fexp.m_cause != CFileException::fileNotFound){
CString strError(_T("Failed to load ") KNOWN2_MET_FILENAME _T(" file"));
TCHAR szError[MAX_CFEXP_ERRORMSG];
if (fexp.GetErrorMessage(szError, ARRSIZE(szError))){
strError += _T(" - ");
strError += szError;
}
LogError(LOG_STATUSBAR, _T("%s"), strError);
}
return false;
}
try {
//setvbuf(file.m_pStream, NULL, _IOFBF, 16384);
uint32 nExistingSize = (UINT)file.GetLength();
uint16 nHashCount;
while (file.GetPosition() < nExistingSize){
liKnown2Hashs.AddTail(CAICHHash(&file));
nHashCount = file.ReadUInt16();
if (file.GetPosition() + nHashCount*CAICHHash::GetHashSize() > nExistingSize){
AfxThrowFileException(CFileException::endOfFile, 0, file.GetFileName());
}
// skip the rest of this hashset
file.Seek(nHashCount*CAICHHash::GetHashSize(), CFile::current);
nLastVerifiedPos = (UINT)file.GetPosition();
}
}
catch(CFileException* error){
if (error->m_cause == CFileException::endOfFile){
LogError(LOG_STATUSBAR,GetResString(IDS_ERR_SERVERMET_BAD));
// truncate the file to the size to the last verified valid pos
try{
file.SetLength(nLastVerifiedPos);
}
catch(CFileException* error2){
error2->Delete();
}
}
else{
TCHAR buffer[MAX_CFEXP_ERRORMSG];
error->GetErrorMessage(buffer, ARRSIZE(buffer));
LogError(LOG_STATUSBAR,GetResString(IDS_ERR_SERVERMET_UNKNOWN),buffer);
}
error->Delete();
return false;
}
// now we check that all files which are in the sharedfilelist have a corresponding hash in out list
// those how don'T are added to the hashinglist
CList<CAICHHash> liUsedHashs;
CSingleLock sharelock(&theApp.sharedfiles->m_mutWriteList);
sharelock.Lock();
for (uint32 i = 0; i < theApp.sharedfiles->GetCount(); i++){
CKnownFile* pCurFile = theApp.sharedfiles->GetFileByIndex(i);
if (pCurFile != NULL && !pCurFile->IsPartFile() ){
if (theApp.emuledlg==NULL || !theApp.emuledlg->IsRunning()) // in case of shutdown while still hashing
return 0;
if (pCurFile->GetAICHHashset()->GetStatus() == AICH_HASHSETCOMPLETE){
bool bFound = false;
for (POSITION pos = liKnown2Hashs.GetHeadPosition();pos != 0;)
{
CAICHHash current_hash = liKnown2Hashs.GetNext(pos);
if (current_hash == pCurFile->GetAICHHashset()->GetMasterHash()){
bFound = true;
liUsedHashs.AddTail(current_hash);
//theApp.QueueDebugLogLine(false, _T("%s - %s"), current_hash.GetString(), pCurFile->GetFileName());
#ifdef _DEBUG
// in debugmode we load and verify all hashsets
ASSERT( pCurFile->GetAICHHashset()->LoadHashSet() );
// pCurFile->GetAICHHashset()->DbgTest();
pCurFile->GetAICHHashset()->FreeHashSet();
#endif
break;
}
}
if (bFound) // hashset is available, everything fine with this file
continue;
}
pCurFile->GetAICHHashset()->SetStatus(AICH_ERROR);
m_liToHash.AddTail(pCurFile);
}
}
sharelock.Unlock();
// removed all unused AICH hashsets from known2.met
if (!thePrefs.IsRememberingDownloadedFiles() && liUsedHashs.GetCount() != liKnown2Hashs.GetCount()){
file.SeekToBegin();
try {
uint32 nExistingSize = (UINT)file.GetLength();
uint16 nHashCount;
ULONGLONG posWritePos = 0;
ULONGLONG posReadPos = 0;
uint32 nPurgeCount = 0;
while (file.GetPosition() < nExistingSize){
CAICHHash aichHash(&file);
nHashCount = file.ReadUInt16();
if (file.GetPosition() + nHashCount*CAICHHash::GetHashSize() > nExistingSize){
AfxThrowFileException(CFileException::endOfFile, 0, file.GetFileName());
}
if (liUsedHashs.Find(aichHash) == NULL){
// unused hashset skip the rest of this hashset
file.Seek(nHashCount*CAICHHash::GetHashSize(), CFile::current);
nPurgeCount++;
}
else if(nPurgeCount == 0){
// used Hashset, but it does not need to be moved as nothing changed yet
file.Seek(nHashCount*CAICHHash::GetHashSize(), CFile::current);
posWritePos = file.GetPosition();
}
else{
// used Hashset, move position in file
BYTE* buffer = new BYTE[nHashCount*CAICHHash::GetHashSize()];
file.Read(buffer, nHashCount*CAICHHash::GetHashSize());
posReadPos = file.GetPosition();
file.Seek(posWritePos, CFile::begin);
file.Write(aichHash.GetRawHash(), CAICHHash::GetHashSize());
file.WriteUInt16(nHashCount);
file.Write(buffer, nHashCount*CAICHHash::GetHashSize());
delete[] buffer;
posWritePos = file.GetPosition();
file.Seek(posReadPos, CFile::begin);
}
}
posReadPos = file.GetPosition();
file.SetLength(posWritePos);
theApp.QueueDebugLogLine(false, _T("Cleaned up known2.met, removed %u hashsets (%s)"), nPurgeCount, CastItoXBytes(posReadPos-posWritePos));
file.Flush();
file.Close();
}
catch(CFileException* error){
if (error->m_cause == CFileException::endOfFile){
// we just parsed this files some ms ago, should never happen here
ASSERT( false );
}
else{
TCHAR buffer[MAX_CFEXP_ERRORMSG];
error->GetErrorMessage(buffer, ARRSIZE(buffer));
LogError(LOG_STATUSBAR,GetResString(IDS_ERR_SERVERMET_UNKNOWN),buffer);
}
error->Delete();
return false;
}
}
lockKnown2Met.Unlock();
// warn the user if he just upgraded
if (thePrefs.IsFirstStart() && !m_liToHash.IsEmpty()){
LogWarning(GetResString(IDS_AICH_WARNUSER));
}
if (!m_liToHash.IsEmpty()){
theApp.QueueLogLine(true, GetResString(IDS_AICH_SYNCTOTAL), m_liToHash.GetCount() );
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.SetAICHHashing(m_liToHash.GetCount());
// let first all normal hashing be done before starting out synchashing
CSingleLock sLock1(&theApp.hashing_mut); // only one filehash at a time
while (theApp.sharedfiles->GetHashingCount() != 0){
Sleep(100);
}
sLock1.Lock();
uint32 cDone = 0;
for (POSITION pos = m_liToHash.GetHeadPosition();pos != 0; cDone++)
{
if (theApp.emuledlg==NULL || !theApp.emuledlg->IsRunning()){ // in case of shutdown while still hashing
return 0;
}
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.SetAICHHashing(m_liToHash.GetCount()-cDone);
if (theApp.emuledlg->sharedfileswnd->sharedfilesctrl.m_hWnd != NULL)
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.ShowFilesCount();
CKnownFile* pCurFile = m_liToHash.GetNext(pos);
// just to be sure that the file hasnt been deleted lately
if (!(theApp.knownfiles->IsKnownFile(pCurFile) && theApp.sharedfiles->GetFileByID(pCurFile->GetFileHash())) )
continue;
theApp.QueueLogLine(false, GetResString(IDS_AICH_CALCFILE), pCurFile->GetFileName());
if(!pCurFile->CreateAICHHashSetOnly())
theApp.QueueDebugLogLine(false, _T("Failed to create AICH Hashset while sync. for file %s"), pCurFile->GetFileName());
}
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.SetAICHHashing(0);
if (theApp.emuledlg->sharedfileswnd->sharedfilesctrl.m_hWnd != NULL)
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.ShowFilesCount();
sLock1.Unlock();
}
theApp.QueueDebugLogLine(false, _T("AICHSyncThread finished"));
return 0;
}
bool CAICHSyncThread::ConvertToKnown2ToKnown264(CSafeFile* pTargetFile){
// converting known2.met to known2_64.met to support large files
// changing hashcount from uint16 to uint32
// there still exists a lock on known2_64.met and it should be not opened at this point
CString oldfullpath = thePrefs.GetMuleDirectory(EMULE_CONFIGDIR);
oldfullpath.Append(OLD_KNOWN2_MET_FILENAME);
CString newfullpath = thePrefs.GetMuleDirectory(EMULE_CONFIGDIR);
newfullpath.Append(KNOWN2_MET_FILENAME);
if (PathFileExists(newfullpath) || !PathFileExists(oldfullpath)){
// only continue if the old file doe and the new file does not exists
return false;
}
CSafeFile oldfile;
CFileException fexp;
if (!oldfile.Open(oldfullpath,CFile::modeRead|CFile::osSequentialScan|CFile::typeBinary|CFile::shareDenyNone, &fexp)){
if (fexp.m_cause != CFileException::fileNotFound){
CString strError(_T("Failed to load ") OLD_KNOWN2_MET_FILENAME _T(" file"));
TCHAR szError[MAX_CFEXP_ERRORMSG];
if (fexp.GetErrorMessage(szError, ARRSIZE(szError))){
strError += _T(" - ");
strError += szError;
}
LogError(LOG_STATUSBAR, _T("%s"), strError);
}
// else -> known2.met also doesn't exists, so nothing to convert
return false;
}
if (!pTargetFile->Open(newfullpath,CFile::modeCreate|CFile::modeReadWrite|CFile::osSequentialScan|CFile::typeBinary|CFile::shareDenyNone, &fexp)){
if (fexp.m_cause != CFileException::fileNotFound){
CString strError(_T("Failed to load ") KNOWN2_MET_FILENAME _T(" file"));
TCHAR szError[MAX_CFEXP_ERRORMSG];
if (fexp.GetErrorMessage(szError, ARRSIZE(szError))){
strError += _T(" - ");
strError += szError;
}
LogError(LOG_STATUSBAR, _T("%s"), strError);
}
return false;
}
theApp.QueueLogLine(false, GetResString(IDS_CONVERTINGKNOWN2MET), OLD_KNOWN2_MET_FILENAME, KNOWN2_MET_FILENAME);
try {
pTargetFile->WriteUInt8(KNOWN2_MET_VERSION);
uint32 nHashCount;
while (oldfile.GetPosition() < oldfile.GetLength()){
CAICHHash aichHash(&oldfile);
nHashCount = oldfile.ReadUInt16();
if (oldfile.GetPosition() + nHashCount*CAICHHash::GetHashSize() > oldfile.GetLength()){
AfxThrowFileException(CFileException::endOfFile, 0, oldfile.GetFileName());
}
BYTE* buffer = new BYTE[nHashCount*CAICHHash::GetHashSize()];
oldfile.Read(buffer, nHashCount*CAICHHash::GetHashSize());
pTargetFile->Write(aichHash.GetRawHash(), CAICHHash::GetHashSize());
pTargetFile->WriteUInt32(nHashCount);
pTargetFile->Write(buffer, nHashCount*CAICHHash::GetHashSize());
delete[] buffer;
}
pTargetFile->Flush();
oldfile.Close();
}
catch(CFileException* error){
if (error->m_cause == CFileException::endOfFile){
LogError(LOG_STATUSBAR,GetResString(IDS_ERR_MET_BAD), OLD_KNOWN2_MET_FILENAME);
ASSERT( false );
}
else{
TCHAR buffer[MAX_CFEXP_ERRORMSG];
error->GetErrorMessage(buffer, ARRSIZE(buffer));
LogError(LOG_STATUSBAR,GetResString(IDS_ERR_SERVERMET_UNKNOWN),buffer);
}
error->Delete();
theApp.QueueLogLine(false, GetResString(IDS_CONVERTINGKNOWN2FAILED));
pTargetFile->Close();
return false;
}
theApp.QueueLogLine(false, GetResString(IDS_CONVERTINGKNOWN2DONE));
// FIXME LARGE FILES (uncomment)
//DeleteFile(oldfullpath);
pTargetFile->SeekToBegin();
return true;
}
int CAICHSyncThread::Run()
{
//MORPH START SLUGFILLER: SafeHash
CReadWriteLock lock(&theApp.m_threadlock);
if (!lock.ReadLock(0))
return 0;
// MORPH END SLUGFILLER: SafeHash
if ( !theApp.emuledlg->IsRunning() )
return 0;
// we need to keep a lock on this file while the thread is running
CSingleLock lockKnown2Met(&CAICHRecoveryHashSet::m_mutKnown2File);
lockKnown2Met.Lock();
CSafeFile file;
bool bJustCreated = ConvertToKnown2ToKnown264(&file);
// we collect all masterhashs which we find in the known2.met and store them in a list
CList<CAICHHash> liKnown2Hashs;
CString fullpath = thePrefs.GetMuleDirectory(EMULE_CONFIGDIR);
fullpath.Append(KNOWN2_MET_FILENAME);
CFileException fexp;
uint32 nLastVerifiedPos = 0;
if (!bJustCreated && !file.Open(fullpath,CFile::modeCreate|CFile::modeReadWrite|CFile::modeNoTruncate|CFile::osSequentialScan|CFile::typeBinary|CFile::shareDenyNone, &fexp)){
if (fexp.m_cause != CFileException::fileNotFound){
CString strError(_T("Failed to load ") KNOWN2_MET_FILENAME _T(" file"));
TCHAR szError[MAX_CFEXP_ERRORMSG];
if (fexp.GetErrorMessage(szError, ARRSIZE(szError))){
strError += _T(" - ");
strError += szError;
}
LogError(LOG_STATUSBAR, _T("%s"), strError);
}
return false;
}
try {
if (file.GetLength() >= 1){
uint8 header = file.ReadUInt8();
if (header != KNOWN2_MET_VERSION){
AfxThrowFileException(CFileException::endOfFile, 0, file.GetFileName());
}
//setvbuf(file.m_pStream, NULL, _IOFBF, 16384);
uint32 nExistingSize = (UINT)file.GetLength();
uint32 nHashCount;
while (file.GetPosition() < nExistingSize){
liKnown2Hashs.AddTail(CAICHHash(&file));
nHashCount = file.ReadUInt32();
if (file.GetPosition() + nHashCount*CAICHHash::GetHashSize() > nExistingSize){
AfxThrowFileException(CFileException::endOfFile, 0, file.GetFileName());
}
// skip the rest of this hashset
file.Seek(nHashCount*CAICHHash::GetHashSize(), CFile::current);
nLastVerifiedPos = (UINT)file.GetPosition();
}
}
else
file.WriteUInt8(KNOWN2_MET_VERSION);
}
catch(CFileException* error){
if (error->m_cause == CFileException::endOfFile){
LogError(LOG_STATUSBAR,GetResString(IDS_ERR_MET_BAD), KNOWN2_MET_FILENAME);
// truncate the file to the size to the last verified valid pos
try{
file.SetLength(nLastVerifiedPos);
if (file.GetLength() == 0){
file.SeekToBegin();
file.WriteUInt8(KNOWN2_MET_VERSION);
}
}
catch(CFileException* error2){
error2->Delete();
}
}
else{
TCHAR buffer[MAX_CFEXP_ERRORMSG];
error->GetErrorMessage(buffer, ARRSIZE(buffer));
LogError(LOG_STATUSBAR,GetResString(IDS_ERR_SERVERMET_UNKNOWN),buffer);
}
error->Delete();
return false;
}
// now we check that all files which are in the sharedfilelist have a corresponding hash in out list
// those who don'T are added to the hashinglist
CList<CAICHHash> liUsedHashs;
CSingleLock sharelock(&theApp.sharedfiles->m_mutWriteList);
sharelock.Lock();
bool bDbgMsgCreatingPartHashs = true;
for (int i = 0; i < theApp.sharedfiles->GetCount(); i++){
CKnownFile* pCurFile = theApp.sharedfiles->GetFileByIndex(i);
if (pCurFile != NULL && !pCurFile->IsPartFile() )
{
if (theApp.emuledlg==NULL || !theApp.emuledlg->IsRunning()) // in case of shutdown while still hashing
return 0;
if (pCurFile->GetFileIdentifier().HasAICHHash()){
bool bFound = false;
for (POSITION pos = liKnown2Hashs.GetHeadPosition();pos != 0;)
{
CAICHHash current_hash = liKnown2Hashs.GetNext(pos);
if (current_hash == pCurFile->GetFileIdentifier().GetAICHHash()){
bFound = true;
liUsedHashs.AddTail(current_hash);
pCurFile->SetAICHRecoverHashSetAvailable(true);
// Has the file the proper AICH Parthashset? If not probably upgrading, create it
if (!pCurFile->GetFileIdentifier().HasExpectedAICHHashCount())
{
if (bDbgMsgCreatingPartHashs)
{
bDbgMsgCreatingPartHashs = false;
DebugLogWarning(_T("Missing AICH Part Hashsets for known files - maybe upgrading from earlier version. Creating them out of full AICH Recovery Hashsets, shouldn't take too long"));
}
CAICHRecoveryHashSet tempHashSet(pCurFile, pCurFile->GetFileSize());
tempHashSet.SetMasterHash(pCurFile->GetFileIdentifier().GetAICHHash(), AICH_HASHSETCOMPLETE);
if (!tempHashSet.LoadHashSet())
{
ASSERT( false );
DebugLogError(_T("Failed to load full AICH Recovery Hashset - known2.met might be corrupt. Unable to create AICH Part Hashset - %s"), pCurFile->GetFileName());
}
else
{
if (!pCurFile->GetFileIdentifier().SetAICHHashSet(tempHashSet))
{
DebugLogError(_T("Failed to create AICH Part Hashset out of full AICH Recovery Hashset - %s"), pCurFile->GetFileName());
ASSERT( false );
}
ASSERT(pCurFile->GetFileIdentifier().HasExpectedAICHHashCount());
}
}
//theApp.QueueDebugLogLine(false, _T("%s - %s"), current_hash.GetString(), pCurFile->GetFileName());
/*#ifdef _DEBUG
// in debugmode we load and verify all hashsets
CAICHRecoveryHashSet* pTempHashSet = new CAICHRecoveryHashSet(pCurFile);
pTempHashSet->SetFileSize(pCurFile->GetFileSize());
pTempHashSet->SetMasterHash(pCurFile->GetFileIdentifier().GetAICHHash(), AICH_HASHSETCOMPLETE)
ASSERT( pTempHashSet->LoadHashSet() );
delete pTempHashSet;
#endif*/
break;
}
}
if (bFound) // hashset is available, everything fine with this file
continue;
}
pCurFile->SetAICHRecoverHashSetAvailable(false);
m_liToHash.AddTail(pCurFile);
}
}
sharelock.Unlock();
// removed all unused AICH hashsets from known2.met
if (liUsedHashs.GetCount() != liKnown2Hashs.GetCount() &&
// EastShare START - Added by TAHO, .met file control
/*
(!thePrefs.IsRememberingDownloadedFiles() || thePrefs.DoPartiallyPurgeOldKnownFiles()))
*/
(!thePrefs.IsRememberingDownloadedFiles() ||
thePrefs.DoPartiallyPurgeOldKnownFiles() ||
thePrefs.DoCompletlyPurgeOldKnownFiles() ||
thePrefs.DoRemoveAichImmediatly()
)
)
// EastShare END - Added by TAHO, .met file control
{
file.SeekToBegin();
try {
uint8 header = file.ReadUInt8();
if (header != KNOWN2_MET_VERSION){
AfxThrowFileException(CFileException::endOfFile, 0, file.GetFileName());
}
uint32 nExistingSize = (UINT)file.GetLength();
uint32 nHashCount;
ULONGLONG posWritePos = file.GetPosition();
ULONGLONG posReadPos = file.GetPosition();
uint32 nPurgeCount = 0;
uint32 nPurgeBecauseOld = 0;
while (file.GetPosition() < nExistingSize){
CAICHHash aichHash(&file);
nHashCount = file.ReadUInt32();
if (file.GetPosition() + nHashCount*CAICHHash::GetHashSize() > nExistingSize){
AfxThrowFileException(CFileException::endOfFile, 0, file.GetFileName());
}
if (!thePrefs.IsRememberingDownloadedFiles() && liUsedHashs.Find(aichHash) == NULL)
{
// unused hashset skip the rest of this hashset
file.Seek(nHashCount*CAICHHash::GetHashSize(), CFile::current);
nPurgeCount++;
}
else if (thePrefs.IsRememberingDownloadedFiles() && theApp.knownfiles->ShouldPurgeAICHHashset(aichHash))
{
// EastShare START - Added by TAHO, .met file control
/*
ASSERT( thePrefs.DoPartiallyPurgeOldKnownFiles() );
*/
ASSERT( thePrefs.DoPartiallyPurgeOldKnownFiles() || thePrefs.DoRemoveAichImmediatly());
// EastShare END - Added by TAHO, .met file control
// also unused (purged) hashset skip the rest of this hashset
file.Seek(nHashCount*CAICHHash::GetHashSize(), CFile::current);
nPurgeCount++;
nPurgeBecauseOld++;
}
else if(nPurgeCount == 0){
// used Hashset, but it does not need to be moved as nothing changed yet
file.Seek(nHashCount*CAICHHash::GetHashSize(), CFile::current);
posWritePos = file.GetPosition();
CAICHRecoveryHashSet::AddStoredAICHHash(aichHash);
}
else{
// used Hashset, move position in file
BYTE* buffer = new BYTE[nHashCount*CAICHHash::GetHashSize()];
file.Read(buffer, nHashCount*CAICHHash::GetHashSize());
posReadPos = file.GetPosition();
file.Seek(posWritePos, CFile::begin);
file.Write(aichHash.GetRawHash(), CAICHHash::GetHashSize());
file.WriteUInt32(nHashCount);
file.Write(buffer, nHashCount*CAICHHash::GetHashSize());
delete[] buffer;
posWritePos = file.GetPosition();
file.Seek(posReadPos, CFile::begin);
CAICHRecoveryHashSet::AddStoredAICHHash(aichHash);
}
}
posReadPos = file.GetPosition();
file.SetLength(posWritePos);
theApp.QueueDebugLogLine(false, _T("Cleaned up known2.met, removed %u hashsets and purged %u hashsets of old known files (%s)")
, nPurgeCount - nPurgeBecauseOld, nPurgeBecauseOld, CastItoXBytes(posReadPos-posWritePos));
file.Flush();
file.Close();
}
catch(CFileException* error){
if (error->m_cause == CFileException::endOfFile){
// we just parsed this files some ms ago, should never happen here
ASSERT( false );
}
else{
TCHAR buffer[MAX_CFEXP_ERRORMSG];
error->GetErrorMessage(buffer, ARRSIZE(buffer));
LogError(LOG_STATUSBAR,GetResString(IDS_ERR_SERVERMET_UNKNOWN),buffer);
}
error->Delete();
return false;
}
}
else
{
// remember (/index) all hashs which are stored in the file for faster checking lateron
for (POSITION pos = liKnown2Hashs.GetHeadPosition();pos != 0;)
{
CAICHRecoveryHashSet::AddStoredAICHHash(liKnown2Hashs.GetNext(pos));
}
}
lockKnown2Met.Unlock();
// warn the user if he just upgraded
if (thePrefs.IsFirstStart() && !m_liToHash.IsEmpty() && !bJustCreated){
LogWarning(GetResString(IDS_AICH_WARNUSER));
}
if (!m_liToHash.IsEmpty()){
theApp.QueueLogLine(true, GetResString(IDS_AICH_SYNCTOTAL), m_liToHash.GetCount() );
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.SetAICHHashing(m_liToHash.GetCount());
// let first all normal hashing be done before starting out synchashing
CSingleLock sLock1(&theApp.hashing_mut); // only one filehash at a time
while (theApp.sharedfiles->GetHashingCount() != 0){
Sleep(100);
if (!CemuleDlg::IsRunning())
return 0;
}
sLock1.Lock();
uint32 cDone = 0;
for (POSITION pos = m_liToHash.GetHeadPosition();pos != 0; cDone++)
{
if (!CemuleDlg::IsRunning()){ // in case of shutdown while still hashing
return 0;
}
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.SetAICHHashing(m_liToHash.GetCount()-cDone);
if (theApp.emuledlg->sharedfileswnd->sharedfilesctrl.m_hWnd != NULL)
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.ShowFilesCount();
CKnownFile* pCurFile = m_liToHash.GetNext(pos);
// just to be sure that the file hasnt been deleted lately
if (!(theApp.knownfiles->IsKnownFile(pCurFile) && theApp.sharedfiles->GetFileByID(pCurFile->GetFileHash())) )
continue;
theApp.QueueLogLine(false, GetResString(IDS_AICH_CALCFILE), pCurFile->GetFileName());
if(!pCurFile->CreateAICHHashSetOnly())
theApp.QueueDebugLogLine(false, _T("Failed to create AICH Hashset while sync. for file %s"), pCurFile->GetFileName());
}
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.SetAICHHashing(0);
if (theApp.emuledlg->sharedfileswnd->sharedfilesctrl.m_hWnd != NULL)
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.ShowFilesCount();
sLock1.Unlock();
}
theApp.QueueDebugLogLine(false, _T("AICHSyncThread finished"));
return 0;
}