bool CKnownFileList::SafeAddKFile(CKnownFile* toadd)
{
bool bRemovedDuplicateSharedFile = false;
CCKey key(toadd->GetFileHash());
CKnownFile* pFileInMap;
if (m_Files_map.Lookup(key, pFileInMap))
{
TRACE(_T("%hs: Already in known list: %s %I64u \"%s\"\n"), __FUNCTION__, md4str(pFileInMap->GetFileHash()), pFileInMap->GetFileSize(), pFileInMap->GetFileName());
TRACE(_T("%hs: Old entry replaced with: %s %I64u \"%s\"\n"), __FUNCTION__, md4str(toadd->GetFileHash()), toadd->GetFileSize(), toadd->GetFileName());
// if we hash files which are already in known file list and add them later (when the hashing thread is finished),
// we can not delete any already available entry from known files list. that entry can already be used by the
// shared file list -> crash.
m_Files_map.RemoveKey(CCKey(pFileInMap->GetFileHash()));
m_mapKnownFilesByAICH.RemoveKey(pFileInMap->GetAICHHashset()->GetMasterHash());
//This can happen in a couple situations..
//File was renamed outside of eMule..
//A user decided to redownload a file he has downloaded and unshared..
//RemovingKeyWords I believe is not thread safe if I'm looking at this right.
//Not sure of a good solution yet..
if (theApp.sharedfiles)
{
#if 0
// This may crash the client because of dangling ptr in shared files ctrl.
// This may happen if a file is re-shared which is also currently downloaded.
// After the file was downloaded (again) there is a dangl. ptr in shared files
// ctrl.
// Actually that's also wrong in some cases: Keywords are not always removed
// because the wrong ptr is used to search for in keyword publish list.
theApp.sharedfiles->RemoveKeywords(pFileInMap);
#else
// This solves the problem with dangl. ptr in shared files ctrl,
// but creates a new bug. It may lead to unshared files! Even
// worse it may lead to files which are 'shared' in GUI but
// which are though not shared 'logically'.
//
// To reduce the harm, remove the file from shared files list,
// only if really needed. Right now this 'harm' applies for files
// which are re-shared and then completed (again) because they were
// also in download queue (they were added there when the already
// available file was not in shared file list).
if (theApp.sharedfiles->IsFilePtrInList(pFileInMap))
bRemovedDuplicateSharedFile = theApp.sharedfiles->RemoveFile(pFileInMap);
#endif
ASSERT( !theApp.sharedfiles->IsFilePtrInList(pFileInMap) );
}
//Double check to make sure this is the same file as it's possible that a two files have the same hash.
//Maybe in the furture we can change the client to not just use Hash as a key throughout the entire client..
ASSERT( toadd->GetFileSize() == pFileInMap->GetFileSize() );
ASSERT( toadd != pFileInMap );
if (toadd->GetFileSize() == pFileInMap->GetFileSize())
toadd->statistic.MergeFileStats(&pFileInMap->statistic);
ASSERT( theApp.sharedfiles==NULL || !theApp.sharedfiles->IsFilePtrInList(pFileInMap) );
ASSERT( theApp.downloadqueue==NULL || !theApp.downloadqueue->IsPartFile(pFileInMap) );
// Quick fix: If we downloaded already downloaded files again and if those files all had the same file names
// and were renamed during file completion, we have a pending ptr in transfer window.
if (theApp.emuledlg && theApp.emuledlg->transferwnd && theApp.emuledlg->transferwnd->downloadlistctrl.m_hWnd)
theApp.emuledlg->transferwnd->downloadlistctrl.RemoveFile((CPartFile*)pFileInMap);
delete pFileInMap;
}
m_Files_map.SetAt(key, toadd);
if (bRemovedDuplicateSharedFile) {
theApp.sharedfiles->SafeAddKFile(toadd);
}
if (toadd->GetAICHHashset()->HasValidMasterHash())
m_mapKnownFilesByAICH.SetAt(toadd->GetAICHHashset()->GetMasterHash(), toadd);
return true;
}
int CAICHSyncThread::Run()
{
if ( !theApp.emuledlg->IsRunning() )
return 0;
// we collect all masterhashs which we find in the known2.met and store them in a list
CList<CAICHHash> liKnown2Hashs;
CString fullpath=thePrefs.GetConfigDir();
fullpath.Append(KNOWN2_MET_FILENAME);
CSafeFile file;
CFileException fexp;
uint32 nLastVerifiedPos = 0;
if (!file.Open(fullpath,CFile::modeCreate|CFile::modeReadWrite|CFile::modeNoTruncate|CFile::osSequentialScan|CFile::typeBinary|CFile::shareDenyNone, &fexp)){
if (fexp.m_cause != CFileException::fileNotFound){
CString strError(_T("Failed to load ") KNOWN2_MET_FILENAME _T(" file"));
TCHAR szError[MAX_CFEXP_ERRORMSG];
if (fexp.GetErrorMessage(szError, ARRSIZE(szError))){
strError += _T(" - ");
strError += szError;
}
LogError(LOG_STATUSBAR, _T("%s"), strError);
}
return false;
}
try {
//setvbuf(file.m_pStream, NULL, _IOFBF, 16384);
uint32 nExistingSize = file.GetLength();
uint16 nHashCount;
while (file.GetPosition() < nExistingSize){
liKnown2Hashs.AddTail(CAICHHash(&file));
nHashCount = file.ReadUInt16();
if (file.GetPosition() + nHashCount*HASHSIZE > nExistingSize){
AfxThrowFileException(CFileException::endOfFile, 0, file.GetFileName());
}
// skip the rest of this hashset
file.Seek(nHashCount*HASHSIZE, CFile::current);
nLastVerifiedPos = file.GetPosition();
}
}
catch(CFileException* error){
if (error->m_cause == CFileException::endOfFile){
LogError(LOG_STATUSBAR,GetResString(IDS_ERR_SERVERMET_BAD));
// truncate the file to the size to the last verified valid pos
try{
file.SetLength(nLastVerifiedPos);
}
catch(CFileException* error2){
error2->Delete();
}
}
else{
TCHAR buffer[MAX_CFEXP_ERRORMSG];
error->GetErrorMessage(buffer, ARRSIZE(buffer));
LogError(LOG_STATUSBAR,GetResString(IDS_ERR_SERVERMET_UNKNOWN),buffer);
}
error->Delete();
return false;
}
file.Close();
// now we check that all files which are in the sharedfilelist have a corresponding hash in out list
// those how don'T are added to the hashinglist
for (uint32 i = 0; i < theApp.sharedfiles->GetCount(); i++){
CKnownFile* pCurFile = theApp.sharedfiles->GetFileByIndex(i);
if (pCurFile != NULL && !pCurFile->IsPartFile() ){
if (theApp.emuledlg==NULL || !theApp.emuledlg->IsRunning()) // in case of shutdown while still hashing
return 0;
if (pCurFile->GetAICHHashset()->GetStatus() == AICH_HASHSETCOMPLETE){
bool bFound = false;
for (POSITION pos = liKnown2Hashs.GetHeadPosition();pos != 0;)
{
if (liKnown2Hashs.GetNext(pos) == pCurFile->GetAICHHashset()->GetMasterHash()){
bFound = true;
#ifdef _DEBUG
// in debugmode we load and verify all hashsets
ASSERT( pCurFile->GetAICHHashset()->LoadHashSet() );
// pCurFile->GetAICHHashset()->DbgTest();
pCurFile->GetAICHHashset()->FreeHashSet();
#endif
break;
}
}
if (bFound) // hashset is available, everything fine with this file
continue;
}
pCurFile->GetAICHHashset()->SetStatus(AICH_ERROR);
m_liToHash.AddTail(pCurFile);
}
}
// warn the user if he just upgraded
if (thePrefs.IsFirstStart() && !m_liToHash.IsEmpty()){
LogWarning(GetResString(IDS_AICH_WARNUSER));
}
if (!m_liToHash.IsEmpty()){
AddLogLine(true, GetResString(IDS_AICH_SYNCTOTAL), m_liToHash.GetCount() ); // I18n: ThreadSafeLog
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.SetAICHHashing(m_liToHash.GetCount());
// let first all normal hashing be done before starting out synchashing
CSingleLock sLock1(&theApp.hashing_mut); // only one filehash at a time
while (theApp.sharedfiles->GetHashingCount() != 0){
Sleep(100);
}
sLock1.Lock();
uint32 cDone = 0;
for (POSITION pos = m_liToHash.GetHeadPosition();pos != 0; cDone++)
{
if (theApp.emuledlg==NULL || !theApp.emuledlg->IsRunning()){ // in case of shutdown while still hashing
return 0;
}
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.SetAICHHashing(m_liToHash.GetCount()-cDone);
if (theApp.emuledlg->sharedfileswnd->sharedfilesctrl.m_hWnd != NULL)
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.ShowFilesCount();
CKnownFile* pCurFile = m_liToHash.GetNext(pos);
// just to be sure that the file hasnt been deleted lately
if (!(theApp.knownfiles->IsKnownFile(pCurFile) && theApp.sharedfiles->GetFileByID(pCurFile->GetFileHash())) )
continue;
AddLogLine(false, GetResString(IDS_AICH_CALCFILE), pCurFile->GetFileName()); // I18n: ThreadSafeLog
if(!pCurFile->CreateAICHHashSetOnly())
// [TPT] - WebCache
if(thePrefs.GetLogICHEvents()) //JP log ICH events
AddDebugLogLine(false, _T("Failed to create AICH Hashset while sync. for file %s"), pCurFile->GetFileName()); // I18n: ThreadSafeLog
}
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.SetAICHHashing(0);
if (theApp.emuledlg->sharedfileswnd->sharedfilesctrl.m_hWnd != NULL)
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.ShowFilesCount();
sLock1.Unlock();
}
// [TPT] - WebCache
if(thePrefs.GetLogICHEvents()) //JP log ICH events
AddDebugLogLine(false, _T("AICHSyncThread finished")); // I18n: ThreadSafeLog
return 0;
}
int CAICHSyncThread::Run()
{
if ( !theApp.emuledlg->IsRunning() )
return 0;
// we need to keep a lock on this file while the thread is running
CSingleLock lockKnown2Met(&CAICHHashSet::m_mutKnown2File);
lockKnown2Met.Lock();
// we collect all masterhashs which we find in the known2.met and store them in a list
CList<CAICHHash> liKnown2Hashs;
CString fullpath=thePrefs.GetConfigDir();
fullpath.Append(KNOWN2_MET_FILENAME);
CSafeFile file;
CFileException fexp;
uint32 nLastVerifiedPos = 0;
if (!file.Open(fullpath,CFile::modeCreate|CFile::modeReadWrite|CFile::modeNoTruncate|CFile::osSequentialScan|CFile::typeBinary|CFile::shareDenyNone, &fexp)){
if (fexp.m_cause != CFileException::fileNotFound){
CString strError(_T("Failed to load ") KNOWN2_MET_FILENAME _T(" file"));
TCHAR szError[MAX_CFEXP_ERRORMSG];
if (fexp.GetErrorMessage(szError, ARRSIZE(szError))){
strError += _T(" - ");
strError += szError;
}
LogError(LOG_STATUSBAR, _T("%s"), strError);
}
return false;
}
try {
//setvbuf(file.m_pStream, NULL, _IOFBF, 16384);
uint32 nExistingSize = (UINT)file.GetLength();
uint16 nHashCount;
while (file.GetPosition() < nExistingSize){
liKnown2Hashs.AddTail(CAICHHash(&file));
nHashCount = file.ReadUInt16();
if (file.GetPosition() + nHashCount*CAICHHash::GetHashSize() > nExistingSize){
AfxThrowFileException(CFileException::endOfFile, 0, file.GetFileName());
}
// skip the rest of this hashset
file.Seek(nHashCount*CAICHHash::GetHashSize(), CFile::current);
nLastVerifiedPos = (UINT)file.GetPosition();
}
}
catch(CFileException* error){
if (error->m_cause == CFileException::endOfFile){
LogError(LOG_STATUSBAR,GetResString(IDS_ERR_SERVERMET_BAD));
// truncate the file to the size to the last verified valid pos
try{
file.SetLength(nLastVerifiedPos);
}
catch(CFileException* error2){
error2->Delete();
}
}
else{
TCHAR buffer[MAX_CFEXP_ERRORMSG];
error->GetErrorMessage(buffer, ARRSIZE(buffer));
LogError(LOG_STATUSBAR,GetResString(IDS_ERR_SERVERMET_UNKNOWN),buffer);
}
error->Delete();
return false;
}
// now we check that all files which are in the sharedfilelist have a corresponding hash in out list
// those how don'T are added to the hashinglist
CList<CAICHHash> liUsedHashs;
CSingleLock sharelock(&theApp.sharedfiles->m_mutWriteList);
sharelock.Lock();
for (uint32 i = 0; i < theApp.sharedfiles->GetCount(); i++){
CKnownFile* pCurFile = theApp.sharedfiles->GetFileByIndex(i);
if (pCurFile != NULL && !pCurFile->IsPartFile() ){
if (theApp.emuledlg==NULL || !theApp.emuledlg->IsRunning()) // in case of shutdown while still hashing
return 0;
if (pCurFile->GetAICHHashset()->GetStatus() == AICH_HASHSETCOMPLETE){
bool bFound = false;
for (POSITION pos = liKnown2Hashs.GetHeadPosition();pos != 0;)
{
CAICHHash current_hash = liKnown2Hashs.GetNext(pos);
if (current_hash == pCurFile->GetAICHHashset()->GetMasterHash()){
bFound = true;
liUsedHashs.AddTail(current_hash);
//theApp.QueueDebugLogLine(false, _T("%s - %s"), current_hash.GetString(), pCurFile->GetFileName());
#ifdef _DEBUG
// in debugmode we load and verify all hashsets
ASSERT( pCurFile->GetAICHHashset()->LoadHashSet() );
// pCurFile->GetAICHHashset()->DbgTest();
pCurFile->GetAICHHashset()->FreeHashSet();
#endif
break;
}
}
if (bFound) // hashset is available, everything fine with this file
continue;
}
pCurFile->GetAICHHashset()->SetStatus(AICH_ERROR);
m_liToHash.AddTail(pCurFile);
}
}
sharelock.Unlock();
// removed all unused AICH hashsets from known2.met
if (!thePrefs.IsRememberingDownloadedFiles() && liUsedHashs.GetCount() != liKnown2Hashs.GetCount()){
file.SeekToBegin();
try {
uint32 nExistingSize = (UINT)file.GetLength();
uint16 nHashCount;
ULONGLONG posWritePos = 0;
ULONGLONG posReadPos = 0;
uint32 nPurgeCount = 0;
while (file.GetPosition() < nExistingSize){
CAICHHash aichHash(&file);
nHashCount = file.ReadUInt16();
if (file.GetPosition() + nHashCount*CAICHHash::GetHashSize() > nExistingSize){
AfxThrowFileException(CFileException::endOfFile, 0, file.GetFileName());
}
if (liUsedHashs.Find(aichHash) == NULL){
// unused hashset skip the rest of this hashset
file.Seek(nHashCount*CAICHHash::GetHashSize(), CFile::current);
nPurgeCount++;
}
else if(nPurgeCount == 0){
// used Hashset, but it does not need to be moved as nothing changed yet
file.Seek(nHashCount*CAICHHash::GetHashSize(), CFile::current);
posWritePos = file.GetPosition();
}
else{
// used Hashset, move position in file
BYTE* buffer = new BYTE[nHashCount*CAICHHash::GetHashSize()];
file.Read(buffer, nHashCount*CAICHHash::GetHashSize());
posReadPos = file.GetPosition();
file.Seek(posWritePos, CFile::begin);
file.Write(aichHash.GetRawHash(), CAICHHash::GetHashSize());
file.WriteUInt16(nHashCount);
file.Write(buffer, nHashCount*CAICHHash::GetHashSize());
delete[] buffer;
posWritePos = file.GetPosition();
file.Seek(posReadPos, CFile::begin);
}
}
posReadPos = file.GetPosition();
file.SetLength(posWritePos);
theApp.QueueDebugLogLine(false, _T("Cleaned up known2.met, removed %u hashsets (%s)"), nPurgeCount, CastItoXBytes(posReadPos-posWritePos));
file.Flush();
file.Close();
}
catch(CFileException* error){
if (error->m_cause == CFileException::endOfFile){
// we just parsed this files some ms ago, should never happen here
ASSERT( false );
}
else{
TCHAR buffer[MAX_CFEXP_ERRORMSG];
error->GetErrorMessage(buffer, ARRSIZE(buffer));
LogError(LOG_STATUSBAR,GetResString(IDS_ERR_SERVERMET_UNKNOWN),buffer);
}
error->Delete();
return false;
}
}
lockKnown2Met.Unlock();
// warn the user if he just upgraded
if (thePrefs.IsFirstStart() && !m_liToHash.IsEmpty()){
LogWarning(GetResString(IDS_AICH_WARNUSER));
}
if (!m_liToHash.IsEmpty()){
theApp.QueueLogLine(true, GetResString(IDS_AICH_SYNCTOTAL), m_liToHash.GetCount() );
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.SetAICHHashing(m_liToHash.GetCount());
// let first all normal hashing be done before starting out synchashing
CSingleLock sLock1(&theApp.hashing_mut); // only one filehash at a time
while (theApp.sharedfiles->GetHashingCount() != 0){
Sleep(100);
}
sLock1.Lock();
uint32 cDone = 0;
for (POSITION pos = m_liToHash.GetHeadPosition();pos != 0; cDone++)
{
if (theApp.emuledlg==NULL || !theApp.emuledlg->IsRunning()){ // in case of shutdown while still hashing
return 0;
}
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.SetAICHHashing(m_liToHash.GetCount()-cDone);
if (theApp.emuledlg->sharedfileswnd->sharedfilesctrl.m_hWnd != NULL)
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.ShowFilesCount();
CKnownFile* pCurFile = m_liToHash.GetNext(pos);
// just to be sure that the file hasnt been deleted lately
if (!(theApp.knownfiles->IsKnownFile(pCurFile) && theApp.sharedfiles->GetFileByID(pCurFile->GetFileHash())) )
continue;
theApp.QueueLogLine(false, GetResString(IDS_AICH_CALCFILE), pCurFile->GetFileName());
if(!pCurFile->CreateAICHHashSetOnly())
theApp.QueueDebugLogLine(false, _T("Failed to create AICH Hashset while sync. for file %s"), pCurFile->GetFileName());
}
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.SetAICHHashing(0);
if (theApp.emuledlg->sharedfileswnd->sharedfilesctrl.m_hWnd != NULL)
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.ShowFilesCount();
sLock1.Unlock();
}
theApp.QueueDebugLogLine(false, _T("AICHSyncThread finished"));
return 0;
}