bool CKnownFileList::LoadKnownFiles()
{
CString fullpath = thePrefs.GetMuleDirectory(EMULE_CONFIGDIR);
fullpath.Append(KNOWN_MET_FILENAME);
CSafeBufferedFile file;
CFileException fexp;
if (!file.Open(fullpath,CFile::modeRead|CFile::osSequentialScan|CFile::typeBinary|CFile::shareDenyWrite, &fexp)){
if (fexp.m_cause != CFileException::fileNotFound){
CString strError(_T("Failed to load ") KNOWN_MET_FILENAME _T(" file"));
TCHAR szError[MAX_CFEXP_ERRORMSG];
if (fexp.GetErrorMessage(szError, ARRSIZE(szError))){
strError += _T(" - ");
strError += szError;
}
LogError(LOG_STATUSBAR, _T("%s"), strError);
}
return false;
}
setvbuf(file.m_pStream, NULL, _IOFBF, 16384);
CKnownFile* pRecord = NULL;
try {
uint8 header = file.ReadUInt8();
if (header != MET_HEADER && header != MET_HEADER_I64TAGS){
file.Close();
LogError(LOG_STATUSBAR, GetResString(IDS_ERR_SERVERMET_BAD));
return false;
}
AddDebugLogLine(false, _T("Known.met file version is %u (%s support 64bit tags)"), header, (header == MET_HEADER) ? _T("doesn't") : _T("does"));
UINT RecordsNumber = file.ReadUInt32();
for (UINT i = 0; i < RecordsNumber; i++) {
pRecord = new CKnownFile();
if (!pRecord->LoadFromFile(&file)){
TRACE(_T("*** Failed to load entry %u (name=%s hash=%s size=%I64u parthashs=%u expected parthashs=%u) from known.met\n"), i,
pRecord->GetFileName(), md4str(pRecord->GetFileHash()), pRecord->GetFileSize(), pRecord->GetHashCount(), pRecord->GetED2KPartHashCount());
delete pRecord;
pRecord = NULL;
continue;
}
SafeAddKFile(pRecord);
pRecord = NULL;
}
file.Close();
}
catch(CFileException* error){
if (error->m_cause == CFileException::endOfFile)
LogError(LOG_STATUSBAR, GetResString(IDS_ERR_SERVERMET_BAD));
else{
TCHAR buffer[MAX_CFEXP_ERRORMSG];
error->GetErrorMessage(buffer, ARRSIZE(buffer));
LogError(LOG_STATUSBAR, GetResString(IDS_ERR_SERVERMET_UNKNOWN),buffer);
}
error->Delete();
delete pRecord;
return false;
}
return true;
}
bool CKnownFileList::Append(CKnownFile *Record, bool afterHashing)
{
if (Record->GetFileSize() > 0) {
const CMD4Hash& tkey = Record->GetFileHash();
CKnownFileMap::iterator it = m_knownFileMap.find(tkey);
if (it == m_knownFileMap.end()) {
m_knownFileMap[tkey] = Record;
return true;
} else {
CKnownFile *existing = it->second;
if (KnownFileMatches(Record, existing->GetFileName(), existing->GetLastChangeDatetime(), existing->GetFileSize())) {
// The file is already on the list, ignore it.
AddDebugLogLineN(logKnownFiles, CFormat(wxT("%s is already on the list")) % Record->GetFileName().GetPrintable());
return false;
} else if (IsOnDuplicates(Record->GetFileName(), Record->GetLastChangeDatetime(), Record->GetFileSize())) {
// The file is on the duplicates list, ignore it.
// Should not happen, at least not after hashing. Or why did it get hashed in the first place then?
AddDebugLogLineN(logKnownFiles, CFormat(wxT("%s is on the duplicates list")) % Record->GetFileName().GetPrintable());
return false;
} else {
if (afterHashing && existing->GetFileSize() == Record->GetFileSize()) {
// We just hashed a "new" shared file and find it's already known under a different name or date.
// Guess what - it was probably renamed or touched.
// So copy over all properties from the existing known file and just keep name/date.
time_t newDate = Record->GetLastChangeDatetime();
CPath newName = Record->GetFileName();
CMemFile f;
existing->WriteToFile(&f);
f.Reset();
Record->LoadFromFile(&f);
Record->SetLastChangeDatetime(newDate);
Record->SetFileName(newName);
}
// The file is a duplicated hash. Add THE OLD ONE to the duplicates list.
// (This is used when reading the known file list where the duplicates are stored in front.)
m_duplicateFileList.push_back(existing);
if (theApp->sharedfiles) {
// Removing the old kad keywords created with the old filename
theApp->sharedfiles->RemoveKeywords(existing);
}
m_knownFileMap[tkey] = Record;
return true;
}
}
} else {
AddDebugLogLineN(logGeneral,
CFormat(wxT("%s is 0-size, not added")) %
Record->GetFileName());
return false;
}
}
bool CUploadQueue::CheckForTimeOver(CUpDownClient* client){
//If we have nobody in the queue, do NOT remove the current uploads..
//This will save some bandwidth and some unneeded swapping from upload/queue/upload..
if ( waitinglist.IsEmpty() || client->GetFriendSlot() )
return false;
if(client->HasCollectionUploadSlot()){
CKnownFile* pDownloadingFile = theApp.sharedfiles->GetFileByID(client->requpfileid);
if(pDownloadingFile == NULL)
return true;
if (CCollection::HasCollectionExtention(pDownloadingFile->GetFileName()) && pDownloadingFile->GetFileSize() < (uint64)MAXPRIORITYCOLL_SIZE)
return false;
else{
if (thePrefs.GetLogUlDlEvents())
AddDebugLogLine(DLP_HIGH, false, _T("%s: Upload session ended - client with Collection Slot tried to request blocks from another file"), client->GetUserName());
return true;
}
}
if (!thePrefs.TransferFullChunks()){
if( client->GetUpStartTimeDelay() > SESSIONMAXTIME){ // Try to keep the clients from downloading for ever
if (thePrefs.GetLogUlDlEvents())
AddDebugLogLine(DLP_LOW, false, _T("%s: Upload session ended due to max time %s."), client->GetUserName(), CastSecondsToHM(SESSIONMAXTIME/1000));
return true;
}
// Cache current client score
const uint32 score = client->GetScore(true, true);
// Check if another client has a bigger score
if (score < GetMaxClientScore() && m_dwRemovedClientByScore < GetTickCount()) {
if (thePrefs.GetLogUlDlEvents())
AddDebugLogLine(DLP_VERYLOW, false, _T("%s: Upload session ended due to score."), client->GetUserName());
//Set timer to prevent to many uploadslot getting kick do to score.
//Upload slots are delayed by a min of 1 sec and the maxscore is reset every 5 sec.
//So, I choose 6 secs to make sure the maxscore it updated before doing this again.
m_dwRemovedClientByScore = GetTickCount()+SEC2MS(6);
return true;
}
}
else{
// Allow the client to download a specified amount per session
if( client->GetQueueSessionPayloadUp() > SESSIONMAXTRANS ){
if (thePrefs.GetLogUlDlEvents())
AddDebugLogLine(DLP_DEFAULT, false, _T("%s: Upload session ended due to max transferred amount. %s"), client->GetUserName(), CastItoXBytes(SESSIONMAXTRANS, false, false));
return true;
}
}
return false;
}
/**
* Add a client to the waiting queue for uploads.
*
* @param client address of the client that should be added to the waiting queue
*
* @param bIgnoreTimelimit don't check time limit to possibly ban the client.
*/
void CUploadQueue::AddClientToQueue(CUpDownClient* client, bool bIgnoreTimelimit)
{
//This is to keep users from abusing the limits we put on lowID callbacks.
//1)Check if we are connected to any network and that we are a lowID.
//(Although this check shouldn't matter as they wouldn't have found us..
// But, maybe I'm missing something, so it's best to check as a precaution.)
//2)Check if the user is connected to Kad. We do allow all Kad Callbacks.
//3)Check if the user is in our download list or a friend..
//We give these users a special pass as they are helping us..
//4)Are we connected to a server? If we are, is the user on the same server?
//TCP lowID callbacks are also allowed..
//5)If the queue is very short, allow anyone in as we want to make sure
//our upload is always used.
if (theApp.IsConnected()
&& theApp.IsFirewalled()
&& !client->GetKadPort()
&& client->GetDownloadState() == DS_NONE
&& !client->IsFriend()
&& theApp.serverconnect
&& !theApp.serverconnect->IsLocalServer(client->GetServerIP(),client->GetServerPort())
&& GetWaitingUserCount() > 50)
return;
client->AddAskedCount();
client->SetLastUpRequest();
if (!bIgnoreTimelimit)
client->AddRequestCount(client->GetUploadFileID());
if (client->IsBanned())
return;
uint16 cSameIP = 0;
// check for double
POSITION pos1, pos2;
for (pos1 = waitinglist.GetHeadPosition();( pos2 = pos1 ) != NULL;)
{
waitinglist.GetNext(pos1);
CUpDownClient* cur_client= waitinglist.GetAt(pos2);
if (cur_client == client)
{
if (client->m_bAddNextConnect && AcceptNewClient(client->m_bAddNextConnect))
{
//Special care is given to lowID clients that missed their upload slot
//due to the saving bandwidth on callbacks.
if(thePrefs.GetLogUlDlEvents())
AddDebugLogLine(true, _T("Adding ****lowid when reconnecting. Client: %s"), client->DbgGetClientInfo());
client->m_bAddNextConnect = false;
RemoveFromWaitingQueue(client, true);
// statistic values // TODO: Maybe we should change this to count each request for a file only once and ignore reasks
CKnownFile* reqfile = theApp.sharedfiles->GetFileByID((uchar*)client->GetUploadFileID());
if (reqfile)
reqfile->statistic.AddRequest();
AddUpNextClient(_T("Adding ****lowid when reconnecting."), client);
return;
}
client->SendRankingInfo();
theApp.emuledlg->transferwnd->queuelistctrl.RefreshClient(client);
return;
}
else if ( client->Compare(cur_client) )
{
theApp.clientlist->AddTrackClient(client); // in any case keep track of this client
// another client with same ip:port or hash
// this happens only in rare cases, because same userhash / ip:ports are assigned to the right client on connecting in most cases
if (cur_client->credits != NULL && cur_client->credits->GetCurrentIdentState(cur_client->GetIP()) == IS_IDENTIFIED)
{
//cur_client has a valid secure hash, don't remove him
if (thePrefs.GetVerbose())
AddDebugLogLine(false, GetResString(IDS_SAMEUSERHASH), client->GetUserName(), cur_client->GetUserName(), client->GetUserName());
return;
}
if (client->credits != NULL && client->credits->GetCurrentIdentState(client->GetIP()) == IS_IDENTIFIED)
{
//client has a valid secure hash, add him remove other one
if (thePrefs.GetVerbose())
AddDebugLogLine(false, GetResString(IDS_SAMEUSERHASH), client->GetUserName(), cur_client->GetUserName(), cur_client->GetUserName());
RemoveFromWaitingQueue(pos2,true);
if (!cur_client->socket)
{
if(cur_client->Disconnected(_T("AddClientToQueue - same userhash 1")))
delete cur_client;
}
}
else
{
// remove both since we do not know who the bad one is
if (thePrefs.GetVerbose())
AddDebugLogLine(false, GetResString(IDS_SAMEUSERHASH), client->GetUserName() ,cur_client->GetUserName(), _T("Both"));
RemoveFromWaitingQueue(pos2,true);
if (!cur_client->socket)
{
if(cur_client->Disconnected(_T("AddClientToQueue - same userhash 2")))
delete cur_client;
}
return;
}
}
else if (client->GetIP() == cur_client->GetIP())
{
// same IP, different port, different userhash
cSameIP++;
}
}
if (cSameIP >= 3)
{
// do not accept more than 3 clients from the same IP
if (thePrefs.GetVerbose())
DEBUG_ONLY( AddDebugLogLine(false,_T("%s's (%s) request to enter the queue was rejected, because of too many clients with the same IP"), client->GetUserName(), ipstr(client->GetConnectIP())) );
return;
}
else if (theApp.clientlist->GetClientsFromIP(client->GetIP()) >= 3)
{
if (thePrefs.GetVerbose())
DEBUG_ONLY( AddDebugLogLine(false,_T("%s's (%s) request to enter the queue was rejected, because of too many clients with the same IP (found in TrackedClientsList)"), client->GetUserName(), ipstr(client->GetConnectIP())) );
return;
}
// done
// statistic values // TODO: Maybe we should change this to count each request for a file only once and ignore reasks
CKnownFile* reqfile = theApp.sharedfiles->GetFileByID((uchar*)client->GetUploadFileID());
if (reqfile)
reqfile->statistic.AddRequest();
// emule collection will bypass the queue
if (reqfile != NULL && CCollection::HasCollectionExtention(reqfile->GetFileName()) && reqfile->GetFileSize() < (uint64)MAXPRIORITYCOLL_SIZE
&& !client->IsDownloading() && client->socket != NULL && client->socket->IsConnected())
{
client->SetCollectionUploadSlot(true);
RemoveFromWaitingQueue(client, true);
AddUpNextClient(_T("Collection Priority Slot"), client);
return;
}
else
client->SetCollectionUploadSlot(false);
// cap the list
// the queue limit in prefs is only a soft limit. Hard limit is 25% higher, to let in powershare clients and other
// high ranking clients after soft limit has been reached
uint32 softQueueLimit = thePrefs.GetQueueSize();
uint32 hardQueueLimit = thePrefs.GetQueueSize() + max(thePrefs.GetQueueSize()/4, 200);
// if soft queue limit has been reached, only let in high ranking clients
if ((uint32)waitinglist.GetCount() >= hardQueueLimit ||
(uint32)waitinglist.GetCount() >= softQueueLimit && // soft queue limit is reached
(client->IsFriend() && client->GetFriendSlot()) == false && // client is not a friend with friend slot
client->GetCombinedFilePrioAndCredit() < GetAverageCombinedFilePrioAndCredit()) { // and client has lower credits/wants lower prio file than average client in queue
// then block client from getting on queue
return;
}
if (client->IsDownloading())
{
// he's already downloading and wants probably only another file
if (thePrefs.GetDebugClientTCPLevel() > 0)
DebugSend("OP__AcceptUploadReq", client);
Packet* packet = new Packet(OP_ACCEPTUPLOADREQ,0);
theStats.AddUpDataOverheadFileRequest(packet->size);
client->SendPacket(packet, true);
return;
}
if (waitinglist.IsEmpty() && ForceNewClient(true))
{
AddUpNextClient(_T("Direct add with empty queue."), client);
}
else
{
waitinglist.AddTail(client);
client->SetUploadState(US_ONUPLOADQUEUE);
theApp.emuledlg->transferwnd->queuelistctrl.AddClient(client,true);
theApp.emuledlg->transferwnd->ShowQueueCount(waitinglist.GetCount());
client->SendRankingInfo();
}
}
UINT CUpDownClient::ProcessPeerCacheUpHttpRequest(const CStringAArray& astrHeaders)
{
ASSERT( m_ePeerCacheUpState == PCUS_WAIT_CACHE_REPLY );
if (astrHeaders.GetCount() == 0)
return HTTP_STATUS_BAD_REQUEST;
const CStringA& rstrHdr = astrHeaders.GetAt(0);
char szUrl[1024];
UINT uHttpMajVer, uHttpMinVer;
if (sscanf(rstrHdr, "GET %1023s HTTP/%u.%u", szUrl, &uHttpMajVer, &uHttpMinVer) != 3){
DebugHttpHeaders(astrHeaders);
return HTTP_STATUS_BAD_REQUEST;
}
if (uHttpMajVer != 1 || (uHttpMinVer != 0 && uHttpMinVer != 1)){
DebugHttpHeaders(astrHeaders);
return HTTP_STATUS_BAD_REQUEST;
}
char szFileHash[33];
if (sscanf(szUrl, "/.ed2khash=%32s", szFileHash) != 1){
DebugHttpHeaders(astrHeaders);
return HTTP_STATUS_BAD_REQUEST;
}
uchar aucUploadFileID[16];
if (!strmd4(szFileHash, aucUploadFileID)){
DebugHttpHeaders(astrHeaders);
return HTTP_STATUS_BAD_REQUEST;
}
CKnownFile* pUploadFile = theApp.sharedfiles->GetFileByID(aucUploadFileID);
if (pUploadFile == NULL){
DebugHttpHeaders(astrHeaders);
return HTTP_STATUS_NOT_FOUND;
}
bool bValidRange = false;
uint64 ui64RangeStart = 0;
uint64 ui64RangeEnd = 0;
DWORD dwPushID = 0;
for (int i = 1; i < astrHeaders.GetCount(); i++)
{
const CStringA& rstrHdr = astrHeaders.GetAt(i);
if (strnicmp(rstrHdr, "Range:", 6) == 0)
{
int iParams;
if ( (iParams = sscanf((LPCSTR)rstrHdr+6," bytes = %I64u - %I64u", &ui64RangeStart, &ui64RangeEnd)) != 2
&& (iParams = sscanf((LPCSTR)rstrHdr+6," bytes = %I64u -", &ui64RangeStart)) != 1){
DebugHttpHeaders(astrHeaders);
TRACE("*** Unexpected HTTP %hs\n", rstrHdr);
return HTTP_STATUS_BAD_REQUEST;
}
if (iParams == 1)
ui64RangeEnd = pUploadFile->GetFileSize() - (uint64)1;
if (ui64RangeEnd < ui64RangeStart){
DebugHttpHeaders(astrHeaders);
TRACE("*** Unexpected HTTP %hs\n", rstrHdr);
return HTTP_STATUS_INV_RANGE;
}
bValidRange = true;
}
else if (strnicmp(rstrHdr, "X-ED2K-PushId:", 14) == 0)
{
if (sscanf((LPCSTR)rstrHdr+14, "%u", &dwPushID) != 1){
DebugHttpHeaders(astrHeaders);
TRACE("*** Unexpected HTTP %hs\n", rstrHdr);
return HTTP_STATUS_BAD_REQUEST;
}
}
}
if (!bValidRange){
DebugHttpHeaders(astrHeaders);
return HTTP_STATUS_LENGTH_REQUIRED;
}
m_uPeerCacheUploadPushId = dwPushID;
//PC-TODO: Where does this flag need to be cleared again?
// When client is removed from uploading list?
// When client is allowed to send more block requests?
// everything is setup for uploading with PeerCache.
SetPeerCacheUpState(PCUS_UPLOADING);
Requested_Block_Struct* reqblock = new Requested_Block_Struct;
reqblock->StartOffset = ui64RangeStart;
reqblock->EndOffset = ui64RangeEnd + 1;
md4cpy(reqblock->FileID, aucUploadFileID);
reqblock->transferred = 0;
AddReqBlock(reqblock);
return HTTP_STATUS_OK;
}
void CSharedFilesCtrl::OnDrawItem( int item, wxDC* dc, const wxRect& rect, const wxRect& rectHL, bool highlighted )
{
CKnownFile *file = (CKnownFile*)GetItemData(item);
wxASSERT( file );
if ( highlighted ) {
CMuleColour newcol(GetFocus() ? wxSYS_COLOUR_HIGHLIGHT : wxSYS_COLOUR_BTNSHADOW);
dc->SetBackground(newcol.Blend(125).GetBrush());
dc->SetTextForeground( CMuleColour(wxSYS_COLOUR_HIGHLIGHTTEXT));
// The second blending goes over the first one.
dc->SetPen(newcol.Blend(65).GetPen());
} else {
dc->SetBackground( CMuleColour(wxSYS_COLOUR_LISTBOX).GetBrush() );
dc->SetTextForeground(CMuleColour(wxSYS_COLOUR_WINDOWTEXT));
dc->SetPen(*wxTRANSPARENT_PEN);
}
dc->SetBrush(dc->GetBackground());
dc->DrawRectangle(rectHL);
dc->SetPen(*wxTRANSPARENT_PEN);
// Offset based on the height of the fonts
const int textVOffset = ( rect.GetHeight() - dc->GetCharHeight() ) / 2;
// Empty space to each side of a column
const int SPARE_PIXELS_HORZ = 4;
// The leftmost position of the current column
int columnLeft = 0;
for ( int i = 0; i < GetColumnCount(); ++i ) {
const int columnWidth = GetColumnWidth(i);
if (columnWidth > 2*SPARE_PIXELS_HORZ) {
wxRect columnRect(
columnLeft + SPARE_PIXELS_HORZ, rect.y,
columnWidth - 2 * SPARE_PIXELS_HORZ, rect.height);
wxDCClipper clipper(*dc, columnRect);
wxString textBuffer;
switch ( i ) {
case ID_SHARED_COL_NAME:
textBuffer = file->GetFileName().GetPrintable();
if (file->GetFileRating() || file->GetFileComment().Length()) {
int image = Client_CommentOnly_Smiley;
if (file->GetFileRating()) {
image = Client_InvalidRating_Smiley + file->GetFileRating() - 1;
}
wxASSERT(image >= Client_InvalidRating_Smiley);
wxASSERT(image <= Client_CommentOnly_Smiley);
int imgWidth = 16;
theApp->amuledlg->m_imagelist.Draw(image, *dc, columnRect.x,
columnRect.y + 1, wxIMAGELIST_DRAW_TRANSPARENT);
// Move the text to the right
columnRect.x += (imgWidth + 4);
}
break;
case ID_SHARED_COL_SIZE:
textBuffer = CastItoXBytes(file->GetFileSize());
break;
case ID_SHARED_COL_TYPE:
textBuffer = GetFiletypeByName(file->GetFileName());
break;
case ID_SHARED_COL_PRIO:
textBuffer = PriorityToStr(file->GetUpPriority(), file->IsAutoUpPriority());
break;
case ID_SHARED_COL_ID:
textBuffer = file->GetFileHash().Encode();
break;
case ID_SHARED_COL_REQ:
textBuffer = CFormat(wxT("%u (%u)"))
% file->statistic.GetRequests()
% file->statistic.GetAllTimeRequests();
break;
case ID_SHARED_COL_AREQ:
textBuffer = CFormat(wxT("%u (%u)"))
% file->statistic.GetAccepts()
% file->statistic.GetAllTimeAccepts();
break;
case ID_SHARED_COL_TRA:
textBuffer = CastItoXBytes(file->statistic.GetTransferred())
+ wxT(" (") + CastItoXBytes(file->statistic.GetAllTimeTransferred()) + wxT(")");
break;
case ID_SHARED_COL_RTIO:
textBuffer = CFormat(wxT("%.2f")) % ((double)file->statistic.GetAllTimeTransferred() / file->GetFileSize());
break;
case ID_SHARED_COL_PART:
if ( file->GetPartCount() ) {
wxRect barRect(columnRect.x, columnRect. y + 1,
columnRect.width, columnRect.height - 2);
DrawAvailabilityBar(file, dc, barRect);
}
break;
case ID_SHARED_COL_CMPL:
if ( file->m_nCompleteSourcesCountLo == 0 ) {
if ( file->m_nCompleteSourcesCountHi ) {
textBuffer = CFormat(wxT("< %u")) % file->m_nCompleteSourcesCountHi;
} else {
textBuffer = wxT("0");
}
} else if (file->m_nCompleteSourcesCountLo == file->m_nCompleteSourcesCountHi) {
textBuffer = CFormat(wxT("%u")) % file->m_nCompleteSourcesCountLo;
} else {
textBuffer = CFormat(wxT("%u - %u")) % file->m_nCompleteSourcesCountLo % file->m_nCompleteSourcesCountHi;
}
break;
case ID_SHARED_COL_PATH:
if ( file->IsPartFile() ) {
textBuffer = _("[PartFile]");
} else {
textBuffer = file->GetFilePath().GetPrintable();
}
}
if (!textBuffer.IsEmpty()) {
dc->DrawText(textBuffer, columnRect.x, columnRect.y + textVOffset);
}
}
// Move to the next column
columnLeft += columnWidth;
}
}
bool CKnownFileList::SafeAddKFile(CKnownFile* toadd)
{
bool bRemovedDuplicateSharedFile = false;
CCKey key(toadd->GetFileHash());
CKnownFile* pFileInMap;
if (m_Files_map.Lookup(key, pFileInMap))
{
TRACE(_T("%hs: Already in known list: %s %I64u \"%s\"\n"), __FUNCTION__, md4str(pFileInMap->GetFileHash()), pFileInMap->GetFileSize(), pFileInMap->GetFileName());
TRACE(_T("%hs: Old entry replaced with: %s %I64u \"%s\"\n"), __FUNCTION__, md4str(toadd->GetFileHash()), toadd->GetFileSize(), toadd->GetFileName());
// if we hash files which are already in known file list and add them later (when the hashing thread is finished),
// we can not delete any already available entry from known files list. that entry can already be used by the
// shared file list -> crash.
m_Files_map.RemoveKey(CCKey(pFileInMap->GetFileHash()));
m_mapKnownFilesByAICH.RemoveKey(pFileInMap->GetAICHHashset()->GetMasterHash());
//This can happen in a couple situations..
//File was renamed outside of eMule..
//A user decided to redownload a file he has downloaded and unshared..
//RemovingKeyWords I believe is not thread safe if I'm looking at this right.
//Not sure of a good solution yet..
if (theApp.sharedfiles)
{
#if 0
// This may crash the client because of dangling ptr in shared files ctrl.
// This may happen if a file is re-shared which is also currently downloaded.
// After the file was downloaded (again) there is a dangl. ptr in shared files
// ctrl.
// Actually that's also wrong in some cases: Keywords are not always removed
// because the wrong ptr is used to search for in keyword publish list.
theApp.sharedfiles->RemoveKeywords(pFileInMap);
#else
// This solves the problem with dangl. ptr in shared files ctrl,
// but creates a new bug. It may lead to unshared files! Even
// worse it may lead to files which are 'shared' in GUI but
// which are though not shared 'logically'.
//
// To reduce the harm, remove the file from shared files list,
// only if really needed. Right now this 'harm' applies for files
// which are re-shared and then completed (again) because they were
// also in download queue (they were added there when the already
// available file was not in shared file list).
if (theApp.sharedfiles->IsFilePtrInList(pFileInMap))
bRemovedDuplicateSharedFile = theApp.sharedfiles->RemoveFile(pFileInMap);
#endif
ASSERT( !theApp.sharedfiles->IsFilePtrInList(pFileInMap) );
}
//Double check to make sure this is the same file as it's possible that a two files have the same hash.
//Maybe in the furture we can change the client to not just use Hash as a key throughout the entire client..
ASSERT( toadd->GetFileSize() == pFileInMap->GetFileSize() );
ASSERT( toadd != pFileInMap );
if (toadd->GetFileSize() == pFileInMap->GetFileSize())
toadd->statistic.MergeFileStats(&pFileInMap->statistic);
ASSERT( theApp.sharedfiles==NULL || !theApp.sharedfiles->IsFilePtrInList(pFileInMap) );
ASSERT( theApp.downloadqueue==NULL || !theApp.downloadqueue->IsPartFile(pFileInMap) );
// Quick fix: If we downloaded already downloaded files again and if those files all had the same file names
// and were renamed during file completion, we have a pending ptr in transfer window.
if (theApp.emuledlg && theApp.emuledlg->transferwnd && theApp.emuledlg->transferwnd->downloadlistctrl.m_hWnd)
theApp.emuledlg->transferwnd->downloadlistctrl.RemoveFile((CPartFile*)pFileInMap);
delete pFileInMap;
}
m_Files_map.SetAt(key, toadd);
if (bRemovedDuplicateSharedFile) {
theApp.sharedfiles->SafeAddKFile(toadd);
}
if (toadd->GetAICHHashset()->HasValidMasterHash())
m_mapKnownFilesByAICH.SetAt(toadd->GetAICHHashset()->GetMasterHash(), toadd);
return true;
}
int CAICHSyncThread::Run()
{
//MORPH START SLUGFILLER: SafeHash
CReadWriteLock lock(&theApp.m_threadlock);
if (!lock.ReadLock(0))
return 0;
// MORPH END SLUGFILLER: SafeHash
if ( !theApp.emuledlg->IsRunning() )
return 0;
// we need to keep a lock on this file while the thread is running
CSingleLock lockKnown2Met(&CAICHRecoveryHashSet::m_mutKnown2File);
lockKnown2Met.Lock();
CSafeFile file;
bool bJustCreated = ConvertToKnown2ToKnown264(&file);
// we collect all masterhashs which we find in the known2.met and store them in a list
CList<CAICHHash> liKnown2Hashs;
CString fullpath = thePrefs.GetMuleDirectory(EMULE_CONFIGDIR);
fullpath.Append(KNOWN2_MET_FILENAME);
CFileException fexp;
uint32 nLastVerifiedPos = 0;
if (!bJustCreated && !file.Open(fullpath,CFile::modeCreate|CFile::modeReadWrite|CFile::modeNoTruncate|CFile::osSequentialScan|CFile::typeBinary|CFile::shareDenyNone, &fexp)){
if (fexp.m_cause != CFileException::fileNotFound){
CString strError(_T("Failed to load ") KNOWN2_MET_FILENAME _T(" file"));
TCHAR szError[MAX_CFEXP_ERRORMSG];
if (fexp.GetErrorMessage(szError, ARRSIZE(szError))){
strError += _T(" - ");
strError += szError;
}
LogError(LOG_STATUSBAR, _T("%s"), strError);
}
return false;
}
try {
if (file.GetLength() >= 1){
uint8 header = file.ReadUInt8();
if (header != KNOWN2_MET_VERSION){
AfxThrowFileException(CFileException::endOfFile, 0, file.GetFileName());
}
//setvbuf(file.m_pStream, NULL, _IOFBF, 16384);
uint32 nExistingSize = (UINT)file.GetLength();
uint32 nHashCount;
while (file.GetPosition() < nExistingSize){
liKnown2Hashs.AddTail(CAICHHash(&file));
nHashCount = file.ReadUInt32();
if (file.GetPosition() + nHashCount*CAICHHash::GetHashSize() > nExistingSize){
AfxThrowFileException(CFileException::endOfFile, 0, file.GetFileName());
}
// skip the rest of this hashset
file.Seek(nHashCount*CAICHHash::GetHashSize(), CFile::current);
nLastVerifiedPos = (UINT)file.GetPosition();
}
}
else
file.WriteUInt8(KNOWN2_MET_VERSION);
}
catch(CFileException* error){
if (error->m_cause == CFileException::endOfFile){
LogError(LOG_STATUSBAR,GetResString(IDS_ERR_MET_BAD), KNOWN2_MET_FILENAME);
// truncate the file to the size to the last verified valid pos
try{
file.SetLength(nLastVerifiedPos);
if (file.GetLength() == 0){
file.SeekToBegin();
file.WriteUInt8(KNOWN2_MET_VERSION);
}
}
catch(CFileException* error2){
error2->Delete();
}
}
else{
TCHAR buffer[MAX_CFEXP_ERRORMSG];
error->GetErrorMessage(buffer, ARRSIZE(buffer));
LogError(LOG_STATUSBAR,GetResString(IDS_ERR_SERVERMET_UNKNOWN),buffer);
}
error->Delete();
return false;
}
// now we check that all files which are in the sharedfilelist have a corresponding hash in out list
// those who don'T are added to the hashinglist
CList<CAICHHash> liUsedHashs;
CSingleLock sharelock(&theApp.sharedfiles->m_mutWriteList);
sharelock.Lock();
bool bDbgMsgCreatingPartHashs = true;
for (int i = 0; i < theApp.sharedfiles->GetCount(); i++){
CKnownFile* pCurFile = theApp.sharedfiles->GetFileByIndex(i);
if (pCurFile != NULL && !pCurFile->IsPartFile() )
{
if (theApp.emuledlg==NULL || !theApp.emuledlg->IsRunning()) // in case of shutdown while still hashing
return 0;
if (pCurFile->GetFileIdentifier().HasAICHHash()){
bool bFound = false;
for (POSITION pos = liKnown2Hashs.GetHeadPosition();pos != 0;)
{
CAICHHash current_hash = liKnown2Hashs.GetNext(pos);
if (current_hash == pCurFile->GetFileIdentifier().GetAICHHash()){
bFound = true;
liUsedHashs.AddTail(current_hash);
pCurFile->SetAICHRecoverHashSetAvailable(true);
// Has the file the proper AICH Parthashset? If not probably upgrading, create it
if (!pCurFile->GetFileIdentifier().HasExpectedAICHHashCount())
{
if (bDbgMsgCreatingPartHashs)
{
bDbgMsgCreatingPartHashs = false;
DebugLogWarning(_T("Missing AICH Part Hashsets for known files - maybe upgrading from earlier version. Creating them out of full AICH Recovery Hashsets, shouldn't take too long"));
}
CAICHRecoveryHashSet tempHashSet(pCurFile, pCurFile->GetFileSize());
tempHashSet.SetMasterHash(pCurFile->GetFileIdentifier().GetAICHHash(), AICH_HASHSETCOMPLETE);
if (!tempHashSet.LoadHashSet())
{
ASSERT( false );
DebugLogError(_T("Failed to load full AICH Recovery Hashset - known2.met might be corrupt. Unable to create AICH Part Hashset - %s"), pCurFile->GetFileName());
}
else
{
if (!pCurFile->GetFileIdentifier().SetAICHHashSet(tempHashSet))
{
DebugLogError(_T("Failed to create AICH Part Hashset out of full AICH Recovery Hashset - %s"), pCurFile->GetFileName());
ASSERT( false );
}
ASSERT(pCurFile->GetFileIdentifier().HasExpectedAICHHashCount());
}
}
//theApp.QueueDebugLogLine(false, _T("%s - %s"), current_hash.GetString(), pCurFile->GetFileName());
/*#ifdef _DEBUG
// in debugmode we load and verify all hashsets
CAICHRecoveryHashSet* pTempHashSet = new CAICHRecoveryHashSet(pCurFile);
pTempHashSet->SetFileSize(pCurFile->GetFileSize());
pTempHashSet->SetMasterHash(pCurFile->GetFileIdentifier().GetAICHHash(), AICH_HASHSETCOMPLETE)
ASSERT( pTempHashSet->LoadHashSet() );
delete pTempHashSet;
#endif*/
break;
}
}
if (bFound) // hashset is available, everything fine with this file
continue;
}
pCurFile->SetAICHRecoverHashSetAvailable(false);
m_liToHash.AddTail(pCurFile);
}
}
sharelock.Unlock();
// removed all unused AICH hashsets from known2.met
if (liUsedHashs.GetCount() != liKnown2Hashs.GetCount() &&
// EastShare START - Added by TAHO, .met file control
/*
(!thePrefs.IsRememberingDownloadedFiles() || thePrefs.DoPartiallyPurgeOldKnownFiles()))
*/
(!thePrefs.IsRememberingDownloadedFiles() ||
thePrefs.DoPartiallyPurgeOldKnownFiles() ||
thePrefs.DoCompletlyPurgeOldKnownFiles() ||
thePrefs.DoRemoveAichImmediatly()
)
)
// EastShare END - Added by TAHO, .met file control
{
file.SeekToBegin();
try {
uint8 header = file.ReadUInt8();
if (header != KNOWN2_MET_VERSION){
AfxThrowFileException(CFileException::endOfFile, 0, file.GetFileName());
}
uint32 nExistingSize = (UINT)file.GetLength();
uint32 nHashCount;
ULONGLONG posWritePos = file.GetPosition();
ULONGLONG posReadPos = file.GetPosition();
uint32 nPurgeCount = 0;
uint32 nPurgeBecauseOld = 0;
while (file.GetPosition() < nExistingSize){
CAICHHash aichHash(&file);
nHashCount = file.ReadUInt32();
if (file.GetPosition() + nHashCount*CAICHHash::GetHashSize() > nExistingSize){
AfxThrowFileException(CFileException::endOfFile, 0, file.GetFileName());
}
if (!thePrefs.IsRememberingDownloadedFiles() && liUsedHashs.Find(aichHash) == NULL)
{
// unused hashset skip the rest of this hashset
file.Seek(nHashCount*CAICHHash::GetHashSize(), CFile::current);
nPurgeCount++;
}
else if (thePrefs.IsRememberingDownloadedFiles() && theApp.knownfiles->ShouldPurgeAICHHashset(aichHash))
{
// EastShare START - Added by TAHO, .met file control
/*
ASSERT( thePrefs.DoPartiallyPurgeOldKnownFiles() );
*/
ASSERT( thePrefs.DoPartiallyPurgeOldKnownFiles() || thePrefs.DoRemoveAichImmediatly());
// EastShare END - Added by TAHO, .met file control
// also unused (purged) hashset skip the rest of this hashset
file.Seek(nHashCount*CAICHHash::GetHashSize(), CFile::current);
nPurgeCount++;
nPurgeBecauseOld++;
}
else if(nPurgeCount == 0){
// used Hashset, but it does not need to be moved as nothing changed yet
file.Seek(nHashCount*CAICHHash::GetHashSize(), CFile::current);
posWritePos = file.GetPosition();
CAICHRecoveryHashSet::AddStoredAICHHash(aichHash);
}
else{
// used Hashset, move position in file
BYTE* buffer = new BYTE[nHashCount*CAICHHash::GetHashSize()];
file.Read(buffer, nHashCount*CAICHHash::GetHashSize());
posReadPos = file.GetPosition();
file.Seek(posWritePos, CFile::begin);
file.Write(aichHash.GetRawHash(), CAICHHash::GetHashSize());
file.WriteUInt32(nHashCount);
file.Write(buffer, nHashCount*CAICHHash::GetHashSize());
delete[] buffer;
posWritePos = file.GetPosition();
file.Seek(posReadPos, CFile::begin);
CAICHRecoveryHashSet::AddStoredAICHHash(aichHash);
}
}
posReadPos = file.GetPosition();
file.SetLength(posWritePos);
theApp.QueueDebugLogLine(false, _T("Cleaned up known2.met, removed %u hashsets and purged %u hashsets of old known files (%s)")
, nPurgeCount - nPurgeBecauseOld, nPurgeBecauseOld, CastItoXBytes(posReadPos-posWritePos));
file.Flush();
file.Close();
}
catch(CFileException* error){
if (error->m_cause == CFileException::endOfFile){
// we just parsed this files some ms ago, should never happen here
ASSERT( false );
}
else{
TCHAR buffer[MAX_CFEXP_ERRORMSG];
error->GetErrorMessage(buffer, ARRSIZE(buffer));
LogError(LOG_STATUSBAR,GetResString(IDS_ERR_SERVERMET_UNKNOWN),buffer);
}
error->Delete();
return false;
}
}
else
{
// remember (/index) all hashs which are stored in the file for faster checking lateron
for (POSITION pos = liKnown2Hashs.GetHeadPosition();pos != 0;)
{
CAICHRecoveryHashSet::AddStoredAICHHash(liKnown2Hashs.GetNext(pos));
}
}
lockKnown2Met.Unlock();
// warn the user if he just upgraded
if (thePrefs.IsFirstStart() && !m_liToHash.IsEmpty() && !bJustCreated){
LogWarning(GetResString(IDS_AICH_WARNUSER));
}
if (!m_liToHash.IsEmpty()){
theApp.QueueLogLine(true, GetResString(IDS_AICH_SYNCTOTAL), m_liToHash.GetCount() );
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.SetAICHHashing(m_liToHash.GetCount());
// let first all normal hashing be done before starting out synchashing
CSingleLock sLock1(&theApp.hashing_mut); // only one filehash at a time
while (theApp.sharedfiles->GetHashingCount() != 0){
Sleep(100);
if (!CemuleDlg::IsRunning())
return 0;
}
sLock1.Lock();
uint32 cDone = 0;
for (POSITION pos = m_liToHash.GetHeadPosition();pos != 0; cDone++)
{
if (!CemuleDlg::IsRunning()){ // in case of shutdown while still hashing
return 0;
}
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.SetAICHHashing(m_liToHash.GetCount()-cDone);
if (theApp.emuledlg->sharedfileswnd->sharedfilesctrl.m_hWnd != NULL)
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.ShowFilesCount();
CKnownFile* pCurFile = m_liToHash.GetNext(pos);
// just to be sure that the file hasnt been deleted lately
if (!(theApp.knownfiles->IsKnownFile(pCurFile) && theApp.sharedfiles->GetFileByID(pCurFile->GetFileHash())) )
continue;
theApp.QueueLogLine(false, GetResString(IDS_AICH_CALCFILE), pCurFile->GetFileName());
if(!pCurFile->CreateAICHHashSetOnly())
theApp.QueueDebugLogLine(false, _T("Failed to create AICH Hashset while sync. for file %s"), pCurFile->GetFileName());
}
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.SetAICHHashing(0);
if (theApp.emuledlg->sharedfileswnd->sharedfilesctrl.m_hWnd != NULL)
theApp.emuledlg->sharedfileswnd->sharedfilesctrl.ShowFilesCount();
sLock1.Unlock();
}
theApp.QueueDebugLogLine(false, _T("AICHSyncThread finished"));
return 0;
}