// Store a piece of received data (don't know if it's good or bad yet).
// Data is stored in a list for the chunk in belongs to.
void CCorruptionBlackBox::TransferredData(uint64 nStartPos, uint64 nEndPos, uint32 senderIP)
{
if (nStartPos > nEndPos) {
wxFAIL;
return;
}
// convert pos to relative block pos
uint16 nPart = (uint16)(nStartPos / PARTSIZE);
uint32 nRelStartPos = nStartPos - nPart*PARTSIZE;
uint32 nRelEndPos = nEndPos - nPart*PARTSIZE;
if (nRelEndPos >= PARTSIZE) {
// data crosses the partborder, split it
// (for the fun of it, this should never happen)
nRelEndPos = PARTSIZE-1;
TransferredData((nPart+1)*PARTSIZE, nEndPos, senderIP);
}
//
// Let's keep things simple.
// We don't request data we already have.
// We check if received data exceeds block boundaries.
// -> There should not be much overlap here.
// So just stuff everything received into the list and only join adjacent blocks.
//
CRecordList & list = m_Records[nPart]; // this creates the entry if it doesn't exist yet
bool merged = false;
for (CRecordList::iterator it = list.begin(); it != list.end() && !merged; ++it) {
merged = it->Merge(nRelStartPos, nRelEndPos, senderIP);
}
if (!merged) {
list.push_back(CCBBRecord(nRelStartPos, nRelEndPos, senderIP));
AddDebugLogLineN(logPartFile, CFormat(wxT("CorruptionBlackBox(%s): transferred: new record for part %d (%d - %d, %s)"))
% m_partNumber % nPart % nRelStartPos % nRelEndPos % Uint32toStringIP(senderIP));
}
}
void CCorruptionBlackBox::TransferredData(uint32 nStartPos, uint32 nEndPos, const CUpDownClient* pSender){
if (nEndPos - nStartPos >= PARTSIZE){
ASSERT( false );
return;
}
if (nStartPos > nEndPos){
ASSERT( false );
return;
}
uint32 dwSenderIP = pSender->GetIP();
// we store records seperated for each part, so we don't have to search all entries everytime
// convert pos to relative block pos
uint16 nPart = nStartPos / PARTSIZE;
uint32 nRelStartPos = nStartPos - nPart*PARTSIZE;
uint32 nRelEndPos = nEndPos - nPart*PARTSIZE;
if (nRelEndPos >= PARTSIZE){
// data crosses the partborder, split it
nRelEndPos = PARTSIZE-1;
uint32 nTmpStartPos = nPart*PARTSIZE + nRelEndPos + 1;
ASSERT( nTmpStartPos % PARTSIZE == 0); // remove later
TransferredData(nTmpStartPos, nEndPos, pSender);
}
if (nPart >= m_aaRecords.GetCount()){
//ASSERT( false );
m_aaRecords.SetSize(nPart+1);
}
int posMerge = -1;
uint32 ndbgRewritten = 0;
for (int i= 0; i < m_aaRecords[nPart].GetCount(); i++){
if (m_aaRecords[nPart][i].CanMerge(nRelStartPos, nRelEndPos, dwSenderIP, BBR_NONE)){
posMerge = i;
}
// check if there is already an pending entry and overwrite it
else if (m_aaRecords[nPart][i].m_BBRStatus == BBR_NONE){
if (m_aaRecords[nPart][i].m_nStartPos >= nRelStartPos && m_aaRecords[nPart][i].m_nEndPos <= nRelEndPos){
// old one is included in new one -> delete
ndbgRewritten += (m_aaRecords[nPart][i].m_nEndPos-m_aaRecords[nPart][i].m_nStartPos)+1;
m_aaRecords[nPart].RemoveAt(i);
i--;
}
else if (m_aaRecords[nPart][i].m_nStartPos < nRelStartPos && m_aaRecords[nPart][i].m_nEndPos > nRelEndPos){
// old one includes new one
// check if the old one and new one have the same ip
if (dwSenderIP != m_aaRecords[nPart][i].m_dwIP){
// different IP, means we have to split it 2 times
uint32 nTmpEndPos1 = m_aaRecords[nPart][i].m_nEndPos;
uint32 nTmpStartPos1 = nRelEndPos + 1;
uint32 nTmpStartPos2 = m_aaRecords[nPart][i].m_nStartPos;
uint32 nTmpEndPos2 = nRelStartPos - 1;
m_aaRecords[nPart][i].m_nEndPos = nRelEndPos;
m_aaRecords[nPart][i].m_nStartPos = nRelStartPos;
uint32 dwOldIP = m_aaRecords[nPart][i].m_dwIP;
m_aaRecords[nPart][i].m_dwIP = dwSenderIP;
m_aaRecords[nPart].Add(CCBBRecord(nTmpStartPos1,nTmpEndPos1, dwOldIP));
m_aaRecords[nPart].Add(CCBBRecord(nTmpStartPos2,nTmpEndPos2, dwOldIP));
// and are done then
}
DEBUG_ONLY( AddDebugLogLine(DLP_DEFAULT, false, _T("CorruptionBlackBox: Debug: %i bytes were rewritten and records replaced with new stats (1)"), (nRelEndPos - nRelStartPos)+1) );
return;
}
else if (m_aaRecords[nPart][i].m_nStartPos >= nRelStartPos && m_aaRecords[nPart][i].m_nStartPos <= nRelEndPos){
// old one laps over new one on the right site
ASSERT( nRelEndPos - m_aaRecords[nPart][i].m_nStartPos > 0 );
ndbgRewritten += nRelEndPos - m_aaRecords[nPart][i].m_nStartPos;
m_aaRecords[nPart][i].m_nStartPos = nRelEndPos + 1;
}
else if (m_aaRecords[nPart][i].m_nEndPos >= nRelStartPos && m_aaRecords[nPart][i].m_nEndPos <= nRelEndPos){
// old one laps over new one on the left site
ASSERT( m_aaRecords[nPart][i].m_nEndPos - nRelStartPos > 0 );
ndbgRewritten += m_aaRecords[nPart][i].m_nEndPos - nRelStartPos;
m_aaRecords[nPart][i].m_nEndPos = nRelStartPos - 1;
}
}
}
if (posMerge != (-1) ){
VERIFY( m_aaRecords[nPart][posMerge].Merge(nRelStartPos, nRelEndPos, dwSenderIP, BBR_NONE) );
}
else
m_aaRecords[nPart].Add(CCBBRecord(nRelStartPos, nRelEndPos, dwSenderIP, BBR_NONE));
if (ndbgRewritten > 0){
DEBUG_ONLY( AddDebugLogLine(DLP_DEFAULT, false, _T("CorruptionBlackBox: Debug: %i bytes were rewritten and records replaced with new stats (2)"), ndbgRewritten) );
}
}
void CCorruptionBlackBox::CorruptedData(uint32 nStartPos, uint32 nEndPos){
if (nEndPos - nStartPos >= EMBLOCKSIZE){
ASSERT( false );
return;
}
// convert pos to relative block pos
uint16 nPart = nStartPos / PARTSIZE;
uint32 nRelStartPos = nStartPos - nPart*PARTSIZE;
uint32 nRelEndPos = nEndPos - nPart*PARTSIZE;
if (nRelEndPos >= PARTSIZE){
ASSERT( false );
return;
}
if (nPart >= m_aaRecords.GetCount()){
//ASSERT( false );
m_aaRecords.SetSize(nPart+1);
}
uint32 nDbgVerifiedBytes = 0;
CArray<uint32, uint32> aGuiltyClients;
for (int i= 0; i < m_aaRecords[nPart].GetCount(); i++){
if (m_aaRecords[nPart][i].m_BBRStatus == BBR_NONE){
if (m_aaRecords[nPart][i].m_nStartPos >= nRelStartPos && m_aaRecords[nPart][i].m_nEndPos <= nRelEndPos){
nDbgVerifiedBytes += (m_aaRecords[nPart][i].m_nEndPos-m_aaRecords[nPart][i].m_nStartPos)+1;
m_aaRecords[nPart][i].m_BBRStatus = BBR_CORRUPTED;
aGuiltyClients.Add(m_aaRecords[nPart][i].m_dwIP);
}
else if (m_aaRecords[nPart][i].m_nStartPos < nRelStartPos && m_aaRecords[nPart][i].m_nEndPos > nRelEndPos){
// need to split it 2*
uint32 nTmpEndPos1 = m_aaRecords[nPart][i].m_nEndPos;
uint32 nTmpStartPos1 = nRelEndPos + 1;
uint32 nTmpStartPos2 = m_aaRecords[nPart][i].m_nStartPos;
uint32 nTmpEndPos2 = nRelStartPos - 1;
m_aaRecords[nPart][i].m_nEndPos = nRelEndPos;
m_aaRecords[nPart][i].m_nStartPos = nRelStartPos;
m_aaRecords[nPart].Add(CCBBRecord(nTmpStartPos1, nTmpEndPos1, m_aaRecords[nPart][i].m_dwIP, m_aaRecords[nPart][i].m_BBRStatus));
m_aaRecords[nPart].Add(CCBBRecord(nTmpStartPos2, nTmpEndPos2, m_aaRecords[nPart][i].m_dwIP, m_aaRecords[nPart][i].m_BBRStatus));
nDbgVerifiedBytes += (m_aaRecords[nPart][i].m_nEndPos-m_aaRecords[nPart][i].m_nStartPos)+1;
m_aaRecords[nPart][i].m_BBRStatus = BBR_CORRUPTED;
aGuiltyClients.Add(m_aaRecords[nPart][i].m_dwIP);
}
else if (m_aaRecords[nPart][i].m_nStartPos >= nRelStartPos && m_aaRecords[nPart][i].m_nStartPos <= nRelEndPos){
// need to split it
uint32 nTmpEndPos = m_aaRecords[nPart][i].m_nEndPos;
uint32 nTmpStartPos = nRelEndPos + 1;
m_aaRecords[nPart][i].m_nEndPos = nRelEndPos;
m_aaRecords[nPart].Add(CCBBRecord(nTmpStartPos, nTmpEndPos, m_aaRecords[nPart][i].m_dwIP, m_aaRecords[nPart][i].m_BBRStatus));
nDbgVerifiedBytes += (m_aaRecords[nPart][i].m_nEndPos-m_aaRecords[nPart][i].m_nStartPos)+1;
m_aaRecords[nPart][i].m_BBRStatus = BBR_CORRUPTED;
aGuiltyClients.Add(m_aaRecords[nPart][i].m_dwIP);
}
else if (m_aaRecords[nPart][i].m_nEndPos >= nRelStartPos && m_aaRecords[nPart][i].m_nEndPos <= nRelEndPos){
// need to split it
uint32 nTmpStartPos = m_aaRecords[nPart][i].m_nStartPos;
uint32 nTmpEndPos = nRelStartPos - 1;
m_aaRecords[nPart][i].m_nStartPos = nRelStartPos;
m_aaRecords[nPart].Add(CCBBRecord(nTmpStartPos, nTmpEndPos, m_aaRecords[nPart][i].m_dwIP, m_aaRecords[nPart][i].m_BBRStatus));
nDbgVerifiedBytes += (m_aaRecords[nPart][i].m_nEndPos-m_aaRecords[nPart][i].m_nStartPos)+1;
m_aaRecords[nPart][i].m_BBRStatus = BBR_CORRUPTED;
aGuiltyClients.Add(m_aaRecords[nPart][i].m_dwIP);
}
}
}
// check if any IPs are already banned, so we can skip the test for those
for(int k = 0; k < aGuiltyClients.GetCount();){
// remove doubles
for(int y = k+1; y < aGuiltyClients.GetCount();){
if (aGuiltyClients[k] == aGuiltyClients[y])
aGuiltyClients.RemoveAt(y);
else
y++;
}
if (theApp.clientlist->IsBannedClient(aGuiltyClients[k])){
AddDebugLogLine(DLP_DEFAULT, false, _T("CorruptionBlackBox: Suspicous IP (%s) is already banned, skipping recheck"), ipstr(aGuiltyClients[k]));
aGuiltyClients.RemoveAt(k);
}
else
k++;
}
AddDebugLogLine(DLP_HIGH, false, _T("Found and marked %u recorded bytes of %u as corrupted in the CorruptionBlackBox records, %u clients involved"), nDbgVerifiedBytes, (nEndPos-nStartPos)+1, aGuiltyClients.GetCount());
if (aGuiltyClients.GetCount() > 0){
// parse all recorded data for this file to produce a statistic for the involved clients
// first init arrays for the statistic
CArray<uint32, uint32> aDataCorrupt;
CArray<uint32, uint32> aDataVerified;
aDataCorrupt.SetSize(aGuiltyClients.GetCount());
aDataVerified.SetSize(aGuiltyClients.GetCount());
for (int j = 0; j < aGuiltyClients.GetCount(); j++)
aDataCorrupt[j] = aDataVerified[j] = 0;
// now the parsing
for (int nPart = 0; nPart < m_aaRecords.GetCount(); nPart++){
for (int i = 0; i < m_aaRecords[nPart].GetCount(); i++){
for(int k = 0; k < aGuiltyClients.GetCount(); k++){
if (m_aaRecords[nPart][i].m_dwIP == aGuiltyClients[k]){
if (m_aaRecords[nPart][i].m_BBRStatus == BBR_CORRUPTED){
// corrupted data records are always counted as at least blocksize or bigger
aDataCorrupt[k] += max((m_aaRecords[nPart][i].m_nEndPos-m_aaRecords[nPart][i].m_nStartPos)+1, EMBLOCKSIZE);
}
else if(m_aaRecords[nPart][i].m_BBRStatus == BBR_VERIFIED){
aDataVerified[k] += (m_aaRecords[nPart][i].m_nEndPos-m_aaRecords[nPart][i].m_nStartPos)+1;
}
}
}
}
}
for(int k = 0; k < aGuiltyClients.GetCount(); k++){
// calculate the percentage of corrupted data for each client and ban
// him if the limit is reached
int nCorruptPercentage;
if ((aDataVerified[k] + aDataCorrupt[k]) > 0)
nCorruptPercentage = (int)(((uint64)aDataCorrupt[k]*100)/(aDataVerified[k] + aDataCorrupt[k]));
else {
AddDebugLogLine(DLP_HIGH, false, _T("CorruptionBlackBox: Programm Error: No records for guilty client found!"));
ASSERT( false );
nCorruptPercentage = 0;
}
if ( nCorruptPercentage > CBB_BANTHRESHOLD){
CUpDownClient* pEvilClient = theApp.clientlist->FindClientByIP(aGuiltyClients[k]);
if (pEvilClient != NULL){
AddDebugLogLine(DLP_HIGH, false, _T("CorruptionBlackBox: Banning: Found client which send %s of %s corrupted data, %s"), CastItoXBytes(aDataCorrupt[k]), CastItoXBytes((aDataVerified[k] + aDataCorrupt[k])), pEvilClient->DbgGetClientInfo());
theApp.clientlist->AddTrackClient(pEvilClient);
pEvilClient->Ban(_T("Identified as sender of corrupt data"));
}
else{
AddDebugLogLine(DLP_HIGH, false, _T("CorruptionBlackBox: Banning: Found client which send %s of %s corrupted data, %s"), CastItoXBytes(aDataCorrupt[k]), CastItoXBytes((aDataVerified[k] + aDataCorrupt[k])), ipstr(aGuiltyClients[k]));
theApp.clientlist->AddBannedClient(aGuiltyClients[k]);
}
}
else{
CUpDownClient* pSuspectClient = theApp.clientlist->FindClientByIP(aGuiltyClients[k]);
if (pSuspectClient != NULL){
AddDebugLogLine(DLP_DEFAULT, false, _T("CorruptionBlackBox: Reporting: Found client which probably send %s of %s corrupted data, but it is within the acceptable limit, %s"), CastItoXBytes(aDataCorrupt[k]), CastItoXBytes((aDataVerified[k] + aDataCorrupt[k])), pSuspectClient->DbgGetClientInfo());
theApp.clientlist->AddTrackClient(pSuspectClient);
}
else
AddDebugLogLine(DLP_DEFAULT, false, _T("CorruptionBlackBox: Reporting: Found client which probably send %s of %s corrupted data, but it is within the acceptable limit, %s"), CastItoXBytes(aDataCorrupt[k]), CastItoXBytes((aDataVerified[k] + aDataCorrupt[k])), ipstr(aGuiltyClients[k]));
}
}
}
}
void CCorruptionBlackBox::VerifiedData(uint32 nStartPos, uint32 nEndPos){
if (nEndPos - nStartPos >= PARTSIZE){
ASSERT( false );
return;
}
// convert pos to relative block pos
uint16 nPart = nStartPos / PARTSIZE;
uint32 nRelStartPos = nStartPos - nPart*PARTSIZE;
uint32 nRelEndPos = nEndPos - nPart*PARTSIZE;
if (nRelEndPos >= PARTSIZE){
ASSERT( false );
return;
}
if (nPart >= m_aaRecords.GetCount()){
//ASSERT( false );
m_aaRecords.SetSize(nPart+1);
}
uint32 nDbgVerifiedBytes = 0;
//uint32 nDbgOldEntries = m_aaRecords[nPart].GetCount();
#ifdef _DEBUG
CMap<int, int, int, int> mapDebug;
#endif
for (int i= 0; i < m_aaRecords[nPart].GetCount(); i++){
if (m_aaRecords[nPart][i].m_BBRStatus == BBR_NONE || m_aaRecords[nPart][i].m_BBRStatus == BBR_VERIFIED){
if (m_aaRecords[nPart][i].m_nStartPos >= nRelStartPos && m_aaRecords[nPart][i].m_nEndPos <= nRelEndPos){
nDbgVerifiedBytes += (m_aaRecords[nPart][i].m_nEndPos-m_aaRecords[nPart][i].m_nStartPos)+1;
m_aaRecords[nPart][i].m_BBRStatus = BBR_VERIFIED;
DEBUG_ONLY(mapDebug.SetAt(m_aaRecords[nPart][i].m_dwIP, 1));
}
else if (m_aaRecords[nPart][i].m_nStartPos < nRelStartPos && m_aaRecords[nPart][i].m_nEndPos > nRelEndPos){
// need to split it 2*
uint32 nTmpEndPos1 = m_aaRecords[nPart][i].m_nEndPos;
uint32 nTmpStartPos1 = nRelEndPos + 1;
uint32 nTmpStartPos2 = m_aaRecords[nPart][i].m_nStartPos;
uint32 nTmpEndPos2 = nRelStartPos - 1;
m_aaRecords[nPart][i].m_nEndPos = nRelEndPos;
m_aaRecords[nPart][i].m_nStartPos = nRelStartPos;
m_aaRecords[nPart].Add(CCBBRecord(nTmpStartPos1, nTmpEndPos1, m_aaRecords[nPart][i].m_dwIP, m_aaRecords[nPart][i].m_BBRStatus));
m_aaRecords[nPart].Add(CCBBRecord(nTmpStartPos2, nTmpEndPos2, m_aaRecords[nPart][i].m_dwIP, m_aaRecords[nPart][i].m_BBRStatus));
nDbgVerifiedBytes += (m_aaRecords[nPart][i].m_nEndPos-m_aaRecords[nPart][i].m_nStartPos)+1;
m_aaRecords[nPart][i].m_BBRStatus = BBR_VERIFIED;
DEBUG_ONLY(mapDebug.SetAt(m_aaRecords[nPart][i].m_dwIP, 1));
}
else if (m_aaRecords[nPart][i].m_nStartPos >= nRelStartPos && m_aaRecords[nPart][i].m_nStartPos <= nRelEndPos){
// need to split it
uint32 nTmpEndPos = m_aaRecords[nPart][i].m_nEndPos;
uint32 nTmpStartPos = nRelEndPos + 1;
m_aaRecords[nPart][i].m_nEndPos = nRelEndPos;
m_aaRecords[nPart].Add(CCBBRecord(nTmpStartPos, nTmpEndPos, m_aaRecords[nPart][i].m_dwIP, m_aaRecords[nPart][i].m_BBRStatus));
nDbgVerifiedBytes += (m_aaRecords[nPart][i].m_nEndPos-m_aaRecords[nPart][i].m_nStartPos)+1;
m_aaRecords[nPart][i].m_BBRStatus = BBR_VERIFIED;
DEBUG_ONLY(mapDebug.SetAt(m_aaRecords[nPart][i].m_dwIP, 1));
}
else if (m_aaRecords[nPart][i].m_nEndPos >= nRelStartPos && m_aaRecords[nPart][i].m_nEndPos <= nRelEndPos){
// need to split it
uint32 nTmpStartPos = m_aaRecords[nPart][i].m_nStartPos;
uint32 nTmpEndPos = nRelStartPos - 1;
m_aaRecords[nPart][i].m_nStartPos = nRelStartPos;
m_aaRecords[nPart].Add(CCBBRecord(nTmpStartPos, nTmpEndPos, m_aaRecords[nPart][i].m_dwIP, m_aaRecords[nPart][i].m_BBRStatus));
nDbgVerifiedBytes += (m_aaRecords[nPart][i].m_nEndPos-m_aaRecords[nPart][i].m_nStartPos)+1;
m_aaRecords[nPart][i].m_BBRStatus = BBR_VERIFIED;
DEBUG_ONLY(mapDebug.SetAt(m_aaRecords[nPart][i].m_dwIP, 1));
}
}
}
/*#ifdef _DEBUG
uint32 nClients = mapDebug.GetCount();
#else
uint32 nClients = 0;
#endif
AddDebugLogLine(DLP_DEFAULT, false, _T("Found and marked %u recorded bytes of %u as verified in the CorruptionBlackBox records, %u(%u) records found, %u different clients"), nDbgVerifiedBytes, (nEndPos-nStartPos)+1, m_aaRecords[nPart].GetCount(), nDbgOldEntries, nClients);*/
}
// Mark a piece of data as good or bad.
// Piece is removed from the chunk list and added to the client's record.
void CCorruptionBlackBox::VerifiedData(bool ok, uint16 nPart, uint32 nRelStartPos, uint32 nRelEndPos) {
if (nRelStartPos > nRelEndPos) {
wxFAIL;
return;
}
CRecordList & list = m_Records[nPart];
#ifdef __DEBUG__
std::map<uint32, bool> mapDebug;
uint32 nDbgVerifiedBytes = 0;
size_t listsize1 = list.size();
#endif
for (CRecordList::iterator it1 = list.begin(); it1 != list.end();) {
CRecordList::iterator it = it1++;
uint32 & curStart = it->m_nStartPos;
uint32 & curEnd = it->m_nEndPos;
uint32 ip = it->m_dwIP;
uint32 data = 0;
if (curStart >= nRelStartPos && curStart <= nRelEndPos) {
// [arg
// [cur
if (curEnd > nRelEndPos) {
// [arg]
// [cur]
data = nRelEndPos - curStart + 1;
curStart = nRelEndPos + 1;
} else {
// [arg ]
// [cur]
data = curEnd - curStart + 1;
list.erase(it);
}
} else if (curStart < nRelStartPos && curEnd >= nRelStartPos) {
// [arg
// [cur
if (curEnd > nRelEndPos) {
// [arg]
// [cur ]
data = nRelEndPos - nRelStartPos + 1;
// split it: insert new block before current block
list.insert(it, CCBBRecord(curStart, nRelStartPos - 1, ip));
curStart = nRelEndPos + 1;
} else {
// [arg]
// [cur]
data = curEnd - nRelStartPos + 1;
curEnd = nRelStartPos - 1;
}
// else no overlap
}
if (data) {
if (ok) {
m_goodClients[ip].m_downloaded += data;
} else {
// corrupted data records are always counted as at least blocksize or bigger
m_badClients[ip].m_downloaded += (data > EMBLOCKSIZE) ? data : EMBLOCKSIZE;;
}
DEBUG_ONLY(nDbgVerifiedBytes += data);
DEBUG_ONLY(mapDebug[ip] = 1);
}
}
DEBUG_ONLY(size_t listsize2 = list.size());
// when everything is added to the stats drop the whole record
if (list.empty()) {
m_Records.erase(nPart);
}
#ifdef __DEBUG__
AddDebugLogLineN(logPartFile, CFormat(wxT("CorruptionBlackBox(%s): found and marked %d recorded bytes of %d as %s in part %d, %d records found, %d records left, %d different clients"))
% m_partNumber % nDbgVerifiedBytes % (nRelEndPos-nRelStartPos+1) % (ok ? wxT("verified") : wxT("corrupt"))
% nPart % listsize1 % listsize2 % mapDebug.size());
#endif
}
