void DownloadManager::on(TimerManagerListener::Second, uint64_t aTick) noexcept { typedef vector<pair<string, UserPtr> > TargetList; TargetList dropTargets; { Lock l(cs); DownloadList tickList; // Tick each ongoing download for(DownloadList::iterator i = downloads.begin(); i != downloads.end(); ++i) { if((*i)->getPos() > 0) { tickList.push_back(*i); (*i)->tick(); } } if(tickList.size() > 0) fire(DownloadManagerListener::Tick(), tickList); // Automatically remove or disconnect slow sources if((uint32_t)(aTick / 1000) % SETTING(AUTODROP_INTERVAL) == 0) { for(DownloadList::iterator i = downloads.begin(); i != downloads.end(); ++i) { Download* d = *i; uint64_t timeElapsed = aTick - d->getStart(); uint64_t timeInactive = aTick - d->getUserConnection().getLastActivity(); uint64_t bytesDownloaded = d->getPos(); bool timeElapsedOk = timeElapsed >= (uint32_t)SETTING(AUTODROP_ELAPSED) * 1000; bool timeInactiveOk = timeInactive <= (uint32_t)SETTING(AUTODROP_INACTIVITY) * 1000; bool speedTooLow = timeElapsedOk && timeInactiveOk && bytesDownloaded > 0 ? bytesDownloaded / timeElapsed * 1000 < (uint32_t)SETTING(AUTODROP_SPEED) : false; bool isUserList = d->getType() == Transfer::TYPE_FULL_LIST; bool onlineSourcesOk = isUserList ? true : QueueManager::getInstance()->countOnlineSources(d->getPath()) >= SETTING(AUTODROP_MINSOURCES); bool filesizeOk = !isUserList && d->getSize() >= ((int64_t)SETTING(AUTODROP_FILESIZE)) * 1024; bool dropIt = (isUserList && BOOLSETTING(AUTODROP_FILELISTS)) || (filesizeOk && BOOLSETTING(AUTODROP_ALL)); if(speedTooLow && onlineSourcesOk && dropIt) { if(BOOLSETTING(AUTODROP_DISCONNECT) && isUserList) { d->getUserConnection().disconnect(); } else { dropTargets.push_back(make_pair(d->getPath(), d->getUser())); } } } } } for(TargetList::iterator i = dropTargets.begin(); i != dropTargets.end(); ++i) { QueueManager::getInstance()->removeSource(i->first, i->second, QueueItem::Source::FLAG_SLOW_SOURCE); } }
/** Download finished! */ void DownloadManager::handleEndData(UserConnection* aSource) { dcassert(aSource->getState() == UserConnection::STATE_DONE); Download* d = aSource->getDownload(); dcassert(d != NULL); if(d->isSet(Download::FLAG_TREE_DOWNLOAD)) { d->getFile()->flush(); delete d->getFile(); d->setFile(NULL); Download* old = d->getOldDownload(); size_t bl = 1024; while(bl * old->getTigerTree().getLeaves().size() < old->getSize()) bl *= 2; old->getTigerTree().setBlockSize(bl); dcassert(old->getSize() != -1); old->getTigerTree().setFileSize(old->getSize()); old->getTigerTree().calcRoot(); if(!(*old->getTTH() == old->getTigerTree().getRoot())) { // This tree is for a different file, remove from queue... fire(DownloadManagerListener::Failed(), old, STRING(INVALID_TREE)); string target = old->getTarget(); aSource->setDownload(NULL); removeDownload(old, true); QueueManager::getInstance()->removeSource(target, aSource->getUser(), QueueItem::Source::FLAG_BAD_TREE, false); checkDownloads(aSource); return; } d->getOldDownload()->setTreeValid(true); HashManager::getInstance()->addTree(old->getTarget(), old->getTigerTree()); aSource->setDownload(d->getOldDownload()); delete d; // Ok, now we can continue to the actual file... checkDownloads(aSource); return; } u_int32_t crc = 0; bool hasCrc = (d->getCrcCalc() != NULL); // First, finish writing the file (flushing the buffers and closing the file...) try { d->getFile()->flush(); if(hasCrc) crc = d->getCrcCalc()->getFilter().getValue(); delete d->getFile(); d->setFile(NULL); d->setCrcCalc(NULL); // Check if we're anti-fragging... if(d->isSet(Download::FLAG_ANTI_FRAG)) { // Ok, rename the file to what we expect it to be... try { const string& tgt = d->getTempTarget().empty() ? d->getTarget() : d->getTempTarget(); File::renameFile(d->getDownloadTarget(), tgt); d->unsetFlag(Download::FLAG_ANTI_FRAG); } catch(const FileException& e) { dcdebug("AntiFrag: %s\n", e.getError().c_str()); // Now what? } } } catch(const FileException& e) { fire(DownloadManagerListener::Failed(), d, e.getError()); aSource->setDownload(NULL); removeDownload(d, true); removeConnection(aSource); return; } dcassert(d->getPos() == d->getSize()); dcdebug("Download finished: %s, size " I64_FMT ", downloaded " I64_FMT "\n", d->getTarget().c_str(), d->getSize(), d->getTotal()); // Check if we have some crc:s... if(BOOLSETTING(SFV_CHECK)) { SFVReader sfv(d->getTarget()); if(sfv.hasCRC()) { bool crcMatch; string tgt = d->getDownloadTarget(); if(hasCrc) { crcMatch = (crc == sfv.getCRC()); } else { // More complicated, we have to reread the file try { File ff(tgt, File::READ, File::OPEN); CalcInputStream<CRC32Filter, false> f(&ff); const size_t BUF_SIZE = 16 * 65536; AutoArray<u_int8_t> b(BUF_SIZE); size_t n = BUF_SIZE; while(f.read((u_int8_t*)b, n) > 0) ; // Keep on looping... crcMatch = (f.getFilter().getValue() == sfv.getCRC()); } catch (FileException&) { // Nope; read failed... goto noCRC; } } if(!crcMatch) { File::deleteFile(tgt); dcdebug("DownloadManager: CRC32 mismatch for %s\n", d->getTarget().c_str()); LogManager::getInstance()->message(STRING(SFV_INCONSISTENCY) + " (" + STRING(FILE) + ": " + d->getTarget() + ")"); fire(DownloadManagerListener::Failed(), d, STRING(SFV_INCONSISTENCY)); string target = d->getTarget(); aSource->setDownload(NULL); removeDownload(d, true); QueueManager::getInstance()->removeSource(target, aSource->getUser(), QueueItem::Source::FLAG_CRC_WARN, false); checkDownloads(aSource); return; } d->setFlag(Download::FLAG_CRC32_OK); dcdebug("DownloadManager: CRC32 match for %s\n", d->getTarget().c_str()); } } noCRC: if(BOOLSETTING(LOG_DOWNLOADS) && (BOOLSETTING(LOG_FILELIST_TRANSFERS) || !d->isSet(Download::FLAG_USER_LIST))) { StringMap params; params["target"] = d->getTarget(); params["user"] = aSource->getUser()->getNick(); params["hub"] = aSource->getUser()->getLastHubName(); params["hubip"] = aSource->getUser()->getLastHubAddress(); params["size"] = Util::toString(d->getSize()); params["sizeshort"] = Util::formatBytes(d->getSize()); params["chunksize"] = Util::toString(d->getTotal()); params["chunksizeshort"] = Util::formatBytes(d->getTotal()); params["actualsize"] = Util::toString(d->getActual()); params["actualsizeshort"] = Util::formatBytes(d->getActual()); params["speed"] = Util::formatBytes(d->getAverageSpeed()) + "/s"; params["time"] = Util::formatSeconds((GET_TICK() - d->getStart()) / 1000); params["sfv"] = Util::toString(d->isSet(Download::FLAG_CRC32_OK) ? 1 : 0); LOG(DOWNLOAD_AREA, Util::formatParams(SETTING(LOG_FORMAT_POST_DOWNLOAD), params)); } // Check if we need to move the file if( !d->getTempTarget().empty() && (Util::stricmp(d->getTarget().c_str(), d->getTempTarget().c_str()) != 0) ) { try { File::ensureDirectory(d->getTarget()); if(File::getSize(d->getTempTarget()) > MOVER_LIMIT) { mover.moveFile(d->getTempTarget(), d->getTarget()); } else { File::renameFile(d->getTempTarget(), d->getTarget()); } d->setTempTarget(Util::emptyString); } catch(const FileException&) { try { if(!SETTING(DOWNLOAD_DIRECTORY).empty()) { File::renameFile(d->getTempTarget(), SETTING(DOWNLOAD_DIRECTORY) + d->getTargetFileName()); } else { File::renameFile(d->getTempTarget(), Util::getFilePath(d->getTempTarget()) + d->getTargetFileName()); } } catch(const FileException&) { try { File::renameFile(d->getTempTarget(), Util::getFilePath(d->getTempTarget()) + d->getTargetFileName()); } catch(const FileException&) { // Ignore... } } } } fire(DownloadManagerListener::Complete(), d); aSource->setDownload(NULL); removeDownload(d, true, true); checkDownloads(aSource); }
Segment QueueItem::getNextSegment(int64_t blockSize, int64_t wantedSize, int64_t lastSpeed, const PartialSource::Ptr partialSource) const { if(getSize() == -1 || blockSize == 0) { return Segment(0, -1); } if(!BOOLSETTING(MULTI_CHUNK)) { if(!downloads.empty()) { return Segment(-1, 0); } int64_t start = 0; int64_t end = getSize(); if(!done.empty()) { const Segment& first = *done.begin(); if(first.getStart() > 0) { end = Util::roundUp(first.getStart(), blockSize); } else { start = Util::roundDown(first.getEnd(), blockSize); if(done.size() > 1) { const Segment& second = *(++done.begin()); end = Util::roundUp(second.getStart(), blockSize); } } } return Segment(start, std::min(getSize(), end) - start); } if(downloads.size() >= maxSegments || (BOOLSETTING(DONT_BEGIN_SEGMENT) && (size_t)(SETTING(DONT_BEGIN_SEGMENT_SPEED) * 1024) < getAverageSpeed())) { // no other segments if we have reached the speed or segment limit return Segment(-1, 0); } /* added for PFS */ vector<int64_t> posArray; vector<Segment> neededParts; if(partialSource) { posArray.reserve(partialSource->getPartialInfo().size()); // Convert block index to file position for(PartsInfo::const_iterator i = partialSource->getPartialInfo().begin(); i != partialSource->getPartialInfo().end(); i++) posArray.push_back(min(getSize(), (int64_t)(*i) * blockSize)); } /***************************/ double donePart = static_cast<double>(getDownloadedBytes()) / getSize(); // We want smaller blocks at the end of the transfer, squaring gives a nice curve... int64_t targetSize = static_cast<int64_t>(static_cast<double>(wantedSize) * std::max(0.25, (1. - (donePart * donePart)))); if(targetSize > blockSize) { // Round off to nearest block size targetSize = Util::roundDown(targetSize, blockSize); } else { targetSize = blockSize; } int64_t start = 0; int64_t curSize = targetSize; while(start < getSize()) { int64_t end = std::min(getSize(), start + curSize); Segment block(start, end - start); bool overlaps = false; for(SegmentConstIter i = done.begin(); !overlaps && i != done.end(); ++i) { if(curSize <= blockSize) { int64_t dstart = i->getStart(); int64_t dend = i->getEnd(); // We accept partial overlaps, only consider the block done if it is fully consumed by the done block if(dstart <= start && dend >= end) { overlaps = true; } } else { overlaps = block.overlaps(*i); } } for(auto i = downloads.begin(); !overlaps && i != downloads.end(); ++i) { overlaps = block.overlaps((*i)->getSegment()); } if(!overlaps) { if(partialSource) { // store all chunks we could need for(vector<int64_t>::const_iterator j = posArray.begin(); j < posArray.end(); j += 2){ if( (*j <= start && start < *(j+1)) || (start <= *j && *j < end) ) { int64_t b = max(start, *j); int64_t e = min(end, *(j+1)); // segment must be blockSize aligned dcassert(b % blockSize == 0); dcassert(e % blockSize == 0 || e == getSize()); neededParts.push_back(Segment(b, e - b)); } } } else { return block; } } if(overlaps && (curSize > blockSize)) { curSize -= blockSize; } else { start = end; curSize = targetSize; } } if(!neededParts.empty()) { // select random chunk for download dcdebug("Found chunks: %d\n", neededParts.size()); Segment& selected = neededParts[Util::rand(0, neededParts.size())]; selected.setSize(std::min(selected.getSize(), targetSize)); // request only wanted size return selected; } if(partialSource == NULL && BOOLSETTING(OVERLAP_CHUNKS) && lastSpeed > 0) { // overlap slow running chunk for(auto i = downloads.begin(); i != downloads.end(); ++i) { Download* d = *i; // current chunk mustn't be already overlapped if(d->getOverlapped()) continue; // current chunk must be running at least for 4 seconds if(d->getStart() == 0 || GET_TICK() - d->getStart() < 4000) continue; // current chunk mustn't be finished in next 20 seconds if(d->getSecondsLeft() < 20) continue; // overlap current chunk at last block boundary int64_t pos = d->getPos() - (d->getPos() % blockSize); int64_t size = d->getSize() - pos; // new user should finish this chunk more than 2x faster int64_t newChunkLeft = size / lastSpeed; if(2 * newChunkLeft < d->getSecondsLeft()) { dcdebug("Overlapping... old user: %I64d s, new user: %I64d s\n", d->getSecondsLeft(), newChunkLeft); return Segment(d->getStartPos() + pos, size, true); } } } return Segment(0, 0); }
/** Download finished! */ void DownloadManager::endData(UserConnection* aSource) { dcassert(aSource->getState() == UserConnection::STATE_RUNNING); Download* d = aSource->getDownload(); dcassert(d != NULL); if(d->getType() == Transfer::TYPE_TREE) { d->getFile()->flush(); int64_t bl = 1024; while(bl * (int64_t)d->getTigerTree().getLeaves().size() < d->getTigerTree().getFileSize()) bl *= 2; d->getTigerTree().setBlockSize(bl); d->getTigerTree().calcRoot(); if(!(d->getTTH() == d->getTigerTree().getRoot())) { // This tree is for a different file, remove from queue... removeDownload(d); fire(DownloadManagerListener::Failed(), d, _("Full tree does not match TTH root")); QueueManager::getInstance()->removeSource(d->getPath(), aSource->getUser(), QueueItem::Source::FLAG_BAD_TREE, false); QueueManager::getInstance()->putDownload(d, false); checkDownloads(aSource); return; } d->setTreeValid(true); } else { // First, finish writing the file (flushing the buffers and closing the file...) try { d->getFile()->flush(); } catch(const Exception& e) { d->resetPos(); failDownload(aSource, e.getError()); return; } aSource->setSpeed(d->getAverageSpeed()); aSource->updateChunkSize(d->getTigerTree().getBlockSize(), d->getSize(), GET_TICK() - d->getStart()); dcdebug("Download finished: %s, size " I64_FMT ", downloaded " I64_FMT "\n", d->getPath().c_str(), static_cast<long long int>(d->getSize()), static_cast<long long int>(d->getPos())); if(BOOLSETTING(LOG_DOWNLOADS) && (BOOLSETTING(LOG_FILELIST_TRANSFERS) || d->getType() == Transfer::TYPE_FILE)) { logDownload(aSource, d); } } removeDownload(d); fire(DownloadManagerListener::Complete(), d); QueueManager::getInstance()->putDownload(d, true); checkDownloads(aSource); }