void DownloadManager::on(UserConnectionListener::FileNotAvailable, UserConnection* aSource) throw() { Download* d = aSource->getDownload(); dcassert(d != NULL); if(d->isSet(Download::FLAG_TREE_DOWNLOAD)) { // No tree, too bad... aSource->setDownload(d->getOldDownload()); delete d->getFile(); d->setFile(NULL); delete d; checkDownloads(aSource); return; } dcdebug("File Not Available: %s\n", d->getTarget().c_str()); if(d->getFile()) { delete d->getFile(); d->setFile(NULL); d->setCrcCalc(NULL); } fire(DownloadManagerListener::Failed(), d, d->getTargetFileName() + ": " + STRING(FILE_NOT_AVAILABLE)); aSource->setDownload(NULL); QueueManager::getInstance()->removeSource(d->getTarget(), aSource->getUser(), QueueItem::Source::FLAG_FILE_NOT_AVAILABLE, false); removeDownload(d, false, false); checkDownloads(aSource); }
/** Download finished! */ void DownloadManager::endData(UserConnection* aSource) { dcassert(aSource->getState() == UserConnection::STATE_RUNNING); Download* d = aSource->getDownload(); dcassert(d != NULL); if(d->getType() == Transfer::TYPE_TREE) { d->getFile()->flush(); int64_t bl = 1024; while(bl * (int64_t)d->getTigerTree().getLeaves().size() < d->getTigerTree().getFileSize()) bl *= 2; d->getTigerTree().setBlockSize(bl); d->getTigerTree().calcRoot(); if(!(d->getTTH() == d->getTigerTree().getRoot())) { // This tree is for a different file, remove from queue... removeDownload(d); fire(DownloadManagerListener::Failed(), d, _("Full tree does not match TTH root")); QueueManager::getInstance()->removeSource(d->getPath(), aSource->getUser(), QueueItem::Source::FLAG_BAD_TREE, false); QueueManager::getInstance()->putDownload(d, false); checkDownloads(aSource); return; } d->setTreeValid(true); } else { // First, finish writing the file (flushing the buffers and closing the file...) try { d->getFile()->flush(); } catch(const Exception& e) { d->resetPos(); failDownload(aSource, e.getError()); return; } aSource->setSpeed(d->getAverageSpeed()); aSource->updateChunkSize(d->getTigerTree().getBlockSize(), d->getSize(), GET_TICK() - d->getStart()); dcdebug("Download finished: %s, size " I64_FMT ", downloaded " I64_FMT "\n", d->getPath().c_str(), static_cast<long long int>(d->getSize()), static_cast<long long int>(d->getPos())); if(BOOLSETTING(LOG_DOWNLOADS) && (BOOLSETTING(LOG_FILELIST_TRANSFERS) || d->getType() == Transfer::TYPE_FILE)) { logDownload(aSource, d); } } removeDownload(d); fire(DownloadManagerListener::Complete(), d); QueueManager::getInstance()->putDownload(d, true); checkDownloads(aSource); }
void DownloadManager::on(Command::SND, UserConnection* aSource, const Command& cmd) throw() { int64_t bytes = Util::toInt64(cmd.getParam(3)); if(cmd.getParam(0) == "tthl") { if(aSource->getState() != UserConnection::STATE_TREE) { dcdebug("DM::SND Bad state, ignoring\n"); return; } Download* d = aSource->getDownload(); d->setFile(new TreeOutputStream(d->getOldDownload()->getTigerTree())); d->setSize(bytes); d->setPos(0); dcassert(d->isSet(Download::FLAG_TREE_DOWNLOAD)); aSource->setState(UserConnection::STATE_DONE); if(cmd.hasFlag("ZL", 4)) { d->setFile(new FilteredOutputStream<UnZFilter, true>(d->getFile())); } aSource->setDataMode(); } else if(cmd.getParam(0) == "file") { if(aSource->getState() != UserConnection::STATE_FILELENGTH) { dcdebug("DM::onFileLength Bad state, ignoring\n"); return; } if(prepareFile(aSource, (bytes == -1) ? -1 : aSource->getDownload()->getPos() + bytes)) { aSource->setDataMode(); } } }
void DownloadManager::on(UserConnectionListener::Data, UserConnection* aSource, const uint8_t* aData, size_t aLen) noexcept { Download* d = aSource->getDownload(); dcassert(d != NULL); try { d->addPos(d->getFile()->write(aData, aLen), aLen); d->tick(); if(d->getFile()->eof()) { endData(aSource); aSource->setLineMode(0); } } catch(const Exception& e) { failDownload(aSource, e.getError()); } }
void DownloadManager::on(UserConnectionListener::Data, UserConnection* aSource, const u_int8_t* aData, size_t aLen) throw() { Download* d = aSource->getDownload(); dcassert(d != NULL); try { d->addPos(d->getFile()->write(aData, aLen), aLen); if(d->getPos() > d->getSize()) { throw Exception(STRING(TOO_MUCH_DATA)); } else if(d->getPos() == d->getSize()) { handleEndData(aSource); aSource->setLineMode(); } } catch(const RollbackException& e) { string target = d->getTarget(); QueueManager::getInstance()->removeSource(target, aSource->getUser(), QueueItem::Source::FLAG_ROLLBACK_INCONSISTENCY); fire(DownloadManagerListener::Failed(), d, e.getError()); d->resetPos(); aSource->setDownload(NULL); removeDownload(d, true); removeConnection(aSource); return; } catch(const FileException& e) { fire(DownloadManagerListener::Failed(), d, e.getError()); d->resetPos(); aSource->setDownload(NULL); removeDownload(d, true); removeConnection(aSource); return; } catch(const Exception& e) { fire(DownloadManagerListener::Failed(), d, e.getError()); // Nuke the bytes we have written, this is probably a compression error d->resetPos(); aSource->setDownload(NULL); removeDownload(d, true); removeConnection(aSource); return; } }
/** Download finished! */ void DownloadManager::handleEndData(UserConnection* aSource) { dcassert(aSource->getState() == UserConnection::STATE_DONE); Download* d = aSource->getDownload(); dcassert(d != NULL); if(d->isSet(Download::FLAG_TREE_DOWNLOAD)) { d->getFile()->flush(); delete d->getFile(); d->setFile(NULL); Download* old = d->getOldDownload(); size_t bl = 1024; while(bl * old->getTigerTree().getLeaves().size() < old->getSize()) bl *= 2; old->getTigerTree().setBlockSize(bl); dcassert(old->getSize() != -1); old->getTigerTree().setFileSize(old->getSize()); old->getTigerTree().calcRoot(); if(!(*old->getTTH() == old->getTigerTree().getRoot())) { // This tree is for a different file, remove from queue... fire(DownloadManagerListener::Failed(), old, STRING(INVALID_TREE)); string target = old->getTarget(); aSource->setDownload(NULL); removeDownload(old, true); QueueManager::getInstance()->removeSource(target, aSource->getUser(), QueueItem::Source::FLAG_BAD_TREE, false); checkDownloads(aSource); return; } d->getOldDownload()->setTreeValid(true); HashManager::getInstance()->addTree(old->getTarget(), old->getTigerTree()); aSource->setDownload(d->getOldDownload()); delete d; // Ok, now we can continue to the actual file... checkDownloads(aSource); return; } u_int32_t crc = 0; bool hasCrc = (d->getCrcCalc() != NULL); // First, finish writing the file (flushing the buffers and closing the file...) try { d->getFile()->flush(); if(hasCrc) crc = d->getCrcCalc()->getFilter().getValue(); delete d->getFile(); d->setFile(NULL); d->setCrcCalc(NULL); // Check if we're anti-fragging... if(d->isSet(Download::FLAG_ANTI_FRAG)) { // Ok, rename the file to what we expect it to be... try { const string& tgt = d->getTempTarget().empty() ? d->getTarget() : d->getTempTarget(); File::renameFile(d->getDownloadTarget(), tgt); d->unsetFlag(Download::FLAG_ANTI_FRAG); } catch(const FileException& e) { dcdebug("AntiFrag: %s\n", e.getError().c_str()); // Now what? } } } catch(const FileException& e) { fire(DownloadManagerListener::Failed(), d, e.getError()); aSource->setDownload(NULL); removeDownload(d, true); removeConnection(aSource); return; } dcassert(d->getPos() == d->getSize()); dcdebug("Download finished: %s, size " I64_FMT ", downloaded " I64_FMT "\n", d->getTarget().c_str(), d->getSize(), d->getTotal()); // Check if we have some crc:s... if(BOOLSETTING(SFV_CHECK)) { SFVReader sfv(d->getTarget()); if(sfv.hasCRC()) { bool crcMatch; string tgt = d->getDownloadTarget(); if(hasCrc) { crcMatch = (crc == sfv.getCRC()); } else { // More complicated, we have to reread the file try { File ff(tgt, File::READ, File::OPEN); CalcInputStream<CRC32Filter, false> f(&ff); const size_t BUF_SIZE = 16 * 65536; AutoArray<u_int8_t> b(BUF_SIZE); size_t n = BUF_SIZE; while(f.read((u_int8_t*)b, n) > 0) ; // Keep on looping... crcMatch = (f.getFilter().getValue() == sfv.getCRC()); } catch (FileException&) { // Nope; read failed... goto noCRC; } } if(!crcMatch) { File::deleteFile(tgt); dcdebug("DownloadManager: CRC32 mismatch for %s\n", d->getTarget().c_str()); LogManager::getInstance()->message(STRING(SFV_INCONSISTENCY) + " (" + STRING(FILE) + ": " + d->getTarget() + ")"); fire(DownloadManagerListener::Failed(), d, STRING(SFV_INCONSISTENCY)); string target = d->getTarget(); aSource->setDownload(NULL); removeDownload(d, true); QueueManager::getInstance()->removeSource(target, aSource->getUser(), QueueItem::Source::FLAG_CRC_WARN, false); checkDownloads(aSource); return; } d->setFlag(Download::FLAG_CRC32_OK); dcdebug("DownloadManager: CRC32 match for %s\n", d->getTarget().c_str()); } } noCRC: if(BOOLSETTING(LOG_DOWNLOADS) && (BOOLSETTING(LOG_FILELIST_TRANSFERS) || !d->isSet(Download::FLAG_USER_LIST))) { StringMap params; params["target"] = d->getTarget(); params["user"] = aSource->getUser()->getNick(); params["hub"] = aSource->getUser()->getLastHubName(); params["hubip"] = aSource->getUser()->getLastHubAddress(); params["size"] = Util::toString(d->getSize()); params["sizeshort"] = Util::formatBytes(d->getSize()); params["chunksize"] = Util::toString(d->getTotal()); params["chunksizeshort"] = Util::formatBytes(d->getTotal()); params["actualsize"] = Util::toString(d->getActual()); params["actualsizeshort"] = Util::formatBytes(d->getActual()); params["speed"] = Util::formatBytes(d->getAverageSpeed()) + "/s"; params["time"] = Util::formatSeconds((GET_TICK() - d->getStart()) / 1000); params["sfv"] = Util::toString(d->isSet(Download::FLAG_CRC32_OK) ? 1 : 0); LOG(DOWNLOAD_AREA, Util::formatParams(SETTING(LOG_FORMAT_POST_DOWNLOAD), params)); } // Check if we need to move the file if( !d->getTempTarget().empty() && (Util::stricmp(d->getTarget().c_str(), d->getTempTarget().c_str()) != 0) ) { try { File::ensureDirectory(d->getTarget()); if(File::getSize(d->getTempTarget()) > MOVER_LIMIT) { mover.moveFile(d->getTempTarget(), d->getTarget()); } else { File::renameFile(d->getTempTarget(), d->getTarget()); } d->setTempTarget(Util::emptyString); } catch(const FileException&) { try { if(!SETTING(DOWNLOAD_DIRECTORY).empty()) { File::renameFile(d->getTempTarget(), SETTING(DOWNLOAD_DIRECTORY) + d->getTargetFileName()); } else { File::renameFile(d->getTempTarget(), Util::getFilePath(d->getTempTarget()) + d->getTargetFileName()); } } catch(const FileException&) { try { File::renameFile(d->getTempTarget(), Util::getFilePath(d->getTempTarget()) + d->getTargetFileName()); } catch(const FileException&) { // Ignore... } } } } fire(DownloadManagerListener::Complete(), d); aSource->setDownload(NULL); removeDownload(d, true, true); checkDownloads(aSource); }
bool DownloadManager::prepareFile(UserConnection* aSource, int64_t newSize /* = -1 */) { Download* d = aSource->getDownload(); dcassert(d != NULL); if(newSize != -1) { d->setSize(newSize); } if(d->getPos() >= d->getSize()) { // Already finished? aSource->setDownload(NULL); removeDownload(d, true, true); removeConnection(aSource); return false; } dcassert(d->getSize() != -1); string target = d->getDownloadTarget(); File::ensureDirectory(target); if(d->isSet(Download::FLAG_USER_LIST)) { if(aSource->isSet(UserConnection::FLAG_SUPPORTS_XML_BZLIST)) { target += ".xml.bz2"; } else { target += ".DcLst"; } } File* file = NULL; try { // Let's check if we can find this file in a any .SFV... int trunc = d->isSet(Download::FLAG_RESUME) ? 0 : File::TRUNCATE; file = new File(target, File::RW, File::OPEN | File::CREATE | trunc); if(d->isSet(Download::FLAG_ANTI_FRAG)) { file->setSize(d->getSize()); } file->setPos(d->getPos()); } catch(const FileException& e) { delete file; fire(DownloadManagerListener::Failed(), d, STRING(COULD_NOT_OPEN_TARGET_FILE) + e.getError()); aSource->setDownload(NULL); removeDownload(d, true); removeConnection(aSource); return false; } catch(const Exception& e) { delete file; fire(DownloadManagerListener::Failed(), d, e.getError()); aSource->setDownload(NULL); removeDownload(d, true); removeConnection(aSource); return false; } d->setFile(file); if(d->isSet(Download::FLAG_ROLLBACK)) { d->setFile(new RollbackOutputStream<true>(file, d->getFile(), (size_t)min((int64_t)SETTING(ROLLBACK), d->getSize() - d->getPos()))); } if(SETTING(BUFFER_SIZE) != 0) { d->setFile(new BufferedOutputStream<true>(d->getFile())); } bool sfvcheck = BOOLSETTING(SFV_CHECK) && (d->getPos() == 0) && (SFVReader(d->getTarget()).hasCRC()); if(sfvcheck) { d->setFlag(Download::FLAG_CALC_CRC32); Download::CrcOS* crc = new Download::CrcOS(d->getFile()); d->setCrcCalc(crc); d->setFile(crc); } if(d->getPos() == 0) { if(!d->getTreeValid() && d->getTTH() != NULL && d->getSize() < numeric_limits<size_t>::max()) { // We make a single node tree... d->getTigerTree().setFileSize(d->getSize()); d->getTigerTree().setBlockSize((size_t)d->getSize()); d->getTigerTree().getLeaves().push_back(*d->getTTH()); d->getTigerTree().calcRoot(); d->setTreeValid(true); } if(d->getTreeValid()) { d->setFile(new TigerCheckOutputStream<true>(d->getTigerTree(), d->getFile())); } } if(d->isSet(Download::FLAG_ZDOWNLOAD)) { d->setFile(new FilteredOutputStream<UnZFilter, true>(d->getFile())); } dcassert(d->getPos() != -1); d->setStart(GET_TICK()); aSource->setState(UserConnection::STATE_DONE); fire(DownloadManagerListener::Starting(), d); return true; }
void DownloadManager::startData(UserConnection* aSource, int64_t start, int64_t bytes, bool z) { Download* d = aSource->getDownload(); dcassert(d != NULL); dcdebug("Preparing " I64_FMT ":" I64_FMT ", " I64_FMT ":" I64_FMT"\n", static_cast<long long int>(d->getStartPos()), static_cast<long long int>(start), static_cast<long long int>(d->getSize()), static_cast<long long int>(bytes)); if(d->getSize() == -1) { if(bytes >= 0) { d->setSize(bytes); } else { failDownload(aSource, _("Invalid size")); return; } } else if(d->getSize() != bytes || d->getStartPos() != start) { // This is not what we requested... failDownload(aSource, _("Response does not match request")); return; } try { QueueManager::getInstance()->setFile(d); } catch(const FileException& e) { failDownload(aSource, str(F_("Could not open target file: %1%") % e.getError())); return; } catch(const Exception& e) { failDownload(aSource, e.getError()); return; } if((d->getType() == Transfer::TYPE_FILE || d->getType() == Transfer::TYPE_FULL_LIST) && SETTING(BUFFER_SIZE) > 0 ) { d->setFile(new BufferedOutputStream<true>(d->getFile())); } if(d->getType() == Transfer::TYPE_FILE) { typedef MerkleCheckOutputStream<TigerTree, true> MerkleStream; d->setFile(new MerkleStream(d->getTigerTree(), d->getFile(), d->getStartPos())); d->setFlag(Download::FLAG_TTH_CHECK); } // Check that we don't get too many bytes d->setFile(new LimitedOutputStream<true>(d->getFile(), bytes)); if(z) { d->setFlag(Download::FLAG_ZDOWNLOAD); d->setFile(new FilteredOutputStream<UnZFilter, true>(d->getFile())); } d->setStart(GET_TICK()); d->tick(); aSource->setState(UserConnection::STATE_RUNNING); fire(DownloadManagerListener::Starting(), d); if(d->getPos() == d->getSize()) { try { // Already finished? A zero-byte file list could cause this... endData(aSource); } catch(const Exception& e) { failDownload(aSource, e.getError()); } } else { aSource->setDataMode(); } }