api_return QueueApi::handleAddFileBundle(ApiRequest& aRequest) {
const auto& reqJson = aRequest.getRequestBody();
string targetDirectory, targetFileName;
TargetUtil::TargetType targetType;
QueueItemBase::Priority prio;
Deserializer::deserializeDownloadParams(aRequest.getRequestBody(), targetDirectory, targetFileName, targetType, prio);
BundlePtr b = nullptr;
try {
b = QueueManager::getInstance()->createFileBundle(
targetDirectory + targetFileName,
JsonUtil::getField<int64_t>("size", reqJson, false),
Deserializer::deserializeTTH(reqJson),
Deserializer::deserializeHintedUser(reqJson),
JsonUtil::getField<time_t>("time", reqJson, false),
0,
prio
);
}
catch (const Exception& e) {
aRequest.setResponseErrorStr(e.getError());
return websocketpp::http::status_code::internal_server_error;
}
if (b) {
json retJson = {
{ "id", b->getToken() }
};
aRequest.setResponseBody(retJson);
}
return websocketpp::http::status_code::ok;
}
void Master::onStart() {
CliArguments &args = CliArguments::getInstance();
#ifdef WIN32
m_settings = new Settings("mbedsys.org", "NodeBus", QSettings::NativeFormat);
#else
m_settings = new Settings(args.getValue("config").toString(), QSettings::NativeFormat);
#endif
m_settings->define("master/pidfile", tr("Path of the file where the service PID will be written in"),
NODEBUS_DEFAULT_PIDFILE);
m_settings->define("master/bundle-rootpath", tr("Bundle root directory path"),
NODEBUS_DEFAULT_PLUGIN_DIR_PATH);
m_settings->define("master/registry-service-name", tr("Registry service name"),
NODEBUS_DEFAULT_REGISTRY_SERVICE_NAME);
if (args.isEnabled("edit-settings")) {
m_settings->setup();
throw ExitApplicationException();
}
QString path = m_settings->value("master/bundle-rootpath").toString();
QDirIterator it(path, QStringList("*.so"), QDir::Files, QDirIterator::Subdirectories);
logFiner() << "Search for bundles in the directory " << path;
while (it.hasNext()) {
QString file = it.next();
logFinest() << "Found file " << file;
try {
BundlePtr bundle = new Bundle(file);
m_bundles[bundle->property("Bundle-SymbolicName").toString()] = bundle;
logFine() << "Found bundle " << bundle->property("Bundle-Name") << " (" << bundle->property("Bundle-SymbolicName") << ')';
} catch (Exception &e) {
logWarn() << "Invalid bundle file " << file << " (" << e.message() << ')';
}
}
if (m_bundles.isEmpty()) {
throw ApplicationException("No valid bundle found in the directory " + path);
}
}
void BundleQueue::forEachPath(const BundlePtr& aBundle, const string& aFilePath, PathInfoHandler&& aHandler) noexcept {
auto currentPath = Util::getFilePath(aFilePath);
auto& pathInfos = bundlePaths[const_cast<string*>(&aBundle->getTarget())];
while (true) {
dcassert(currentPath.find(aBundle->getTarget()) != string::npos);
// TODO: make this case insensitive
auto infoIter = pathInfos.find(currentPath);
PathInfo* info;
// New pathinfo?
if (infoIter == pathInfos.end()) {
info = addPathInfo(currentPath, aBundle);
} else {
info = *infoIter;
}
aHandler(*info);
// Empty pathinfo?
if (info->finishedFiles == 0 && info->queuedFiles == 0) {
dcassert(info->size == 0);
removePathInfo(info);
}
if (currentPath.length() == aBundle->getTarget().length()) {
break;
}
currentPath = Util::getParentDir(currentPath);
}
}
QueueItemList BundleQueue::getSearchItems(const BundlePtr& aBundle) const noexcept {
if (aBundle->getQueueItems().size() <= 1) {
return aBundle->getQueueItems();
}
// File bundles shouldn't come here
QueueItemList searchItems;
auto pathInfos = getPathInfos(aBundle->getTarget());
if (!pathInfos) {
return searchItems;
}
{
// Get the main directories inside this bundle
// We'll choose a single search item from each main directory later
// This helps with getting best coverage for complex bundles that aren't
// shared with same structure by most users
StringSet mainBundlePaths;
for (const auto& pathInfo : *pathInfos) {
if (pathInfo->queuedFiles == 0) {
continue;
}
mainBundlePaths.insert(AirUtil::getReleaseDirLocal(pathInfo->path, false));
}
auto searchPaths = pickRandomItems(mainBundlePaths, 5);
for (const auto& path : searchPaths) {
QueueItemList ql;
// Get all queued files inside this directory
// This doesn't scale so well for large bundles but shouldn't cause issues with maximum of 5 paths
aBundle->getDirQIs(path, ql);
auto searchItem = QueueItem::pickSearchItem(ql);
// We'll also get search items for parent directories that have no files directly inside them
// so we need to filter duplicate items as well
if (searchItem && find_if(searchItems, QueueItem::HashComp(searchItem->getTTH())) == searchItems.end()) {
searchItems.push_back(searchItem);
}
}
}
#if 0
StringList targets;
for (const auto& qi : searchItems) {
targets.push_back(qi->getTarget());
}
LogManager::getInstance()->message("Search items from bundle " + aBundle->getName() + ": " + Util::listToString(targets), LogMessage::SEV_INFO);
#endif
return searchItems;
}
std::string QueueBundleUtils::formatBundleType(const BundlePtr& aBundle) noexcept {
if (aBundle->isFileBundle()) {
return Format::formatFileType(aBundle->getTarget());
} else {
size_t files = 0, folders = 0;
QueueManager::getInstance()->getBundleContent(aBundle, files, folders);
return Format::formatFolderContent(files, folders);
}
}
std::string QueueUtils::getStringInfo(const BundlePtr& b, int aPropertyName) noexcept {
switch (aPropertyName) {
case QueueApi::PROP_NAME: return b->getName();
case QueueApi::PROP_TARGET: return b->getTarget();
case QueueApi::PROP_TYPE: return formatBundleType(b);
case QueueApi::PROP_STATUS: return formatBundleStatus(b);
case QueueApi::PROP_PRIORITY: return AirUtil::getPrioText(b->getPriority());
case QueueApi::PROP_SOURCES: return formatBundleSources(b);
default: dcassert(0); return Util::emptyString;
}
}
void BundleQueue::addBundle(BundlePtr& aBundle) noexcept {
bundles[aBundle->getToken()] = aBundle;
if (aBundle->filesCompleted()) {
aBundle->setStatus(Bundle::STATUS_COMPLETED);
return;
}
aBundle->setStatus(Bundle::STATUS_QUEUED);
aBundle->setDownloadedBytes(0); //sets to downloaded segments
addSearchPrio(aBundle);
}
void UserQueue::setBundlePriority(BundlePtr& aBundle, QueueItemBase::Priority p) noexcept {
dcassert(!aBundle->isFinished());
HintedUserList sources;
aBundle->getSourceUsers(sources);
for(const auto& u: sources)
removeBundle(aBundle, u);
aBundle->setPriority(p);
for(const auto& u: sources)
addBundle(aBundle, u);
}
void UserQueue::setBundlePriority(BundlePtr& aBundle, Bundle::Priority p) {
dcassert(!aBundle->isFinished());
HintedUserList sources;
aBundle->getSources(sources);
for(auto& u: sources)
removeBundle(aBundle, u);
aBundle->setPriority(p);
for(auto& u: sources)
addBundle(aBundle, u);
}
std::string QueueUtils::formatBundleType(const BundlePtr& aBundle) noexcept {
if (aBundle->isFileBundle()) {
return Format::formatFileType(aBundle->getTarget());
} else {
size_t files = 0;
size_t folders = 0;
{
RLock l(QueueManager::getInstance()->getCS());
files = aBundle->getQueueItems().size() + aBundle->getFinishedFiles().size();
folders = aBundle->getDirectories().size();
}
return Format::formatFolderContent(files, folders);
}
}
void UserQueue::addQI(QueueItemPtr& qi, const HintedUser& aUser, bool aIsBadSource /*false*/) noexcept{
if (qi->getPriority() == QueueItem::HIGHEST) {
auto& l = userPrioQueue[aUser.user];
l.insert(upper_bound(l.begin(), l.end(), qi, QueueItem::SizeSortOrder()), qi);
}
BundlePtr bundle = qi->getBundle();
if (bundle) {
aUser.user->addQueued(qi->getSize());
if (bundle->addUserQueue(qi, aUser, aIsBadSource)) {
addBundle(bundle, aUser);
} else {
dcassert(userBundleQueue.find(aUser.user) != userBundleQueue.end());
}
}
}
size_t BundleQueue::getDirectoryCount(const BundlePtr& aBundle) const noexcept {
auto pathInfos = getPathInfos(aBundle->getTarget());
if (!pathInfos) {
return 0;
}
return (*pathInfos).size();
}
void UserQueue::addQI(QueueItemPtr& qi, const HintedUser& aUser, bool newBundle /*false*/, bool isBadSource /*false*/) {
if (qi->getPriority() == QueueItem::HIGHEST) {
auto& l = userPrioQueue[aUser.user];
l.insert(upper_bound(l.begin(), l.end(), qi, QueueItem::SizeSortOrder()), qi);
}
BundlePtr bundle = qi->getBundle();
if (bundle) {
if (bundle->addUserQueue(qi, aUser, isBadSource)) {
addBundle(bundle, aUser);
if (!newBundle) {
QueueManager::getInstance()->fire(QueueManagerListener::BundleSources(), bundle);
}
} else {
dcassert(userBundleQueue.find(aUser.user) != userBundleQueue.end());
}
}
}
void UserQueue::removeQI(QueueItemPtr& qi, const UserPtr& aUser, bool removeRunning /*true*/, bool addBad /*false*/, bool fireSources /*false*/) {
if(removeRunning) {
qi->removeDownloads(aUser);
}
dcassert(qi->isSource(aUser));
BundlePtr bundle = qi->getBundle();
if (bundle) {
if (!bundle->isSource(aUser)) {
return;
}
if (qi->getBundle()->removeUserQueue(qi, aUser, addBad)) {
removeBundle(bundle, aUser);
if (fireSources) {
QueueManager::getInstance()->fire(QueueManagerListener::BundleSources(), bundle);
}
} else {
dcassert(userBundleQueue.find(aUser) != userBundleQueue.end());
}
}
if (qi->getPriority() == QueueItem::HIGHEST) {
auto j = userPrioQueue.find(aUser);
dcassert(j != userPrioQueue.end());
if (j == userPrioQueue.end()) {
return;
}
auto& l = j->second;
auto i = find(l.begin(), l.end(), qi);
dcassert(i != l.end());
if (i == l.end()) {
return;
}
l.erase(i);
if(l.empty()) {
userPrioQueue.erase(j);
}
}
}
api_return QueueApi::handleAddDirectoryBundle(ApiRequest& aRequest) {
const auto& reqJson = aRequest.getRequestBody();
BundleFileInfo::List files;
for (const auto& fileJson : reqJson["files"]) {
files.push_back(BundleFileInfo(
JsonUtil::getField<string>("name", reqJson),
Deserializer::deserializeTTH(fileJson),
JsonUtil::getField<int64_t>("size", reqJson),
JsonUtil::getField<time_t>("time", reqJson),
Deserializer::deserializePriority(fileJson, true))
);
}
BundlePtr b = nullptr;
std::string errors;
try {
b = QueueManager::getInstance()->createDirectoryBundle(
JsonUtil::getField<string>("target", reqJson),
Deserializer::deserializeHintedUser(reqJson),
files,
Deserializer::deserializePriority(reqJson, true),
JsonUtil::getField<time_t>("time", reqJson),
errors
);
}
catch (const QueueException& e) {
aRequest.setResponseErrorStr(e.getError());
return websocketpp::http::status_code::internal_server_error;
}
if (b) {
json retJson = {
{ "id", b->getToken() },
{ "errors", errors }
};
aRequest.setResponseBody(retJson);
}
return websocketpp::http::status_code::ok;
}
double QueueUtils::getNumericInfo(const BundlePtr& b, int aPropertyName) noexcept {
dcassert(b->getSize() != 0);
switch (aPropertyName) {
case QueueApi::PROP_SIZE: return (double)b->getSize();
case QueueApi::PROP_BYTES_DOWNLOADED: return (double)b->getDownloadedBytes();
case QueueApi::PROP_PRIORITY: return b->getPriority();
case QueueApi::PROP_TIME_ADDED: return (double)b->getTimeAdded();
case QueueApi::PROP_TIME_FINISHED: return (double)b->getTimeFinished();
case QueueApi::PROP_SPEED: return (double)b->getSpeed();
case QueueApi::PROP_SECONDS_LEFT: return (double)b->getSecondsLeft();
default: dcassert(0); return 0;
}
}
void UserQueue::removeQI(QueueItemPtr& qi, const UserPtr& aUser, bool removeRunning /*true*/, Flags::MaskType reason) noexcept{
if(removeRunning) {
qi->removeDownloads(aUser);
}
dcassert(qi->isSource(aUser));
BundlePtr bundle = qi->getBundle();
if (bundle) {
if (!bundle->isSource(aUser)) {
return;
}
aUser->removeQueued(qi->getSize());
if (qi->getBundle()->removeUserQueue(qi, aUser, reason)) {
removeBundle(bundle, aUser);
} else {
dcassert(userBundleQueue.find(aUser) != userBundleQueue.end());
}
}
if (qi->getPriority() == QueueItem::HIGHEST) {
auto j = userPrioQueue.find(aUser);
dcassert(j != userPrioQueue.end());
if (j == userPrioQueue.end()) {
return;
}
auto& l = j->second;
auto i = find(l.begin(), l.end(), qi);
dcassert(i != l.end());
if (i == l.end()) {
return;
}
l.erase(i);
if(l.empty()) {
userPrioQueue.erase(j);
}
}
}
void BundleQueue::addBundleItem(QueueItemPtr& aQI, BundlePtr& aBundle) noexcept {
dcassert(!aQI->getBundle());
aBundle->addQueue(aQI);
aQI->setBundle(aBundle);
if (!aBundle->isFileBundle()) {
forEachPath(aBundle, aQI->getTarget(), [&](PathInfo& aInfo) {
if (aQI->isDownloaded()) {
aInfo.finishedFiles++;
} else {
aInfo.queuedFiles++;
}
aInfo.size += aQI->getSize();
});
}
if (!aQI->isDownloaded()) {
queueSize += aQI->getSize();
}
}
std::string QueueUtils::formatBundleStatus(const BundlePtr& aBundle) noexcept {
auto getPercentage = [&] {
return aBundle->getSize() > 0 ? (double)aBundle->getDownloadedBytes() *100.0 / (double)aBundle->getSize() : 0;
};
switch (aBundle->getStatus()) {
case Bundle::STATUS_NEW:
case Bundle::STATUS_QUEUED: {
if (aBundle->isPausedPrio())
return STRING_F(PAUSED_PCT, getPercentage());
if (aBundle->getSpeed() > 0) { // Bundle->isRunning() ?
return STRING_F(RUNNING_PCT, getPercentage());
}
else {
return STRING_F(WAITING_PCT, getPercentage());
}
}
case Bundle::STATUS_RECHECK: return STRING(RECHECKING);
case Bundle::STATUS_DOWNLOADED: return STRING(MOVING);
case Bundle::STATUS_MOVED: return STRING(DOWNLOADED);
case Bundle::STATUS_DOWNLOAD_FAILED:
case Bundle::STATUS_FAILED_MISSING:
case Bundle::STATUS_SHARING_FAILED: return aBundle->getLastError();
case Bundle::STATUS_FINISHED: return STRING(FINISHED);
case Bundle::STATUS_HASHING: return STRING(HASHING);
case Bundle::STATUS_HASH_FAILED: return STRING(HASH_FAILED);
case Bundle::STATUS_HASHED: return STRING(HASHING_FINISHED);
case Bundle::STATUS_SHARED: return STRING(SHARED);
default:
return Util::emptyString;
}
}
void BundleQueue::removeBundle(BundlePtr& aBundle) noexcept{
if (aBundle->getStatus() == Bundle::STATUS_NEW) {
return;
}
{
auto infoPtr = getPathInfos(aBundle->getTarget());
if (infoPtr) {
auto pathInfos = *infoPtr;
for (const auto& p : pathInfos) {
removePathInfo(p);
}
}
}
dcassert(aBundle->getFinishedFiles().empty());
dcassert(aBundle->getQueueItems().empty());
removeSearchPrio(aBundle);
bundles.erase(aBundle->getToken());
dcassert(bundlePaths.size() == static_cast<size_t>(boost::count_if(bundles | map_values, [](const BundlePtr& b) { return !b->isFileBundle(); })));
aBundle->deleteXmlFile();
}
json QueueUtils::serializeBundleProperty(const BundlePtr& aBundle, int aPropertyName) noexcept {
switch (aPropertyName) {
case QueueApi::PROP_SOURCES:
{
int total = 0, online = 0;
std::string str;
getBundleSourceInfo(aBundle, online, total, str);
return {
{ "online", online },
{ "total", total },
{ "str", str },
};
}
case QueueApi::PROP_STATUS:
{
return{
{ "id", aBundle->getStatus() },
{ "failed", aBundle->isFailed() },
{ "str", formatBundleStatus(aBundle) },
};
}
case QueueApi::PROP_TYPE:
{
if (aBundle->isFileBundle()) {
return Serializer::serializeFileType(aBundle->getTarget());
} else {
size_t files = 0;
size_t folders = 0;
{
RLock l(QueueManager::getInstance()->getCS());
files = aBundle->getQueueItems().size() + aBundle->getFinishedFiles().size();
folders = aBundle->getDirectories().size();
}
return Serializer::serializeFolderType(files, folders);
}
}
case QueueApi::PROP_PRIORITY: {
return serializePriority(*aBundle.get());
}
}
dcassert(0);
return json();
}
json QueueBundleUtils::serializeBundleProperty(const BundlePtr& aBundle, int aPropertyName) noexcept {
switch (aPropertyName) {
case PROP_SOURCES:
{
auto c = QueueManager::getInstance()->getSourceCount(aBundle);
return Serializer::serializeSourceCount(c);
}
case PROP_STATUS:
{
return{
{ "id", formatStatusId(aBundle) },
{ "failed", aBundle->isFailed() },
{ "finished", aBundle->getStatus() >= Bundle::STATUS_MOVED },
{ "str", formatDisplayStatus(aBundle) },
};
}
case PROP_TYPE:
{
if (aBundle->isFileBundle()) {
return Serializer::serializeFileType(aBundle->getTarget());
} else {
size_t files = 0, folders = 0;
QueueManager::getInstance()->getBundleContent(aBundle, files, folders);
return Serializer::serializeFolderType(static_cast<int>(files), static_cast<int>(folders));
}
}
case PROP_PRIORITY: {
return Serializer::serializePriority(*aBundle.get());
}
}
dcassert(0);
return nullptr;
}
string QueueBundleUtils::formatStatusId(const BundlePtr& aBundle) noexcept {
switch (aBundle->getStatus()) {
case Bundle::STATUS_NEW: return "new";
case Bundle::STATUS_QUEUED: return "queued";
case Bundle::STATUS_RECHECK: return "recheck";
case Bundle::STATUS_DOWNLOADED: return "downloaded";
case Bundle::STATUS_MOVED: return "moved";
case Bundle::STATUS_DOWNLOAD_FAILED: return "download_failed";
case Bundle::STATUS_FAILED_MISSING: return "scan_failed_files_missing";
case Bundle::STATUS_SHARING_FAILED: return "scan_failed";
case Bundle::STATUS_FINISHED: return "finished";
case Bundle::STATUS_HASHING: return "hashing";
case Bundle::STATUS_HASH_FAILED: return "hash_failed";
case Bundle::STATUS_HASHED: return "hashed";
case Bundle::STATUS_SHARED: return "shared";
}
dcassert(0);
return Util::emptyString;
}
bool AutoSearch::onBundleRemoved(const BundlePtr& aBundle, bool finished) noexcept {
removeBundle(aBundle);
auto usingInc = usingIncrementation();
auto expired = usingInc && maxNumberReached() && finished && SETTING(AS_DELAY_HOURS) == 0 && bundles.empty();
if (finished) {
auto time = GET_TIME();
addPath(aBundle->getTarget(), time);
if (usingInc) {
if (SETTING(AS_DELAY_HOURS) > 0) {
lastIncFinish = time;
setStatus(AutoSearch::STATUS_POSTSEARCH);
expired = false;
} else {
changeNumber(true);
}
}
}
updateStatus();
return expired;
}
int QueueBundleUtils::compareBundles(const BundlePtr& a, const BundlePtr& b, int aPropertyName) noexcept {
switch (aPropertyName) {
case PROP_NAME: {
COMPARE_TYPE(a, b);
return Util::stricmp(a->getName(), b->getName());
}
case PROP_TYPE: {
COMPARE_TYPE(a, b);
if (!a->isFileBundle() && !b->isFileBundle()) {
// Directory bundles
RLock l(QueueManager::getInstance()->getCS());
auto dirsA = QueueManager::getInstance()->bundleQueue.getDirectoryCount(a);
auto dirsB = QueueManager::getInstance()->bundleQueue.getDirectoryCount(b);
if (dirsA != dirsB) {
return compare(dirsA, dirsB);
}
auto filesA = a->getQueueItems().size() + a->getFinishedFiles().size();
auto filesB = b->getQueueItems().size() + b->getFinishedFiles().size();
return compare(filesA, filesB);
}
return Util::stricmp(Util::getFileExt(a->getTarget()), Util::getFileExt(b->getTarget()));
}
case PROP_PRIORITY: {
COMPARE_FINISHED(a, b);
if (a->isFinished() != b->isFinished()) {
return a->isFinished() ? 1 : -1;
}
return compare(static_cast<int>(a->getPriority()), static_cast<int>(b->getPriority()));
}
case PROP_STATUS: {
if (a->getStatus() != b->getStatus()) {
return compare(a->getStatus(), b->getStatus());
}
return compare(
a->getPercentage(a->getDownloadedBytes()),
b->getPercentage(b->getDownloadedBytes())
);
}
case PROP_SOURCES: {
COMPARE_FINISHED(a, b);
auto countsA = QueueManager::getInstance()->getSourceCount(a);
auto countsB = QueueManager::getInstance()->getSourceCount(b);
return QueueItemBase::SourceCount::compare(countsA, countsB);
}
default:
dcassert(0);
}
return 0;
}
int QueueUtils::compareBundles(const BundlePtr& a, const BundlePtr& b, int aPropertyName) noexcept {
switch (aPropertyName) {
case QueueApi::PROP_NAME: {
if (a->isFileBundle() && !b->isFileBundle()) return 1;
if (!a->isFileBundle() && b->isFileBundle()) return -1;
return Util::stricmp(a->getName(), b->getName());
}
case QueueApi::PROP_TYPE: {
if (a->isFileBundle() != b->isFileBundle()) {
// Directories go first
return a->isFileBundle() ? 1 : -1;
}
if (!a->isFileBundle() && !b->isFileBundle()) {
// Directory bundles
RLock l(QueueManager::getInstance()->getCS());
auto dirsA = a->getDirectories().size();
auto dirsB = a->getDirectories().size();
if (dirsA != dirsB) {
return compare(dirsA, dirsB);
}
auto filesA = a->getQueueItems().size() + a->getFinishedFiles().size();
auto filesB = b->getQueueItems().size() + b->getFinishedFiles().size();
return compare(filesA, filesB);
}
return Util::stricmp(Util::getFileExt(a->getTarget()), Util::getFileExt(b->getTarget()));
}
case QueueApi::PROP_PRIORITY: {
if (a->isFinished() != b->isFinished()) {
return a->isFinished() ? 1 : -1;
}
return compare(static_cast<int>(a->getPriority()), static_cast<int>(b->getPriority()));
}
case QueueApi::PROP_STATUS: {
if (a->getStatus() != b->getStatus()) {
return compare(a->getStatus(), b->getStatus());
}
return compare(a->getDownloadedBytes(), b->getDownloadedBytes());
}
case QueueApi::PROP_SOURCES: {
if (a->isFinished() != b->isFinished()) {
return a->isFinished() ? 1 : -1;
}
int onlineA = 0, totalA = 0, onlineB = 0, totalB = 0;
std::string str;
getBundleSourceInfo(a, onlineA, totalA, str);
getBundleSourceInfo(b, onlineB, totalB, str);
if (onlineA != onlineB) {
return compare(onlineA, onlineB);
}
return compare(totalA, totalB);
}
default:
dcassert(0);
}
return 0;
}