void AssetUpload::start() {
if (QThread::currentThread() != thread()) {
QMetaObject::invokeMethod(this, "start");
return;
}
if (_data.isEmpty() && !_filename.isEmpty()) {
// try to open the file at the given filename
QFile file { _filename };
if (file.open(QIODevice::ReadOnly)) {
_data = file.readAll();
} else {
// we couldn't open the file - set the error result
_error = FileOpenError;
// emit that we are done
emit finished(this, QString());
return;
}
}
// ask the AssetClient to upload the asset and emit the proper signals from the passed callback
auto assetClient = DependencyManager::get<AssetClient>();
if (!_filename.isEmpty()) {
qCDebug(asset_client) << "Attempting to upload" << _filename << "to asset-server.";
}
assetClient->uploadAsset(_data, [this](bool responseReceived, AssetServerError error, const QString& hash){
if (!responseReceived) {
_error = NetworkError;
} else {
switch (error) {
case AssetServerError::NoError:
_error = NoError;
break;
case AssetServerError::AssetTooLarge:
_error = TooLarge;
break;
case AssetServerError::PermissionDenied:
_error = PermissionDenied;
break;
case AssetServerError::FileOperationFailed:
_error = ServerFileError;
break;
default:
_error = FileOpenError;
break;
}
}
if (_error == NoError && hash == hashData(_data).toHex()) {
saveToCache(getATPUrl(hash), _data);
}
emit finished(this, hash);
});
}
QString ContentDisplay::generateFileMD5(QString fileName)
{
QFile contentFile(fileName);
if(!contentFile.open(QFile::ReadOnly)) {
displayErrorMessage("Could not open content file: \n" + fileName);
return QString("");
}
QString hashData(QCryptographicHash::hash(contentFile.readAll(), QCryptographicHash::Md5).toHex().constData());
contentFile.close();
return hashData;
}
static int processSNI( INOUT SSL_HANDSHAKE_INFO *handshakeInfo,
INOUT STREAM *stream,
IN_LENGTH_SHORT_Z const int extLength,
const BOOLEAN isServer )
{
BYTE nameBuffer[ MAX_DNS_SIZE + 8 ];
int listLen, nameLen, status;
assert( isWritePtr( handshakeInfo, sizeof( SSL_HANDSHAKE_INFO ) ) );
assert( isWritePtr( stream, sizeof( STREAM ) ) );
REQUIRES( extLength >= 0 && extLength < MAX_INTLENGTH_SHORT );
/* If we're the client then the server should have sent us an empty
extension */
if( !isServer )
return( ( extLength != 0 ) ? CRYPT_ERROR_BADDATA : CRYPT_OK );
/* Remember that we've seen the server-name extension so that we can
send a zero-length reply to the client */
handshakeInfo->needSNIResponse = TRUE;
/* Read the extension wrapper */
status = listLen = readUint16( stream );
if( cryptStatusError( status ) )
return( status );
if( listLen != extLength - UINT16_SIZE || \
listLen < 1 + UINT16_SIZE || \
listLen >= MAX_INTLENGTH_SHORT )
return( CRYPT_ERROR_BADDATA );
/* Read the name type and length */
if( sgetc( stream ) != 0 ) /* Name type 0 = hostname */
return( CRYPT_ERROR_BADDATA );
status = nameLen = readUint16( stream );
if( cryptStatusError( status ) )
return( status );
if( nameLen != listLen - ( 1 + UINT16_SIZE ) || \
nameLen < MIN_DNS_SIZE || nameLen > MAX_DNS_SIZE )
return( CRYPT_ERROR_BADDATA );
/* Read the SNI and hash it */
status = sread( stream, nameBuffer, nameLen );
if( cryptStatusError( status ) )
return( status );
hashData( handshakeInfo->hashedSNI, KEYID_SIZE, nameBuffer, nameLen );
handshakeInfo->hashedSNIpresent = TRUE;
return( CRYPT_OK );
}
void AssetServer::run() {
ThreadedAssignment::commonInit(ASSET_SERVER_LOGGING_TARGET_NAME, NodeType::AssetServer);
auto nodeList = DependencyManager::get<NodeList>();
nodeList->addNodeTypeToInterestSet(NodeType::Agent);
_resourcesDirectory = QDir(QCoreApplication::applicationDirPath()).filePath("resources/assets");
if (!_resourcesDirectory.exists()) {
qDebug() << "Creating resources directory";
_resourcesDirectory.mkpath(".");
}
qDebug() << "Serving files from: " << _resourcesDirectory.path();
// Scan for new files
qDebug() << "Looking for new files in asset directory";
auto files = _resourcesDirectory.entryInfoList(QDir::Files);
QRegExp filenameRegex { "^[a-f0-9]{" + QString::number(SHA256_HASH_HEX_LENGTH) + "}(\\..+)?$" };
for (const auto& fileInfo : files) {
auto filename = fileInfo.fileName();
if (!filenameRegex.exactMatch(filename)) {
qDebug() << "Found file: " << filename;
if (!fileInfo.isReadable()) {
qDebug() << "\tCan't open file for reading: " << filename;
continue;
}
// Read file
QFile file { fileInfo.absoluteFilePath() };
file.open(QFile::ReadOnly);
QByteArray data = file.readAll();
auto hash = hashData(data);
auto hexHash = hash.toHex();
qDebug() << "\tMoving " << filename << " to " << hexHash;
file.rename(_resourcesDirectory.absoluteFilePath(hexHash) + "." + fileInfo.suffix());
}
}
}
void UploadAssetTask::run() {
auto data = _receivedMessage->getMessage();
QBuffer buffer { &data };
buffer.open(QIODevice::ReadOnly);
MessageID messageID;
buffer.read(reinterpret_cast<char*>(&messageID), sizeof(messageID));
uint64_t fileSize;
buffer.read(reinterpret_cast<char*>(&fileSize), sizeof(fileSize));
qDebug() << "UploadAssetTask reading a file of " << fileSize << "bytes from"
<< uuidStringWithoutCurlyBraces(_senderNode->getUUID());
auto replyPacket = NLPacket::create(PacketType::AssetUploadReply);
replyPacket->writePrimitive(messageID);
if (fileSize > MAX_UPLOAD_SIZE) {
replyPacket->writePrimitive(AssetServerError::AssetTooLarge);
} else {
QByteArray fileData = buffer.read(fileSize);
auto hash = hashData(fileData);
auto hexHash = hash.toHex();
qDebug() << "Hash for uploaded file from" << uuidStringWithoutCurlyBraces(_senderNode->getUUID())
<< "is: (" << hexHash << ") ";
QFile file { _resourcesDir.filePath(QString(hexHash)) };
bool existingCorrectFile = false;
if (file.exists()) {
// check if the local file has the correct contents, otherwise we overwrite
if (file.open(QIODevice::ReadOnly) && hashData(file.readAll()) == hash) {
qDebug() << "Not overwriting existing verified file: " << hexHash;
existingCorrectFile = true;
replyPacket->writePrimitive(AssetServerError::NoError);
replyPacket->write(hash);
} else {
qDebug() << "Overwriting an existing file whose contents did not match the expected hash: " << hexHash;
file.close();
}
}
if (!existingCorrectFile) {
if (file.open(QIODevice::WriteOnly) && file.write(fileData) == qint64(fileSize)) {
qDebug() << "Wrote file" << hexHash << "to disk. Upload complete";
file.close();
replyPacket->writePrimitive(AssetServerError::NoError);
replyPacket->write(hash);
} else {
qWarning() << "Failed to upload or write to file" << hexHash << " - upload failed.";
// upload has failed - remove the file and return an error
auto removed = file.remove();
if (!removed) {
qWarning() << "Removal of failed upload file" << hexHash << "failed.";
}
replyPacket->writePrimitive(AssetServerError::FileOperationFailed);
}
}
}
auto nodeList = DependencyManager::get<NodeList>();
nodeList->sendPacket(std::move(replyPacket), *_senderNode);
}
void AssetRequest::start() {
if (QThread::currentThread() != thread()) {
QMetaObject::invokeMethod(this, "start", Qt::AutoConnection);
return;
}
if (_state != NotStarted) {
qCWarning(asset_client) << "AssetRequest already started.";
return;
}
// in case we haven't parsed a valid hash, return an error now
if (!isValidHash(_hash)) {
_error = InvalidHash;
_state = Finished;
emit finished(this);
return;
}
// Try to load from cache
_data = loadFromCache(getUrl());
if (!_data.isNull()) {
_error = NoError;
_loadedFromCache = true;
_state = Finished;
emit finished(this);
return;
}
_state = WaitingForData;
auto assetClient = DependencyManager::get<AssetClient>();
auto that = QPointer<AssetRequest>(this); // Used to track the request's lifetime
auto hash = _hash;
_assetRequestID = assetClient->getAsset(_hash, _byteRange.fromInclusive, _byteRange.toExclusive,
[this, that, hash](bool responseReceived, AssetServerError serverError, const QByteArray& data) {
if (!that) {
qCWarning(asset_client) << "Got reply for dead asset request " << hash << "- error code" << _error;
// If the request is dead, return
return;
}
_assetRequestID = INVALID_MESSAGE_ID;
if (!responseReceived) {
_error = NetworkError;
} else if (serverError != AssetServerError::NoError) {
switch (serverError) {
case AssetServerError::AssetNotFound:
_error = NotFound;
break;
case AssetServerError::InvalidByteRange:
_error = InvalidByteRange;
break;
default:
_error = UnknownError;
break;
}
} else {
if (!_byteRange.isSet() && hashData(data).toHex() != _hash) {
// the hash of the received data does not match what we expect, so we return an error
_error = HashVerificationFailed;
}
if (_error == NoError) {
_data = data;
_totalReceived += data.size();
emit progress(_totalReceived, data.size());
if (!_byteRange.isSet()) {
saveToCache(getUrl(), data);
}
}
}
if (_error != NoError) {
qCWarning(asset_client) << "Got error retrieving asset" << _hash << "- error code" << _error;
}
_state = Finished;
emit finished(this);
}, [this, that](qint64 totalReceived, qint64 total) {
if (!that) {
// If the request is dead, return
return;
}
emit progress(totalReceived, total);
});
}
void AssetServer::run() {
ThreadedAssignment::commonInit(ASSET_SERVER_LOGGING_TARGET_NAME, NodeType::AssetServer);
auto nodeList = DependencyManager::get<NodeList>();
nodeList->addNodeTypeToInterestSet(NodeType::Agent);
const QString RESOURCES_PATH = "assets";
_resourcesDirectory = QDir(ServerPathUtils::getDataDirectory()).filePath(RESOURCES_PATH);
qDebug() << "Creating resources directory";
_resourcesDirectory.mkpath(".");
bool noExistingAssets = !_resourcesDirectory.exists() \
|| _resourcesDirectory.entryList(QDir::Files).size() == 0;
if (noExistingAssets) {
qDebug() << "Asset resources directory not found, searching for existing asset resources";
QString oldDataDirectory = QCoreApplication::applicationDirPath();
auto oldResourcesDirectory = QDir(oldDataDirectory).filePath("resources/" + RESOURCES_PATH);
if (QDir(oldResourcesDirectory).exists()) {
qDebug() << "Existing assets found in " << oldResourcesDirectory << ", copying to " << _resourcesDirectory;
QDir resourcesParentDirectory = _resourcesDirectory.filePath("..");
if (!resourcesParentDirectory.exists()) {
qDebug() << "Creating data directory " << resourcesParentDirectory.absolutePath();
resourcesParentDirectory.mkpath(".");
}
auto files = QDir(oldResourcesDirectory).entryList(QDir::Files);
for (auto& file : files) {
auto from = oldResourcesDirectory + QDir::separator() + file;
auto to = _resourcesDirectory.absoluteFilePath(file);
qDebug() << "\tCopying from " << from << " to " << to;
QFile::copy(from, to);
}
}
}
qDebug() << "Serving files from: " << _resourcesDirectory.path();
// Scan for new files
qDebug() << "Looking for new files in asset directory";
auto files = _resourcesDirectory.entryInfoList(QDir::Files);
QRegExp filenameRegex { "^[a-f0-9]{" + QString::number(SHA256_HASH_HEX_LENGTH) + "}(\\..+)?$" };
for (const auto& fileInfo : files) {
auto filename = fileInfo.fileName();
if (!filenameRegex.exactMatch(filename)) {
qDebug() << "Found file: " << filename;
if (!fileInfo.isReadable()) {
qDebug() << "\tCan't open file for reading: " << filename;
continue;
}
// Read file
QFile file { fileInfo.absoluteFilePath() };
file.open(QFile::ReadOnly);
QByteArray data = file.readAll();
auto hash = hashData(data);
auto hexHash = hash.toHex();
qDebug() << "\tMoving " << filename << " to " << hexHash;
file.rename(_resourcesDirectory.absoluteFilePath(hexHash) + "." + fileInfo.suffix());
}
}
}
