static std::shared_ptr<RetainedFileRecord> GetRetainedFileRecord(StringSection<ResChar> filename)
{
// We should normalize to avoid problems related to
// case insensitivity and slash differences
ResolvedAssetFile assetName;
MakeAssetName(assetName, filename);
auto hash = Hash64(assetName._fn);
{
ScopedLock(RetainedRecordsLock);
auto i = LowerBound(RetainedRecords, hash);
if (i!=RetainedRecords.end() && i->first == hash) {
return i->second;
}
// we should call "AttachFileSystemMonitor" before we query for the
// file's current modification time
auto newRecord = std::make_shared<RetainedFileRecord>(assetName._fn);
RegisterFileDependency(newRecord, assetName._fn);
newRecord->_state._timeMarker = GetFileModificationTime(assetName._fn);
RetainedRecords.insert(i, std::make_pair(hash, newRecord));
return std::move(newRecord);
}
}
void Selector::Update(ISelectable *p)
{
ScopedLock lock = ScopedLock(&m_mutex);
m_modified = true;
UpdateMap(p->GetFD(this));
WakeUp();
}
bool FMallocBinned::validateHeap()
{
#ifdef USE_COARSE_GRAIN_LOCKS
FScopeLock ScopedLock(&AccessGuard);
#endif
for( int32_t i = 0; i < PoolCount; i++ )
{
FPoolTable* table = &poolTable[i];
#ifdef USE_FINE_GRAIN_LOCKS
std::lock_guard<std::mutex> tableLock(table->mutex);
#endif
for( FPoolInfo** poolPtr = &table->firstPool; *poolPtr; poolPtr = &(*poolPtr)->next )
{
FPoolInfo* pool = *poolPtr;
FAssert(pool->prevLink == poolPtr);
FAssert(!!pool->firstMem);
for( FFreeMem* free = pool->firstMem; free; free = free->next )
{
FAssert(free->numFreeBlocks > 0);
}
}
for( FPoolInfo** poolPtr = &table->exhaustedPool; *poolPtr; poolPtr = &(*poolPtr)->next )
{
FPoolInfo* pool = *poolPtr;
FAssert(pool->prevLink == poolPtr);
FAssert(!pool->firstMem);
}
}
return true;
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// @{
/// Shader dynamic linking
intrusive_ptr<ID3D::ClassLinkage> ObjectFactory::CreateClassLinkage() const
{
ScopedLock(_attachedData->_creationLock);
ID3D::ClassLinkage* tempPtr = 0;
auto hresult = _device->CreateClassLinkage(&tempPtr);
return D3DDevice_FinalizeCreate(tempPtr, hresult, nullptr);
}
Selector::~Selector()
{
m_loop = false;
WakeUp();
Thread::Stop();
do
{
ScopedLock lock = ScopedLock(&m_mutex);
restart:
m_modified = false;
for(std::map<int, ISelectable *>::iterator it = m_map.begin(); it != m_map.end(); it++)
{
it->second->DoClose(this);
if (m_modified)
goto restart;
}
} while(0);
if (m_map.size() > 0)
{
abort(); //Tried to delete selector with items left in it
}
if (close(m_controlfd) < 0)
abort();
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// @{
/// Misc
intrusive_ptr<ID3D::DeviceContext> ObjectFactory::CreateDeferredContext() const
{
ScopedLock(_attachedData->_creationLock);
ID3D::DeviceContext* tempPtr = nullptr;
auto hresult = _device->CreateDeferredContext(0, &tempPtr);
return D3DDevice_FinalizeCreate(tempPtr, hresult, nullptr);
}
void TContainerHolder::RemoveLeftovers() {
TError error;
for (auto hy: Hierarchies) {
std::vector<TCgroup> cgroups;
error = hy->Cgroup(PORTO_ROOT_CGROUP).ChildsAll(cgroups);
if (error)
L_ERR() << "Cannot dump porto " << hy->Type << " cgroups : "
<< error << std::endl;
for (auto cg = cgroups.rbegin(); cg != cgroups.rend(); cg++) {
std::string name = cg->Name.substr(strlen(PORTO_ROOT_CGROUP) + 1);
if (Containers.count(name))
continue;
if (!cg->IsEmpty())
(void)cg->KillAll(9);
(void)cg->Remove();
}
}
for (auto it: Containers) {
auto container = it.second;
if (container->Prop->Get<bool>(P_WEAK)) {
auto holder_lock = ScopedLock();
L_ACT() << "Destroy weak container " << it.first << std::endl;
Destroy(holder_lock, container);
}
}
}
bool InvalidAssetManager::HasInvalidAssets() const
{
if (_pimpl->_active) {
ScopedLock(_pimpl->_assetsLock);
return !_pimpl->_assets.empty();
}
return false;
}
bool ClingoPropagator::isModel(Solver& s) {
POTASSCO_REQUIRE(prop_ == trail_.size(), "Assignment not propagated");
if (call_->checkMode() == ClingoPropagatorCheck_t::Total) {
Control ctrl(*this, s);
ScopedLock(call_->lock(), call_->propagator(), Inc(epoch_))->check(ctrl);
return addClause(s, 0u) && s.numFreeVars() == 0 && s.queueSize() == 0;
}
return true;
}
intrusive_ptr<ID3D::InputLayout> ObjectFactory::CreateInputLayout(
const D3D11_INPUT_ELEMENT_DESC inputElements[], unsigned inputElementsCount,
const void * byteCode, size_t byteCodeSize) const
{
ScopedLock(_attachedData->_creationLock);
ID3D::InputLayout* tempPtr = nullptr;
auto hresult = _device->CreateInputLayout(inputElements, inputElementsCount, byteCode, byteCodeSize, &tempPtr);
return D3DDevice_FinalizeCreate(tempPtr, hresult, nullptr);
}
void Selector::WakeUp()
{
uint64_t x = 1;
ScopedLock lock = ScopedLock(&m_mutex);
if (write(m_controlfd, &x, sizeof(x)) != sizeof(x))
{
abort();
}
}
void Selector::FindHighestFD()
{
ScopedLock lock = ScopedLock(&m_mutex);
int high = m_controlfd;
for(std::map<int, ISelectable *>::iterator it = m_map.begin(); it != m_map.end(); it++)
{
if (it->first > high)
high = it->first;
}
m_maxfd = high;
}
bool ClingoPropagator::propagateFixpoint(Clasp::Solver& s, Clasp::PostPropagator*) {
POTASSCO_REQUIRE(prop_ <= trail_.size(), "Invalid propagate");
for (Control ctrl(*this, s, state_prop); prop_ != trail_.size() || front_ < (int32)s.numAssignedVars();) {
if (prop_ != trail_.size()) {
// create copy because trail might change during call to user propagation
temp_.assign(trail_.begin() + prop_, trail_.end());
prop_ = static_cast<uint32>(trail_.size());
ScopedLock(call_->lock(), call_->propagator(), Inc(epoch_))->propagate(ctrl, Potassco::toSpan(temp_));
}
else {
registerUndo(s);
front_ = (int32)s.numAssignedVars();
ScopedLock(call_->lock(), call_->propagator(), Inc(epoch_))->check(ctrl);
}
if (!addClause(s, state_prop) || (s.queueSize() && !s.propagateUntil(this))) {
return false;
}
}
return true;
}
void InvalidAssetManager::MarkValid(const ResChar name[])
{
if (_pimpl->_active) {
ScopedLock(_pimpl->_assetsLock);
auto hashName = Hash64(name);
auto i = LowerBound(_pimpl->_assets, hashName);
if (i != _pimpl->_assets.end() && i->first == hashName) {
_pimpl->_assets.erase(i);
}
}
}
/** Helper to log any ET event. Used by all the LogXXX functions. */
void FAnalyticsProviderET::RecordEvent(const FString& EventName, const TArray<FAnalyticsEventAttribute>& Attributes)
{
// There are much better ways to do this, but since most events are recorded and handled on the same (game) thread,
// this is probably mostly fine for now, and simply favoring not crashing at the moment
FScopeLock ScopedLock(&CachedEventsCS);
CachedEvents.Add(FAnalyticsEventEntry(EventName, Attributes));
// if we aren't caching events, flush immediately. This is really only for debugging as it will significantly affect bandwidth.
if (!bShouldCacheEvents)
{
FlushEvents();
}
}
intrusive_ptr<ID3D::GeometryShader> ObjectFactory::CreateGeometryShaderWithStreamOutput(
const void* data, size_t size,
const D3D11_SO_DECLARATION_ENTRY* declEntries,
unsigned declEntryCount, const unsigned bufferStrides[], unsigned stridesCount,
unsigned rasterizedStreamIndex, ID3D::ClassLinkage* linkage) const
{
ScopedLock(_attachedData->_creationLock);
ID3D::GeometryShader* tempPtr = nullptr;
auto hresult = _device->CreateGeometryShaderWithStreamOutput(
data, size, declEntries, declEntryCount, bufferStrides, stridesCount,
rasterizedStreamIndex, linkage, &tempPtr);
return D3DDevice_FinalizeCreate(tempPtr, hresult, nullptr);
}
FArchive* FPakFile::GetSharedReader(IPlatformFile* LowerLevel)
{
uint32 Thread = FPlatformTLS::GetCurrentThreadId();
FArchive* PakReader = NULL;
{
FScopeLock ScopedLock(&CriticalSection);
TAutoPtr<FArchive>* ExistingReader = ReaderMap.Find(Thread);
if (ExistingReader)
{
PakReader = *ExistingReader;
}
}
if (!PakReader)
{
// Create a new FArchive reader and pass it to the new handle.
if (LowerLevel != NULL)
{
IFileHandle* PakHandle = LowerLevel->OpenRead(*GetFilename());
if (PakHandle)
{
PakReader = CreatePakReader(*PakHandle, *GetFilename());
}
}
else
{
PakReader = CreatePakReader(*GetFilename());
}
if (!PakReader)
{
UE_LOG(LogPakFile, Fatal, TEXT("Unable to create pak \"%s\" handle"), *GetFilename());
}
{
FScopeLock ScopedLock(&CriticalSection);
ReaderMap.Emplace(Thread, PakReader);
}
}
return PakReader;
}
bool TContainerHolder::RestoreFromStorage() {
std::vector<std::shared_ptr<TKeyValueNode>> nodes;
auto holder_lock = ScopedLock();
TError error = Storage->ListNodes(nodes);
if (error) {
L_ERR() << "Can't list key-value nodes: " << error << std::endl;
return false;
}
auto name2node = SortNodes(nodes);
bool restored = false;
for (auto &pair : name2node) {
auto node = pair.second;
auto name = pair.first;
L_ACT() << "Found " << name << " container in kvs" << std::endl;
kv::TNode n;
error = node->Load(n);
if (error)
continue;
restored = true;
error = Restore(holder_lock, name, n);
if (error) {
L_ERR() << "Can't restore " << name << ": " << error << std::endl;
Statistics->RestoreFailed++;
node->Remove();
continue;
}
// FIXME since v1.0 we need to cleanup kvalue nodes with old naming
if (TKeyValueStorage::Get(n, P_RAW_NAME, name))
node->Remove();
}
if (restored) {
for (auto &c: Containers) {
if (c.second->IsLostAndRestored()) {
ScheduleCgroupSync();
break;
}
}
}
return restored;
}
void InvalidAssetManager::MarkInvalid(const rstring& name, const rstring& errorString)
{
if (_pimpl->_active) {
ScopedLock(_pimpl->_assetsLock);
auto hashName = Hash64(name);
auto i = LowerBound(_pimpl->_assets, hashName);
if (i != _pimpl->_assets.end() && i->first == hashName) {
assert(i->second._name == name);
i->second._errorString = errorString;
} else {
_pimpl->_assets.insert(
i, std::make_pair(hashName, AssetRef { name, errorString }));
}
}
}
void Selector::Add(ISelectable *p)
{
int fd = p->GetFD(this);
if (fd < 0 || fd >= FD_SETSIZE)
abort(); //Invalid FD this is a bug
if (m_map.find(fd) != m_map.end())
abort(); //Duplicate FD?
ScopedLock lock = ScopedLock(&m_mutex);
m_modified = true;
m_map[fd] = p;
UpdateMap(fd);
FindHighestFD();
WakeUp();
}
void ClingoPropagator::undoLevel(Solver& s) {
POTASSCO_REQUIRE(s.decisionLevel() == level_, "Invalid undo");
uint32 beg = undo_.back();
undo_.pop_back();
if (prop_ > beg) {
Potassco::LitSpan change = Potassco::toSpan(&trail_[0] + beg, prop_ - beg);
ScopedLock(call_->lock(), call_->propagator(), Inc(epoch_))->undo(Control(*this, s), change);
prop_ = beg;
}
trail_.resize(beg);
if (front_ != INT32_MAX) {
front_ = -1;
--level_;
}
else {
level_ = !trail_.empty() ? s.level(decodeLit(trail_.back()).var()) : 0;
}
}
bool FPakPlatformFile::Unmount(const TCHAR* InPakFilename)
{
{
FScopeLock ScopedLock(&PakListCritical);
for (int32 PakIndex = 0; PakIndex < PakFiles.Num(); PakIndex++)
{
if (PakFiles[PakIndex].PakFile->GetFilename() == InPakFilename)
{
delete PakFiles[PakIndex].PakFile;
PakFiles.RemoveAt(PakIndex);
return true;
}
}
}
return false;
}
TError TNetwork::Destroy() {
auto lock = ScopedLock();
L_ACT() << "Removing network..." << std::endl;
if (Tclass) {
TError error = Tclass->Remove();
if (error)
return error;
Tclass = nullptr;
}
if (Qdisc) {
TError error = Qdisc->Remove();
if (error)
return error;
Qdisc = nullptr;
}
return TError::Success();
}
FPakPlatformFile::~FPakPlatformFile()
{
FCoreDelegates::OnMountPak.Unbind();
FCoreDelegates::OnUnmountPak.Unbind();
// We need to flush async IO... if it hasn't been shut down already.
if (FIOSystem::HasShutdown() == false)
{
FIOSystem& IOSystem = FIOSystem::Get();
IOSystem.BlockTillAllRequestsFinishedAndFlushHandles();
}
{
FScopeLock ScopedLock(&PakListCritical);
for (int32 PakFileIndex = 0; PakFileIndex < PakFiles.Num(); PakFileIndex++)
{
delete PakFiles[PakFileIndex].PakFile;
PakFiles[PakFileIndex].PakFile = nullptr;
}
}
}
bool FPakPlatformFile::Mount(const TCHAR* InPakFilename, uint32 PakOrder, const TCHAR* InPath /*= NULL*/)
{
bool bSuccess = false;
TSharedPtr<IFileHandle> PakHandle = MakeShareable(LowerLevel->OpenRead(InPakFilename));
if (PakHandle.IsValid())
{
FPakFile* Pak = new FPakFile(LowerLevel, InPakFilename, bSigned);
if (Pak->IsValid())
{
if (InPath != NULL)
{
Pak->SetMountPoint(InPath);
}
FString PakFilename = InPakFilename;
if ( PakFilename.EndsWith(TEXT("_P.pak")) )
{
PakOrder += 100;
}
{
// Add new pak file
FScopeLock ScopedLock(&PakListCritical);
FPakListEntry Entry;
Entry.ReadOrder = PakOrder;
Entry.PakFile = Pak;
PakFiles.Add(Entry);
PakFiles.StableSort();
}
bSuccess = true;
}
else
{
UE_LOG(LogPakFile, Warning, TEXT("Failed to mount pak \"%s\", pak is invalid."), InPakFilename);
}
}
else
{
UE_LOG(LogPakFile, Warning, TEXT("Pak \"%s\" does not exist!"), InPakFilename);
}
return bSuccess;
}
void Selector::Remove(ISelectable *p)
{
int fd = p->GetFD(this);
if (fd < 0 || fd >= FD_SETSIZE)
abort(); //Invalid FD this is a bug
if (m_map.find(fd) == m_map.end())
abort(); //No FD to remove
ScopedLock lock = ScopedLock(&m_mutex);
m_modified = true;
FD_CLR(fd, &m_freads);
FD_CLR(fd, &m_fwrites);
FD_CLR(fd, &m_fexcept);
m_map.erase(m_map.find(fd));
std::map<int, struct timespec>::iterator it = m_timeout.find(fd);
if (it != m_timeout.end())
m_timeout.erase(it);
FindHighestFD();
WakeUp();
}
TError TNetwork::Update() {
if (!config().network().dynamic_ifaces())
return TError::Success();
L() << "Update network" << std::endl;
std::vector<std::shared_ptr<TNlLink>> newLinks;
auto net_lock = ScopedLock();
TError error = OpenLinks(newLinks);
if (error)
return error;
for (auto link : newLinks) {
auto i = std::find_if(Links.begin(), Links.end(),
[link](std::shared_ptr<TNlLink> i) {
return i->GetAlias() == link->GetAlias();
});
if (i == Links.end()) {
L() << "Found new link: " << link->GetAlias() << std::endl;
TError error = PrepareLink(link);
if (error)
return error;
} else {
L() << "Found existing link: " << link->GetAlias() << std::endl;
TError error = link->RefillClassCache();
if (error)
return error;
}
}
Links = newLinks;
return TError::Success();
}
TError TNetwork::Prepare() {
PORTO_ASSERT(Qdisc == nullptr);
PORTO_ASSERT(Tclass == nullptr);
PORTO_ASSERT(Filter == nullptr);
PORTO_ASSERT(Links.size() == 0);
auto lock = ScopedLock();
TError error = OpenLinks(Links);
if (error)
return error;
for (auto link : Links) {
TError error = PrepareLink(link);
if (error)
return error;
}
Qdisc = std::make_shared<TQdisc>(shared_from_this(), rootHandle, defClass);
Filter = std::make_shared<TFilter>(shared_from_this(), Qdisc);
Tclass = std::make_shared<TTclass>(shared_from_this(), Qdisc, defClass);
return TError::Success();
}
void FAnalyticsProviderET::FlushEvents()
{
// Make sure we don't try to flush too many times. When we are not caching events it's possible this can be called when there are no events in the array.
if (CachedEvents.Num() == 0)
{
return;
}
// There are much better ways to do this, but since most events are recorded and handled on the same (game) thread,
// this is probably mostly fine for now, and simply favoring not crashing at the moment
FScopeLock ScopedLock(&CachedEventsCS);
if(ensure(FModuleManager::Get().IsModuleLoaded("HTTP")))
{
FString Payload;
FDateTime CurrentTime = FDateTime::UtcNow();
if (!UseLegacyProtocol)
{
TSharedRef< TJsonWriter<TCHAR, TCondensedJsonPrintPolicy<TCHAR> > > JsonWriter = TJsonWriterFactory<TCHAR, TCondensedJsonPrintPolicy<TCHAR> >::Create(&Payload);
JsonWriter->WriteObjectStart();
JsonWriter->WriteArrayStart(TEXT("Events"));
for (int32 EventIdx = 0; EventIdx < CachedEvents.Num(); EventIdx++)
{
const FAnalyticsEventEntry& Entry = CachedEvents[EventIdx];
// event entry
JsonWriter->WriteObjectStart();
JsonWriter->WriteValue(TEXT("EventName"), Entry.EventName);
FString DateOffset = (CurrentTime - Entry.TimeStamp).ToString();
JsonWriter->WriteValue(TEXT("DateOffset"), DateOffset);
JsonWriter->WriteValue(TEXT("IsEditor"), FString::FromInt(GIsEditor));
if (Entry.Attributes.Num() > 0)
{
// optional attributes for this event
for (int32 AttrIdx = 0; AttrIdx < Entry.Attributes.Num(); AttrIdx++)
{
const FAnalyticsEventAttribute& Attr = Entry.Attributes[AttrIdx];
JsonWriter->WriteValue(Attr.AttrName, Attr.AttrValue);
}
}
JsonWriter->WriteObjectEnd();
}
JsonWriter->WriteArrayEnd();
JsonWriter->WriteObjectEnd();
JsonWriter->Close();
FString URLPath = FString::Printf(TEXT("datarouter/api/v1/public/data?SessionID=%s&AppID=%s&AppVersion=%s&UserID=%s&AppEnvironment=%s&UploadType=%s"),
*FPlatformHttp::UrlEncode(SessionID),
*FPlatformHttp::UrlEncode(APIKey),
*FPlatformHttp::UrlEncode(AppVersion),
*FPlatformHttp::UrlEncode(UserID),
*FPlatformHttp::UrlEncode(AppEnvironment),
*FPlatformHttp::UrlEncode(UploadType));
// Recreate the URLPath for logging because we do not want to escape the parameters when logging.
// We cannot simply UrlEncode the entire Path after logging it because UrlEncode(Params) != UrlEncode(Param1) & UrlEncode(Param2) ...
FString LogString = FString::Printf(TEXT("[%s] AnalyticsET URL:datarouter/api/v1/public/data?SessionID=%s&AppID=%s&AppVersion=%s&UserID=%s&AppEnvironment=%s&UploadType=%s. Payload:%s"),
*APIKey,
*SessionID,
*APIKey,
*AppVersion,
*UserID,
*AppEnvironment,
*UploadType,
*Payload);
UE_LOG(LogAnalytics, VeryVerbose, TEXT("%s"), *LogString);
// Duplicate the same log message with a separate category. This is used as an "last chance" backup on the servers in the unlikely case
// if the backend lost the events due to overload - then we can scrape the logs manually for them.
UE_LOG(LogAnalyticsDumpEventPayload, Log, TEXT("%s"), *LogString);
// Create/send Http request for an event
TSharedRef<IHttpRequest> HttpRequest = FHttpModule::Get().CreateRequest();
HttpRequest->SetHeader(TEXT("Content-Type"), TEXT("application/json; charset=utf-8"));
HttpRequest->SetURL(APIServer + URLPath);
HttpRequest->SetVerb(TEXT("POST"));
HttpRequest->SetContentAsString(Payload);
// Don't set a response callback if we are in our destructor, as the instance will no longer be there to call.
if (!bInDestructor)
{
HttpRequest->OnProcessRequestComplete().BindSP(this, &FAnalyticsProviderET::EventRequestComplete);
}
HttpRequest->ProcessRequest();
}
else
{
// this is a legacy pathway that doesn't accept batch payloads of cached data. We'll just send one request for each event, which will be slow for a large batch of requests at once.
for (const auto& Event : CachedEvents)
{
FString EventParams;
if (Event.Attributes.Num() > 0)
{
for (int Ndx = 0; Ndx<FMath::Min(Event.Attributes.Num(), 40); ++Ndx)
{
EventParams += FString::Printf(TEXT("&AttributeName%d=%s&AttributeValue%d=%s"),
Ndx,
*FPlatformHttp::UrlEncode(Event.Attributes[Ndx].AttrName),
Ndx,
*FPlatformHttp::UrlEncode(Event.Attributes[Ndx].AttrValue));
}
}
// log out the un-encoded values to make reading the log easier.
UE_LOG(LogAnalytics, VeryVerbose, TEXT("[%s] AnalyticsET URL:SendEvent.1?SessionID=%s&AppID=%s&AppVersion=%s&UserID=%s&EventName=%s%s"),
*APIKey,
*SessionID,
*APIKey,
*AppVersion,
*UserID,
*Event.EventName,
*EventParams);
// Create/send Http request for an event
TSharedRef<IHttpRequest> HttpRequest = FHttpModule::Get().CreateRequest();
HttpRequest->SetHeader(TEXT("Content-Type"), TEXT("text/plain"));
// Don't need to URL encode the APIServer or the EventParams, which are already encoded, and contain parameter separaters that we DON'T want encoded.
HttpRequest->SetURL(FString::Printf(TEXT("%sSendEvent.1?SessionID=%s&AppID=%s&AppVersion=%s&UserID=%s&EventName=%s%s"),
*APIServer,
*FPlatformHttp::UrlEncode(SessionID),
*FPlatformHttp::UrlEncode(APIKey),
*FPlatformHttp::UrlEncode(AppVersion),
*FPlatformHttp::UrlEncode(UserID),
*FPlatformHttp::UrlEncode(Event.EventName),
*EventParams));
HttpRequest->SetVerb(TEXT("GET"));
HttpRequest->OnProcessRequestComplete().BindRaw(this, &FAnalyticsProviderET::EventRequestComplete);
HttpRequest->ProcessRequest();
}
}
FlushEventsCountdown = MaxCachedElapsedTime;
CachedEvents.Empty();
}
}
intrusive_ptr<ID3D::Query> ObjectFactory::CreateQuery(const D3D11_QUERY_DESC* desc) const
{
ScopedLock(_attachedData->_creationLock);
return D3DDeviceCreate<ID3D::Query>(_device.get(), &ID3D::Device::CreateQuery, desc);
}
