ErrorCode CachedFileTest::WriteToFile(wstring const & filePath, wstring const & fileText)
{
wstring directoryName = Path::GetDirectoryName(filePath);
wstring tempFileName = Guid::NewGuid().ToString();
wstring tempFilePath = Path::Combine(directoryName, tempFileName);
int currentRetryCount = 0;
int maxRetryCount = 10;
ErrorCode error = ErrorCodeValue::Success;
do
{
if(currentRetryCount)
{
LOG_INFO("CachedFileTest::WriteToFile | RetryCount:{0} , Sleeping..", currentRetryCount);
// Sleep for 10 seconds
Sleep(10 * 1000);
}
File file;
error = file.TryOpen(tempFilePath, FileMode::Create, FileAccess::Write, FileShare::ReadWrite);
if (error.IsSuccess())
{
int size = static_cast<int>(fileText.length()) * 2;
int written = file.TryWrite((void*)fileText.c_str(), size);
if (error.IsSuccess())
{
written;
file.Flush();
file.Close();
File::Copy(tempFilePath, filePath, true);
File::Delete(tempFilePath, Common::NOTHROW());
}
else
{
File::Delete(tempFilePath, Common::NOTHROW());
}
}
else
{
File::Delete(tempFilePath, Common::NOTHROW());
}
} while(error.ReadValue() == ErrorCodeValue::AccessDenied
&& currentRetryCount++ < maxRetryCount);
if(error.ReadValue() != ErrorCodeValue::Success)
{
FAIL_TEST("Failed write file '{0}'", filePath);
}
return error;
}
void PlacementAndLoadBalancingUnitTest::VerifyNodeLoadQueryWithDoubleValues(
Reliability::LoadBalancingComponent::PlacementAndLoadBalancing & plb,
int nodeIndex,
std::wstring const& metricName,
uint expectedValue,
double expectedValueD,
uint remainingCapacity,
double remainingCapacityD)
{
ServiceModel::NodeLoadInformationQueryResult queryResult;
ErrorCode retValue = plb.GetNodeLoadInformationQueryResult(CreateNodeId(nodeIndex), queryResult);
VERIFY_ARE_EQUAL(ErrorCodeValue::Success, retValue.ReadValue());
for (auto itMetric = queryResult.NodeLoadMetricInformation.begin(); itMetric != queryResult.NodeLoadMetricInformation.end(); ++itMetric)
{
if (itMetric->Name == metricName)
{
VERIFY_ARE_EQUAL(itMetric->NodeLoad, expectedValue);
VERIFY_ARE_EQUAL(itMetric->NodeRemainingCapacity, remainingCapacity);
double diff = abs(itMetric->CurrentNodeLoad - expectedValueD);
VERIFY_IS_TRUE(diff < 0.000001);
diff = abs(itMetric->NodeCapacityRemaining - remainingCapacityD);
VERIFY_IS_TRUE(diff < 0.000001);
return;
}
}
VERIFY_FAIL(wformatString("Metric {0} is not present in the result for node {1}",
metricName,
nodeIndex).c_str());
}
void PlacementAndLoadBalancingUnitTest::VerifyNodeCapacity(
Reliability::LoadBalancingComponent::PlacementAndLoadBalancing & plb,
int nodeIndex,
std::wstring const& metricName,
uint expectedValue)
{
ServiceModel::NodeLoadInformationQueryResult queryResult;
ErrorCode retValue = plb.GetNodeLoadInformationQueryResult(CreateNodeId(nodeIndex), queryResult);
VERIFY_ARE_EQUAL(ErrorCodeValue::Success, retValue.ReadValue());
for (auto itMetric = queryResult.NodeLoadMetricInformation.begin(); itMetric != queryResult.NodeLoadMetricInformation.end(); ++itMetric)
{
if (itMetric->Name == metricName)
{
if (itMetric->NodeCapacity == expectedValue)
{
return;
}
VERIFY_FAIL(wformatString("Metric capacity for metric {0} ({1}) on node {2} does not match the expected value ({3})",
metricName,
itMetric->NodeLoad,
nodeIndex,
expectedValue
).c_str());
}
}
VERIFY_FAIL(wformatString("Metric {0} is not present in the result for node {1}",
metricName,
nodeIndex).c_str());
}
void ResolveNameLocationAsyncOperation::OnResolveServiceComplete(
AsyncOperationSPtr const & asyncOperation,
bool expectedCompletedSynchronously)
{
if (expectedCompletedSynchronously != asyncOperation->CompletedSynchronously)
{
return;
}
vector<ServiceTableEntry> serviceEntries;
GenerationNumber generation;
ErrorCode error = serviceResolver_.EndResolveServicePartition(asyncOperation, serviceEntries, generation);
if (!error.IsSuccess())
{
WriteInfo(
TraceComponent,
"{0} could not resolve Naming Store Service location for {1}: error = {2}",
this->TraceId,
toResolve_,
error);
// Most errors from the FM on resolve are not retryable with the following exceptions
switch (error.ReadValue())
{
case ErrorCodeValue::ServiceOffline:
case ErrorCodeValue::PartitionNotFound:
// Naming Gateway will retry on this error
error = ErrorCodeValue::SystemServiceNotFound;
break;
}
}
else
{
bool shouldRetry = false;
wstring endpointLocation;
auto first = serviceEntries.begin();
if (first->ServiceReplicaSet.IsPrimaryLocationValid)
{
endpointLocation = first->ServiceReplicaSet.PrimaryLocation;
previousVersionUsed_ = ServiceLocationVersion(first->Version, generation, 0);
}
else
{
shouldRetry = true;
}
if (shouldRetry || !SystemServiceLocation::TryParse(endpointLocation, serviceLocationResult_))
{
// Naming Gateway will retry on this error
error = ErrorCode(ErrorCodeValue::SystemServiceNotFound);
}
}
TryComplete(asyncOperation->Parent, error);
}
shared_ptr<FailoverManagerStore> FailoverManagerStoreTest::InitializeStore(
wstring ownerId,
bool shouldPass,
bool existingStore,
Common::Guid const & partitionId,
::FABRIC_REPLICA_ID replicaId,
Common::ComponentRoot const & root,
const wstring storeType)
{
UNREFERENCED_PARAMETER(ownerId);
UNREFERENCED_PARAMETER(shouldPass);
if (storeType == L"ESENT")
{
auto replicatedStore = Store::KeyValueStoreReplica::CreateForUnitTests(
partitionId,
replicaId,
Store::EseLocalStoreSettings(GetEseFilename(), GetEseDirectory()),
root);
ErrorCode error = replicatedStore->InitializeLocalStoreForUnittests(existingStore);
shared_ptr<FailoverManagerStore> store = make_shared<FailoverManagerStore>(move(replicatedStore));
if (shouldPass)
{
VERIFY_ARE_EQUAL(ErrorCodeValue::Success, error.ReadValue(), L"store.Open did not return success");
return store;
}
else
{
VERIFY_ARE_NOT_EQUAL(ErrorCodeValue::Success, error.ReadValue(), L"store.Open returned success");
return nullptr;
}
}
Common::Assert::CodingError("StoreType not found in properties");
}
void PlacementAndLoadBalancingUnitTest::VerifyPLBAction(Reliability::LoadBalancingComponent::PlacementAndLoadBalancing & plb, std::wstring const& action, std::wstring const & metricName)
{
ServiceModel::ClusterLoadInformationQueryResult loadInformationResult;
ErrorCode retValue = plb.GetClusterLoadInformationQueryResult(loadInformationResult);
VERIFY_ARE_EQUAL(ErrorCodeValue::Success, retValue.ReadValue());
for (auto loadMetricInfo : loadInformationResult.LoadMetric)
{
if (loadMetricInfo.Name == metricName || metricName == L"")
{
if (loadMetricInfo.Action == action) return;
}
}
VERIFY_FAIL(wformatString("No Match for action {0}", action).c_str());
}
void RequestReceiverContext::InternalReject(ErrorCode const & error, ActivityId const & activityId)
{
MessageUPtr nack;
if (error.Message.empty())
{
nack = FederationMessage::GetRejectFault().CreateMessage();
}
else
{
nack = FederationMessage::GetRejectFault().CreateMessage(RejectFaultBody(error));
}
nack->Headers.Add(FaultHeader(error.ReadValue(), !error.Message.empty()));
if (!activityId.IsEmpty)
{
nack->Headers.Add(FabricActivityHeader(activityId));
}
this->Reply(std::move(nack));
}
void PlacementAndLoadBalancingUnitTest::GetClusterLoadMetricInformation(
Reliability::LoadBalancingComponent::PlacementAndLoadBalancing & plb,
ServiceModel::LoadMetricInformation& loadMetricInfo,
std::wstring metricName)
{
ServiceModel::ClusterLoadInformationQueryResult queryResult;
ErrorCode result = plb.GetClusterLoadInformationQueryResult(queryResult);
VERIFY_ARE_EQUAL(ErrorCodeValue::Success, result.ReadValue());
for (auto itMetric = queryResult.LoadMetric.begin(); itMetric != queryResult.LoadMetric.end(); ++itMetric)
{
if (itMetric->Name == metricName)
{
loadMetricInfo = *itMetric;
return;
}
}
VERIFY_FAIL(L"Metric is not present in the result, fail the test");
}
void PlacementAndLoadBalancingUnitTest::GetNodeLoadMetricInformation(
Reliability::LoadBalancingComponent::PlacementAndLoadBalancing & plb,
ServiceModel::NodeLoadMetricInformation& nodeLoadMetricInfo,
int nodeIndex,
std::wstring const& metricName)
{
ServiceModel::NodeLoadInformationQueryResult queryResult;
ErrorCode retValue = plb.GetNodeLoadInformationQueryResult(CreateNodeId(nodeIndex), queryResult);
VERIFY_ARE_EQUAL(ErrorCodeValue::Success, retValue.ReadValue());
for (auto itMetric = queryResult.NodeLoadMetricInformation.begin(); itMetric != queryResult.NodeLoadMetricInformation.end(); ++itMetric)
{
if (itMetric->Name == metricName)
{
nodeLoadMetricInfo = *itMetric;
return;
}
}
VERIFY_FAIL(L"Metric is not present in the result, fail the test");
}
void LTSendBuffer::Abort()
{
if (++this->abortCount_ != 1)
{
Common::Assert::CodingError("We are doing multiple aborts: {0}", this->abortCount_.load());
}
ErrorCode sendError = connection_->fault_.IsSuccess() ? ErrorCodeValue::OperationCanceled : connection_->fault_;
const uint dropTraceLimit = 9;
uint dropCount = 0;
for (auto & message : messagesDelayedBySecurityNegotiation_)
{
if (dropCount++ < dropTraceLimit)
{
trace.DropMessageOnAbort(
message->TraceId(),
message->Actor,
message->Action,
sendError);
}
if (message->HasSendStatusCallback())
{
message->OnSendStatus(sendError.ReadValue(), move(message));
}
}
auto frame = messageQueue_.begin();
while (frame != messageQueue_.end())
{
if (frame->Message())
{
if (dropCount++ < dropTraceLimit)
{
trace.DropMessageOnAbort(
frame->Message()->TraceId(),
frame->Message()->Actor,
frame->Message()->Action,
sendError);
}
if (frame->Message()->HasSendStatusCallback())
{
auto msg = frame->Dispose();
msg->OnSendStatus(sendError.ReadValue(), move(msg));
}
else
{
//no send status callback, so message is not used to keep things alive
frame->Dispose(); // This is required as bique will not call ~Frame()
}
}
++frame;
}
TcpConnection::WriteInfo(
TraceType, connection_->TraceId(),
"abort send buffer with {0}, dropping {1} messages",
sendError,
dropCount);
messageQueue_.truncate_before(messageQueue_.cend());
totalBufferedBytes_ = 0;
sendingLength_ = 0;
}
void ProcessReportRequestAsyncOperation::HandleRequest(Common::AsyncOperationSPtr const & thisSPtr)
{
if (this->Request.Action != HealthManagerTcpMessage::ReportHealthAction)
{
TRACE_LEVEL_AND_TESTASSERT(
this->WriteInfo,
TraceComponent,
"{0}: ProcessReportRequestAsyncOperation received action {1}, not supported",
this->TraceId,
this->Request.Action);
TryComplete(thisSPtr, ErrorCode(ErrorCodeValue::InvalidMessage));
return;
}
ReportHealthMessageBody messageBody;
if (!this->Request.GetBody(messageBody))
{
TRACE_LEVEL_AND_TESTASSERT(this->WriteInfo, TraceComponent, "{0}: Error getting ReportHealthMessageBody", this->TraceId);
TryComplete(thisSPtr, ErrorCode(ErrorCodeValue::InvalidMessage));
return;
}
auto requests = messageBody.MoveEntries();
auto sequenceStreamsInfo = messageBody.MoveSequenceStreamInfo();
vector<SequenceStreamRequestContext> ssContexts;
map<SequenceStreamId, ErrorCodeValue::Enum> rejectedSequences;
for (auto it = sequenceStreamsInfo.begin(); it != sequenceStreamsInfo.end(); ++it)
{
SequenceStreamId id(it->Kind, it->SourceId);
ErrorCode error = this->EntityManager.AcceptSequenceStream(currentNestedActivityId_, id, it->Instance, it->UpToLsn);
if (error.IsSuccess())
{
auto pendingCount = it->ReportCount;
if (pendingCount == 0)
{
currentNestedActivityId_.IncrementIndex();
ssContexts.push_back(
SequenceStreamRequestContext(
Store::ReplicaActivityId(this->ReplicaActivityId.PartitionedReplicaId, currentNestedActivityId_),
thisSPtr,
*this,
move(*it)));
}
else
{
sequenceStreams_.push_back(SequenceStreamWithPendingCountPair(move(*it), pendingCount));
}
}
else
{
this->WriteInfo(
TraceComponent,
"{0}: Rejecting sequence stream {1}: {2}",
this->TraceId,
*it,
error);
rejectedSequences[id] = error.ReadValue();
}
}
// Create request contexts and pass them to the cache manager.
// If any of the requests are accepted, wait for them to be processed.
// The context will keep track of this async operation and notify it with the result of the processing.
vector<ReportRequestContext> contexts;
uint64 reportIndex = 0;
this->EntityManager.HealthManagerCounters->OnHealthReportsReceived(requests.size());
for (auto it = requests.begin(); it != requests.end(); ++it)
{
// Create a health report result which takes the primary key from the report.
// For node, it uses the node id generated on the client. For server processing,
// the report node id is re-computed to use the server configure generated data.
HealthReportResult result(*it);
if (it->SequenceNumber <= FABRIC_INVALID_SEQUENCE_NUMBER)
{
// If the new report has invalid LSN, reject immediately
HMEvents::Trace->DropReportInvalidSourceLSN(this->TraceId, *it);
result.Error = ErrorCodeValue::HealthStaleReport;
}
else
{
SequenceStreamId id(it->Kind, it->SourceId);
auto itReject = rejectedSequences.find(id);
if (itReject != rejectedSequences.end())
{
result.Error = itReject->second;
}
else
{
// Check whether there is a sequence stream associated with this context
FABRIC_SEQUENCE_NUMBER fromLsn = FABRIC_INVALID_SEQUENCE_NUMBER;
FABRIC_SEQUENCE_NUMBER upToLsn = FABRIC_INVALID_SEQUENCE_NUMBER;
for (auto itSS = sequenceStreams_.begin(); itSS != sequenceStreams_.end(); ++itSS)
{
if (itSS->first.IsHealthInformationRelevant(*it))
{
fromLsn = itSS->first.FromLsn;
upToLsn = itSS->first.UpToLsn;
break;
}
}
// Create a context that will be given to the cache entity manager
// for processing
currentNestedActivityId_.IncrementIndex();
if (HealthEntityKind::CanAccept(it->Kind))
{
contexts.push_back(ReportRequestContext(
Store::ReplicaActivityId(this->ReplicaActivityId.PartitionedReplicaId, currentNestedActivityId_),
thisSPtr,
*this,
reportIndex,
move(*it),
fromLsn,
upToLsn));
}
else
{
WriteInfo(TraceComponent, "{0}: Invalid context kind {1}", currentNestedActivityId_, it->Kind);
// Complete the report with error
result.Error = ErrorCodeValue::InvalidArgument;
}
}
}
// Remember the entry in the vector of results.
// The error code will be set after the request is processed.
// The correlation is done using the reportIndex.
// Add under the lock, as it may race with the timeout timer
// if a very small timeout is provided.
{ // lock
AcquireExclusiveLock lock(lock_);
reportResults_.push_back(move(result));
} // endlock
++reportIndex;
}
requestCount_ = static_cast<uint64>(contexts.size());
uint64 initialCount = requestCount_;
if (!ssContexts.empty())
{
// Increment number of requests that need to complete in order
// to consider the work for this async operation done.
requestCount_ += ssContexts.size();
// Remember request count before changed in callback by contexts being completed
initialCount = requestCount_;
}
// Start timer after everything is initialized, to prevent races when the timeout is really small
// and fires before the global variables are initialized.
TimedAsyncOperation::OnStart(thisSPtr);
for (auto && ssContext : ssContexts)
{
this->EntityManager.AddSequenceStreamContext(move(ssContext));
}
this->EntityManager.AddReports(move(contexts));
CheckIfAllRequestsProcessed(thisSPtr, initialCount);
}
void OneWayReceiverContext::Reject(ErrorCode const & error, ActivityId const & activityId)
{
UNREFERENCED_PARAMETER(activityId);
// TODO: If we reject a one way message we can potentially send some messages back about ring state
error.ReadValue();
}