void s_ThrowError(Int8 code, Int8 sub_code, const string& err_msg)
{
// Issues were reported by a NetStorage server v2.2.0 or later
if (sub_code != SIssue::kEmptySubCode) {
switch (code) {
case 3000:
code = sub_code; // CNetStorageServerError
break;
case 3010:
throw CNetServiceException(DIAG_COMPILE_INFO, 0,
static_cast<CNetServiceException::EErrCode>(sub_code),
err_msg);
case 3020:
throw CNetStorageException(DIAG_COMPILE_INFO, 0,
static_cast<CNetStorageException::EErrCode>(sub_code),
err_msg);
}
}
switch (code) {
case CNetStorageServerError::eNetStorageObjectNotFound:
case CNetStorageServerError::eRemoteObjectNotFound:
NCBI_THROW_FMT(CNetStorageException, eNotExists, err_msg);
break;
case CNetStorageServerError::eNetStorageObjectExpired:
NCBI_THROW_FMT(CNetStorageException, eExpired, err_msg);
break;
default:
NCBI_THROW_FMT(CNetStorageException, eServerError, err_msg);
}
}
void SNetStorage::SConfig::Validate(const string& init_string)
{
SNetStorage::SLimits::Check<SNetStorage::SLimits::SNamespace>(app_domain);
if (client_name.empty()) {
CNcbiApplication* app = CNcbiApplication::Instance();
if (app != NULL) {
string path;
CDirEntry::SplitPath(app->GetProgramExecutablePath(),
&path, &client_name);
if (NStr::EndsWith(path, CDirEntry::GetPathSeparator()))
path.erase(path.length() - 1);
string parent_dir;
CDirEntry::SplitPath(path, NULL, &parent_dir);
if (!parent_dir.empty()) {
client_name += '-';
client_name += parent_dir;
}
}
}
if (client_name.empty()) {
NCBI_THROW_FMT(CNetStorageException, eAuthError,
"Client name is required.");
}
switch (default_storage) {
case eUndefined:
default_storage =
!service.empty() ? eNetStorage :
!nc_service.empty() ? eNetCache :
eNoCreate;
break;
case eNetStorage:
if (service.empty()) {
NCBI_THROW_FMT(CNetStorageException, eInvalidArg,
init_string << ": 'nst=' parameter is required "
"when 'default_storage=nst'");
}
break;
case eNetCache:
if (nc_service.empty()) {
NCBI_THROW_FMT(CNetStorageException, eInvalidArg,
init_string << ": 'nc=' parameter is required "
"when 'default_storage=nc'");
}
break;
default: /* eNoCreate */
break;
}
}
void CNetScheduleJobSerializer::LoadJobInput(const string& source_file)
{
CNcbiIfstream input_stream(source_file.c_str(), CNcbiIfstream::binary);
if (input_stream.fail() && !input_stream.eof()) {
NCBI_THROW_FMT(CIOException, eRead,
"Error while reading job input from '" << source_file << '\'');
}
string header;
getline(input_stream, header);
CAttrListParser attr_parser;
attr_parser.Reset(header);
CAttrListParser::ENextAttributeType attr_type;
CTempString attr_name;
string attr_value;
size_t attr_column;
#define ATTR_POS " at column " << attr_column
while ((attr_type = attr_parser.NextAttribute(&attr_name,
&attr_value, &attr_column)) != CAttrListParser::eNoMoreAttributes) {
if (attr_name == "affinity")
m_Job.affinity = attr_value;
else if (attr_name == "group")
m_Job.group = attr_value;
else if (attr_name == "exclusive") {
m_Job.mask = CNetScheduleAPI::eExclusiveJob;
continue;
} else {
NCBI_THROW_FMT(CArgException, eInvalidArg,
"unknown attribute '" << attr_name << "'" ATTR_POS);
}
if (attr_type != CAttrListParser::eAttributeWithValue) {
NCBI_THROW_FMT(CArgException, eInvalidArg,
"attribute '" << attr_name << "' requires a value" ATTR_POS);
}
}
if (!input_stream.eof()) {
CStringOrBlobStorageWriter job_input_writer(
numeric_limits<size_t>().max(), NULL, m_Job.input);
CWStream job_input_ostream(&job_input_writer, 0, NULL);
NcbiStreamCopy(job_input_ostream, input_stream);
}
CDirEntry file_name(source_file);
m_Job.job_id = file_name.GetBase();
}
void SNetStorageObjectRPC::ReadConfirmation()
{
CSocket& sock = m_Connection->m_Socket;
CJsonOverUTTPReader json_reader;
try {
size_t bytes_read;
while (!json_reader.ReadMessage(m_UTTPReader)) {
s_ReadSocket(sock, m_ReadBuffer,
READ_BUFFER_SIZE, &bytes_read);
m_UTTPReader.SetNewBuffer(m_ReadBuffer, bytes_read);
}
if (m_UTTPReader.GetNextEvent() != CUTTPReader::eEndOfBuffer) {
NCBI_THROW_FMT(CNetStorageException, eIOError,
"Extra bytes past confirmation message "
"while reading " << m_Locator << " from " <<
sock.GetPeerAddress() << '.');
}
}
catch (...) {
m_UTTPReader.Reset();
m_Connection->Close();
throw;
}
s_TrapErrors(m_OriginalRequest, json_reader.GetMessage(), sock,
m_NetStorageRPC->m_Config.err_mode);
}
CJsonNode CReadJsonFromSocket::ReadMessage(CSocket& sock)
{
try {
char read_buffer[READ_BUFFER_SIZE];
size_t bytes_read;
do {
s_ReadSocket(sock, read_buffer, READ_BUFFER_SIZE, &bytes_read);
m_UTTPReader.SetNewBuffer(read_buffer, bytes_read);
} while (!m_JSONReader.ReadMessage(m_UTTPReader));
}
catch (...) {
sock.Close();
throw;
}
if (m_UTTPReader.GetNextEvent() != CUTTPReader::eEndOfBuffer) {
string server_address(sock.GetPeerAddress());
sock.Close();
NCBI_THROW_FMT(CNetStorageException, eIOError,
"Extra bytes past message end while reading from " <<
server_address << " after receiving " <<
m_JSONReader.GetMessage().Repr() << '.');
}
return m_JSONReader.GetMessage();
}
CNetStorageObject SNetStorageRPC::Create(TNetStorageFlags flags)
{
switch (m_Config.default_storage) {
/* TConfig::eUndefined is overridden in the constructor */
case TConfig::eNetStorage:
break; // This case is handled below the switch.
case TConfig::eNetCache:
x_InitNetCacheAPI();
return CDNCNetStorage::Create(m_NetCacheAPI);
default: /* TConfig::eNoCreate */
NCBI_THROW_FMT(CNetStorageException, eAuthError,
"Object creation is disabled.");
}
m_UseNextSubHitID.ProperCommand();
CJsonNode request(MkStdRequest("CREATE"));
x_SetStorageFlags(request, flags);
CNetServerConnection conn;
string object_loc = Exchange(m_Service,
request, &conn).GetString("ObjectLoc");
return new SNetStorageObjectRPC(this, request, conn,
SNetStorageObjectRPC::eByGeneratedID,
object_loc, flags, SNetStorageObjectRPC::eWriting);
}
void CSeqVector_CI::GetSeqData(string& buffer, TSeqPos count)
{
buffer.erase();
TSeqPos pos = GetPos();
_ASSERT(pos <= x_GetSize());
count = min(count, x_GetSize() - pos);
if ( !count ) {
return;
}
if ( m_TSE && !CanGetRange(pos, pos+count) ) {
NCBI_THROW_FMT(CSeqVectorException, eDataError,
"CSeqVector_CI::GetSeqData: "
"cannot get seq-data in range: "
<<pos<<"-"<<pos+count);
}
buffer.reserve(count);
while ( count ) {
TCache_I cache = m_Cache;
TCache_I cache_end = m_CacheEnd;
TSeqPos chunk_count = min(count, TSeqPos(cache_end - cache));
_ASSERT(chunk_count > 0);
TCache_I chunk_end = cache + chunk_count;
buffer.append(cache, chunk_end);
count -= chunk_count;
//if ( count == 0 ) break;
if ( chunk_end == cache_end ) {
x_NextCacheSeg();
}
else {
m_Cache = chunk_end;
}
}
}
void CSeqVector_CI::x_UpdateSeg(TSeqPos pos)
{
if ( m_Seg.IsInvalid() ) {
x_InitSeg(pos);
}
else if ( m_Seg.GetPosition() > pos ) {
// segment is ahead
do {
x_DecSeg();
} while ( m_Seg && m_Seg.GetLength() == 0 ); // skip 0 length segments
if ( !m_Seg || m_Seg.GetPosition() > pos ) {
// too far
x_InitSeg(pos);
}
}
else if ( m_Seg.GetEndPosition() <= pos ) {
// segment is behind
do {
x_IncSeg();
} while ( m_Seg && m_Seg.GetLength() == 0 ); // skip 0 length segments
if ( !m_Seg || m_Seg.GetEndPosition() <= pos ) {
// too far
x_InitSeg(pos);
}
}
if ( !m_Seg && pos == x_GetSize() ) {
// it's ok to position to the very end
return;
}
if ( !m_Seg || pos<m_Seg.GetPosition() || pos>=m_Seg.GetEndPosition() ) {
NCBI_THROW_FMT(CSeqVectorException, eDataError,
"CSeqVector_CI: cannot locate segment at "<<pos);
}
_ASSERT(m_Seg && pos>=m_Seg.GetPosition() && pos<m_Seg.GetEndPosition());
}
ERW_Result SNetStorageObjectRPC::Write(const void* buf_pos, size_t buf_size,
size_t* bytes_written)
{
if (m_State == eReading) {
NCBI_THROW_FMT(CNetStorageException, eInvalidArg,
"Cannot write a NetStorage file while reading");
}
if (m_State == eReady) {
m_NetStorageRPC->m_UseNextSubHitID.ProperCommand();
m_OriginalRequest = x_MkRequest("WRITE");
m_Locator = ExchangeUsingOwnService(m_OriginalRequest,
&m_Connection).GetString("ObjectLoc");
m_State = eWriting;
}
try {
s_SendUTTPChunk(reinterpret_cast<const char*>(buf_pos),
buf_size, m_Connection->m_Socket);
if (bytes_written != NULL)
*bytes_written = buf_size;
return eRW_Success;
}
catch (exception&) {
m_State = eReady;
m_Connection->Close();
m_Connection = NULL;
throw;
}
}
static void s_ResolveItems(CTypeInfo*& info, const char*& name,
ETypeFamily req_family)
{
TTypeInfo info0 = info;
const char* name0 = name;
while ( const char* dot = strchr(name, '.') ) {
CTempString item_name(name, dot-name);
TTypeInfo new_info;
switch ( info->GetTypeFamily() ) {
case eTypeFamilyClass:
new_info = dynamic_cast<CClassTypeInfo*>(info)
->GetMemberInfo(item_name)->GetTypeInfo();
break;
case eTypeFamilyChoice:
new_info = dynamic_cast<CChoiceTypeInfo*>(info)
->GetVariantInfo(item_name)->GetTypeInfo();
break;
case eTypeFamilyContainer:
if ( item_name != "E" ) {
NCBI_THROW_FMT(CSerialException,eInvalidData,
info0->GetName()<<'.'<<
CTempString(name0, name-name0)<<
": element name must be 'E'");
}
new_info = dynamic_cast<CContainerTypeInfo*>(info)
->GetElementType();
break;
default:
new_info = info;
break;
}
// skip all pointers (CRef<>)
while ( new_info->GetTypeFamily() == eTypeFamilyPointer ) {
new_info = dynamic_cast<const CPointerTypeInfo*>(new_info)
->GetPointedType();
}
info = const_cast<CTypeInfo*>(new_info);
name = dot+1;
}
if ( info->GetTypeFamily() != req_family ) {
NCBI_THROW_FMT(CSerialException,eInvalidData,
info0->GetName()<<'.'<<
CTempString(name0, name-name0)<<
": not a "<<
(req_family == eTypeFamilyClass? "class": "choice"));
}
}
const CGBRequestStatistics& CGBRequestStatistics::GetStatistics(EStatType type)
{
if ( type < eStat_First || type > eStat_Last ) {
NCBI_THROW_FMT(CLoaderException, eOtherError,
"CGBRequestStatistics::GetStatistics: "
"invalid statistics type: "<<type);
}
return sx_Statistics[type];
}
unsigned g_NetService_gethostbyname(const string& hostname)
{
unsigned ip = CSocketAPI::gethostbyname(hostname, eOn);
if (ip == 0) {
NCBI_THROW_FMT(CNetServiceException, eCommunicationError,
"gethostbyname('" << hostname << "') failed");
}
return ip;
}
SNetStorageByKeyRPC::SNetStorageByKeyRPC(const TConfig& config,
TNetStorageFlags default_flags) :
m_NetStorageRPC(new SNetStorageRPC(config, default_flags))
{
if (m_NetStorageRPC->m_Config.app_domain.empty()) {
NCBI_THROW_FMT(CNetStorageException, eInvalidArg,
"'domain' parameter is missing from the initialization string");
}
}
void CSeqTableLocColumns::SetColumn(CSeqTableColumnInfo& field,
const CSeqTable_column& column)
{
if ( field ) {
NCBI_THROW_FMT(CAnnotException, eBadLocation,
"Duplicate "<<m_FieldName<<" column");
}
field = CSeqTableColumnInfo(column);
m_Is_set = true;
}
void SJsonArrayNodeImpl::VerifyIndexBounds(
const char* operation, size_t index) const
{
if (m_Array.size() <= index) {
NCBI_THROW_FMT(CJsonException, eIndexOutOfRange,
operation << ": index " << index <<
" is out of range (array size is " <<
m_Array.size() << ')');
}
}
void SJsonNodeImpl::VerifyType(const char* operation,
CJsonNode::ENodeType required_type) const
{
if (m_NodeType != required_type) {
NCBI_THROW_FMT(CJsonException, eInvalidNodeType,
"Cannot call the " << operation <<
" method for " << GetTypeName() << " node; " <<
GetTypeName(required_type) << " node is required");
}
}
string g_NetService_gethostname(const string& ip_or_hostname)
{
string hostname(CSocketAPI::gethostbyaddr(
g_NetService_gethostbyname(ip_or_hostname), eOn));
if (hostname.empty()) {
NCBI_THROW_FMT(CNetServiceException, eCommunicationError,
"g_NetService_gethostname('" << ip_or_hostname << "') failed");
}
return hostname;
}
Uint8 SNetStorageObjectRPC::GetSize()
{
if (m_State != eReady) {
NCBI_THROW_FMT(CNetStorageException, eInvalidArg,
"Cannot get object size while reading or writing");
}
CJsonNode request(x_MkRequest("GETSIZE"));
return (Uint8) ExchangeUsingOwnService(request).GetInteger("Size");
}
CNetStorageObjectInfo SNetStorageObjectRPC::GetInfo()
{
if (m_State != eReady) {
NCBI_THROW_FMT(CNetStorageException, eInvalidArg,
"Cannot get object info while reading or writing");
}
CJsonNode request(x_MkRequest("GETOBJECTINFO"));
return g_CreateNetStorageObjectInfo(
ExchangeUsingOwnService(request));
}
string SNetStorageObjectRPC::GetAttribute(const string& attr_name) const
{
if (m_State != eReady) {
NCBI_THROW_FMT(CNetStorageException, eInvalidArg,
"Cannot get object attribute while reading or writing");
}
CJsonNode request(x_MkRequest("GETATTR"));
request.SetString("AttrName", attr_name);
return ExchangeUsingOwnService(request).GetString("AttrValue");
}
void CSeqVector::GetPackedSeqData(string& dst_str,
TSeqPos src_pos,
TSeqPos src_end)
{
dst_str.erase();
src_end = min(src_end, size());
if ( src_pos >= src_end ) {
return;
}
if ( m_TSE && !CanGetRange(src_pos, src_end) ) {
NCBI_THROW_FMT(CSeqVectorException, eDataError,
"CSeqVector::GetPackedSeqData: "
"cannot get seq-data in range: "
<<src_pos<<"-"<<src_end);
}
TCoding dst_coding = GetCoding();
switch ( dst_coding ) {
case CSeq_data::e_Iupacna:
case CSeq_data::e_Ncbi8na:
case CSeq_data::e_Iupacaa:
case CSeq_data::e_Ncbieaa:
case CSeq_data::e_Ncbi8aa:
case CSeq_data::e_Ncbistdaa:
x_GetPacked8SeqData(dst_str, src_pos, src_end);
break;
case CSeq_data::e_Ncbi4na:
x_GetPacked4naSeqData(dst_str, src_pos, src_end);
break;
case CSeq_data::e_Ncbi2na:
x_GetPacked2naSeqData(dst_str, src_pos, src_end);
break;
default:
NCBI_THROW_FMT(CSeqVectorException, eCodingError,
"Can not pack data using the selected coding: "<<
GetCoding());
}
}
void g_VerifyAlphabet(const string& str, const CTempString& param_name,
ECharacterClass char_class)
{
char bad_char;
if (!g_CheckAlphabet(str, param_name, char_class, &bad_char)) {
NCBI_THROW_FMT(CConfigException, eParameterMissing,
"Invalid character '" <<
NStr::PrintableString(CTempString(&bad_char, 1)) <<
"' in the " << param_name << " \"" <<
NStr::PrintableString(str) << "\".");
}
}
bool SNetStorageObjectRPC::Eof()
{
switch (m_State) {
case eReady:
return false;
case eReading:
return m_CurrentChunkSize == 0 && m_EOF;
default: /* case eWriting: */
NCBI_THROW_FMT(CNetStorageException, eInvalidArg,
"Cannot read a NetStorage file while writing");
}
}
BEGIN_NCBI_SCOPE
#define NS_KEY_V1_PREFIX "JSID_01_"
#define NS_KEY_V1_PREFIX_LEN (sizeof(NS_KEY_V1_PREFIX) - 1)
CNetScheduleKey::CNetScheduleKey(const string& key_str,
CCompoundIDPool::TInstance id_pool)
{
if (!ParseJobKey(key_str, id_pool)) {
NCBI_THROW_FMT(CNetScheduleException, eKeyFormatError,
"Invalid job key format: '" <<
NStr::PrintableString(key_str) << '\'');
}
}
SNetStorage::SConfig::EDefaultStorage
SNetStorage::SConfig::GetDefaultStorage(const string& value)
{
if (NStr::CompareNocase(value, "nst") == 0)
return eNetStorage;
else if (NStr::CompareNocase(value, "nc") == 0)
return eNetCache;
else if (NStr::CompareNocase(value, "nocreate") == 0 ||
NStr::CompareNocase(value, "no_create") == 0)
return eNoCreate;
else {
NCBI_THROW_FMT(CNetStorageException, eInvalidArg,
"Invalid default_storage value '" << value << '\'');
return eUndefined;
}
}
void SNetStorageObjectRPC::SetExpiration(const CTimeout& ttl)
{
if (m_State != eReady) {
NCBI_THROW_FMT(CNetStorageException, eInvalidArg,
"Cannot set expiration while reading or writing");
}
CJsonNode request(x_MkRequest("SETEXPTIME"));
if (ttl.IsFinite()) {
request.SetString("TTL", ttl.GetAsTimeSpan().AsString("dTh:m:s"));
} else {
request.SetString("TTL", "infinity");
}
ExchangeUsingOwnService(request);
}
static void s_ReadSocket(CSocket& sock, void* buffer,
size_t buffer_size, size_t* bytes_read)
{
EIO_Status status;
while ((status = sock.Read(buffer,
buffer_size, bytes_read)) == eIO_Interrupt)
/* no-op */;
if (status != eIO_Success) {
// eIO_Timeout, eIO_Closed, eIO_InvalidArg,
// eIO_NotSupported, eIO_Unknown
NCBI_THROW_FMT(CNetStorageException, eIOError,
"I/O error while reading from NetStorage server " <<
sock.GetPeerAddress() << ". "
"Socket status: " << IO_StatusStr(status) << '.');
}
}
string SNetStorageRPC::Relocate(const string& object_loc,
TNetStorageFlags flags)
{
if (x_NetCacheMode(object_loc))
NCBI_THROW_FMT(CNetStorageException, eNotSupported, object_loc <<
": Relocate for NetCache blobs is not implemented");
m_UseNextSubHitID.ProperCommand();
CJsonNode request(MkObjectRequest("RELOCATE", object_loc));
CJsonNode new_location(CJsonNode::NewObjectNode());
x_SetStorageFlags(new_location, flags);
request.SetByKey("NewLocation", new_location);
return Exchange(GetServiceFromLocator(object_loc),
request).GetString("ObjectLoc");
}
void CSeqVector_CI::x_NextCacheSeg()
{
_ASSERT(m_SeqMap);
TSeqPos pos = x_CacheEndPos();
TSeqPos size = x_GetSize();
if ( pos >= size ) {
if ( x_CachePos() < pos ) {
x_SwapCache();
x_ResetCache();
m_CachePos = pos;
return;
}
else {
// Can not go further
NCBI_THROW(CSeqVectorException, eOutOfRange,
"Can not update cache: iterator beyond end");
}
}
// save current cache in backup
_ASSERT(x_CacheSize());
x_SwapCache();
// update segment if needed
x_UpdateSeg(pos);
if ( !m_Seg ) {
// end of sequence
NCBI_THROW_FMT(CSeqVectorException, eDataError,
"CSeqVector_CI: invalid sequence length: "
<<pos<<" <> "<<size);
}
// Try to re-use backup cache
if ( pos < x_CacheEndPos() && pos >= x_CachePos() ) {
m_Cache = m_CacheData.get() + pos - x_CachePos();
}
else {
// can not use backup cache
x_ResetCache();
x_UpdateCacheUp(pos);
_ASSERT(GetPos() == pos);
_ASSERT(x_CacheSize());
_ASSERT(x_CachePos() == pos);
}
}
void CVDBCursor::ReadElements(TVDBRowId row, const CVDBColumn& column,
uint32_t elem_bits,
uint32_t start, uint32_t count,
void* buffer) const
{
DECLARE_SDK_GET_GUARD();
uint32_t read_count, remaining_count;
if ( rc_t rc = VCursorReadBitsDirect(*this, row, column.GetIndex(),
elem_bits, start, buffer, 0, count,
&read_count, &remaining_count) ) {
NCBI_THROW2_FMT(CSraException, eNotFoundValue,
"Cannot read VDB value array: "<<*this<<column<<
'['<<row<<"]["<<start<<".."<<(start+count-1)<<']', rc);
}
if ( read_count != count ) {
NCBI_THROW_FMT(CSraException, eNotFoundValue,
"Cannot read VDB value array: "<<*this<<column<<
'['<<row<<"]["<<start<<".."<<(start+count-1)<<
"] only "<<read_count<<" elements are read");
}
}
