void Definition::toString(char *buf, int max, int indent) const {
int index=0;
OID temp;
int modi;
char buf2[50];
Event *evt2 = NEW Event(Event::GET_RANGE, m_def);
evt2->param<0>(0);
evt2->param<1>(m_size);
evt2->send();
OID boid = evt2->result();
OID ifobj = Null;
Buffer *def = Buffer::lookup(boid);
delete evt2;
if (def == 0) {
buf[0] = 0;
return;
}
for (int i=0; i<indent; i++) strcat(buf, "\t");
while (index < m_size) {
temp = def->get(index++);
if (temp.isModifier()) {
modi = temp.d();
switch (modi) {
case modifiers::ENDSUB:
strcat(buf, ")");
break;
case modifiers::BEGINSUB:
strcat(buf, "(");
break;
case modifiers::UNION:
strcat(buf, "union ");
break;
case modifiers::COMPARE:
strcat(buf, "== ");
break;
default: break;
}
} else {
if (temp == This) {
strcat(buf, ".");
} else if (index == 1 && temp.get("type") == OID("if")) {
ifobj = temp;
strcat(buf, "if ");
//buildif(buf, temp);
} else {
temp.toString(buf2, 50);
strcat(buf, buf2);
strcat(buf, " ");
}
}
}
if (ifobj != Null) buildif(buf, ifobj, indent);
Buffer::free(boid);
}
INT32 insertData ( sdbCollection *collection, CHAR **json_str )
{
INT32 rc = SDB_OK ;
OID oid ;
INT32 len = 0 ;
CHAR *temp = NULL ;
BSONObj bs ;
rc = fromjson ( *json_str, bs ) ;
if ( rc )
{
return rc ;
}
rc = collection->insert ( bs, &oid ) ;
if ( rc )
{
return rc ;
}
std::string str = oid.toString () ;
const CHAR *p = str.c_str () ;
INT32 eleSize = strlen(p) ;
if ( eleSize > 0 )
{
CHAR *pTemp = (CHAR*)malloc(eleSize+1) ;
if ( !pTemp )
return SDB_OOM ;
memcpy ( pTemp, p, eleSize ) ;
pTemp[eleSize] = '\0' ;
*json_str = pTemp ;
}
return rc ;
}
void
PublicKey::decode(CryptoPP::BufferedTransformation& in)
{
// SubjectPublicKeyInfo ::= SEQUENCE {
// algorithm AlgorithmIdentifier
// keybits BIT STRING }
using namespace CryptoPP;
try
{
std::string out;
StringSink sink(out);
////////////////////////
// part 1: copy as is //
////////////////////////
BERSequenceDecoder decoder(in);
{
assert(decoder.IsDefiniteLength());
DERSequenceEncoder encoder(sink);
decoder.TransferTo(encoder, decoder.RemainingLength());
encoder.MessageEnd();
}
decoder.MessageEnd();
////////////////////////
// part 2: check if the key is RSA (since it is the only supported for now)
////////////////////////
StringSource checkedSource(out, true);
BERSequenceDecoder subjectPublicKeyInfo(checkedSource);
{
BERSequenceDecoder algorithmInfo(subjectPublicKeyInfo);
{
OID algorithm;
algorithm.decode(algorithmInfo);
if (algorithm == oid::RSA)
m_type = KEY_TYPE_RSA;
else if (algorithm == oid::ECDSA)
m_type = KEY_TYPE_ECDSA;
else
BOOST_THROW_EXCEPTION(Error("Only RSA/ECDSA public keys are supported for now (" +
algorithm.toString() + " requested)"));
}
}
m_key.assign(out.begin(), out.end());
}
catch (CryptoPP::BERDecodeErr& err)
{
m_type = KEY_TYPE_NULL;
BOOST_THROW_EXCEPTION(Error("PublicKey decoding error"));
}
m_digest.reset();
}
DString cadence::operator+(DString m, const OID &o) {
char buf[200];
o.toString(buf,200);
int s = m.size();
int s2 = strlen(buf);
for (int i=0; i<s2; i++) {
m.m_obj[s+i].set(buf[i], true);
}
m.m_obj[Size].set(s+s2);
return m;
}
string SavePaymentJson(int amount)
{
//BSONObj paymentBSON = mongo::fromjson(newPyamentJson);
BSONObj paymentBSON = BSON(GENOID
<< "PayedToUserId" << 8888
<< "PayedDate" << "2015-01-25 12:00:00"
<< "PayedPeriodStartDate" << "2015-01-01 00:00:00"
<< "PayedPeriodEndDate" << "2015-01-29 23:59:59"
<< "Amount" << amount);
db.insert(PAYMENTS_COLLECTION_NAMESPASE, paymentBSON);
BSONElement oi;
paymentBSON.getObjectID(oi);
OID oid = oi.__oid();
return oid.toString();
}
void DString::toString(char *str, int max) {
//The whole object may be an integer etc...
if (m_obj[Size] == Null || !((OID)m_obj[0]).isChar()) {
m_obj.toString(str,max);
return;
}
int size = (OID)m_obj[Size];
if (size >= max) size = max-1;
int j = 0;
OID v;
Event *evt = new Event(Event::GET_RANGE, m_obj);
evt->param<0>(0);
evt->param<1>(size);
evt->send();
OID boid = evt->result();
Buffer *str2 = Buffer::lookup(boid);
delete evt;
if (str2 == 0) {
str[0] = 0;
return;
}
for (int i=0; i<size; i++) {
v = str2->get(i);
if (v.isChar())
str[j++] = str2->get(i);
else {
int k = 0;
char buf[500];
v.toString(buf,500);
while (buf[k] != 0) {
str[j++] = buf[k++];
}
}
}
Buffer::free(boid);
str[j] = 0;
};
/* ****************************************************************************
*
* processRegisterContext -
*
* This function has a slightly different behaviour depending on whether the id
* parameter is null (new registration case) or not null (update case), in
* particular:
*
* - In the new registration case, the _id is generated and insert() is used to
* put the document in the DB.
* - In the update case, the _id is set according to the argument 'id' and update() is
* used to put the document in the DB.
*
*/
HttpStatusCode processRegisterContext
(
RegisterContextRequest* requestP,
RegisterContextResponse* responseP,
OID* id,
const std::string& tenant,
const std::string& servicePath,
const std::string& format,
const std::string& fiwareCorrelator
)
{
std::string err;
/* If expiration is not present, then use a default one */
if (requestP->duration.isEmpty())
{
requestP->duration.set(DEFAULT_DURATION);
}
/* Calculate expiration (using the current time and the duration field in the request) */
long long expiration = getCurrentTime() + requestP->duration.parse();
LM_T(LmtMongo, ("Registration expiration: %lu", expiration));
/* Create the mongoDB registration document */
BSONObjBuilder reg;
OID oid;
if (id == NULL)
{
oid.init();
}
else
{
oid = *id;
}
reg.append("_id", oid);
reg.append(REG_EXPIRATION, expiration);
reg.append(REG_SERVICE_PATH, servicePath == "" ? DEFAULT_SERVICE_PATH_UPDATES : servicePath);
reg.append(REG_FORMAT, format);
//
// We accumulate the subscriptions in a map. The key of the map is the string representing
// subscription id
//
std::map<string, TriggeredSubscription*> subsToNotify;
// This vector is used to define which entities to include in notifications
EntityIdVector triggerEntitiesV;
BSONArrayBuilder contextRegistration;
for (unsigned int ix = 0; ix < requestP->contextRegistrationVector.size(); ++ix)
{
ContextRegistration* cr = requestP->contextRegistrationVector[ix];
BSONArrayBuilder entities;
for (unsigned int jx = 0; jx < cr->entityIdVector.size(); ++jx)
{
EntityId* en = cr->entityIdVector[jx];
triggerEntitiesV.push_back(en);
if (en->type == "")
{
entities.append(BSON(REG_ENTITY_ID << en->id));
LM_T(LmtMongo, ("Entity registration: {id: %s}", en->id.c_str()));
}
else
{
entities.append(BSON(REG_ENTITY_ID << en->id << REG_ENTITY_TYPE << en->type));
LM_T(LmtMongo, ("Entity registration: {id: %s, type: %s}", en->id.c_str(), en->type.c_str()));
}
}
BSONArrayBuilder attrs;
for (unsigned int jx = 0; jx < cr->contextRegistrationAttributeVector.size(); ++jx)
{
ContextRegistrationAttribute* cra = cr->contextRegistrationAttributeVector[jx];
attrs.append(BSON(REG_ATTRS_NAME << cra->name << REG_ATTRS_TYPE << cra->type << "isDomain" << cra->isDomain));
LM_T(LmtMongo, ("Attribute registration: {name: %s, type: %s, isDomain: %s}",
cra->name.c_str(),
cra->type.c_str(),
cra->isDomain.c_str()));
for (unsigned int kx = 0;
kx < requestP->contextRegistrationVector[ix]->contextRegistrationAttributeVector[jx]->metadataVector.size();
++kx)
{
// FIXME: metadata not supported at the moment
}
}
contextRegistration.append(
BSON(
REG_ENTITIES << entities.arr() <<
REG_ATTRS << attrs.arr() <<
REG_PROVIDING_APPLICATION << requestP->contextRegistrationVector[ix]->providingApplication.get()));
LM_T(LmtMongo, ("providingApplication registration: %s",
requestP->contextRegistrationVector[ix]->providingApplication.c_str()));
std::string err;
if (!addTriggeredSubscriptions(*cr, subsToNotify, err, tenant))
{
responseP->errorCode.fill(SccReceiverInternalError, err);
return SccOk;
}
}
reg.append(REG_CONTEXT_REGISTRATION, contextRegistration.arr());
/* Note we are using upsert = "true". This means that if the document doesn't previously
* exist in the collection, it is created. Thus, this way both uses of registerContext are OK
* (either new registration or updating an existing one) */
if (!collectionUpdate(getRegistrationsCollectionName(tenant), BSON("_id" << oid), reg.obj(), true, &err))
{
responseP->errorCode.fill(SccReceiverInternalError, err);
releaseTriggeredSubscriptions(subsToNotify);
return SccOk;
}
//
// Send notifications for each one of the subscriptions accumulated by
// previous addTriggeredSubscriptions() invocations
//
processSubscriptions(triggerEntitiesV, subsToNotify, err, tenant, fiwareCorrelator);
// Fill the response element
responseP->duration = requestP->duration;
responseP->registrationId.set(oid.toString());
responseP->errorCode.fill(SccOk);
return SccOk;
}
bool mergeChunks(OperationContext* txn,
const NamespaceString& nss,
const BSONObj& minKey,
const BSONObj& maxKey,
const OID& epoch,
string* errMsg) {
// Get the distributed lock
string whyMessage = stream() << "merging chunks in " << nss.ns() << " from " << minKey << " to "
<< maxKey;
auto scopedDistLock = grid.catalogManager(txn)->distLock(
txn, nss.ns(), whyMessage, DistLockManager::kSingleLockAttemptTimeout);
if (!scopedDistLock.isOK()) {
*errMsg = stream() << "could not acquire collection lock for " << nss.ns()
<< " to merge chunks in [" << minKey << "," << maxKey << ")"
<< causedBy(scopedDistLock.getStatus());
warning() << *errMsg;
return false;
}
ShardingState* shardingState = ShardingState::get(txn);
//
// We now have the collection lock, refresh metadata to latest version and sanity check
//
ChunkVersion shardVersion;
Status status = shardingState->refreshMetadataNow(txn, nss.ns(), &shardVersion);
if (!status.isOK()) {
*errMsg = str::stream() << "could not merge chunks, failed to refresh metadata for "
<< nss.ns() << causedBy(status.reason());
warning() << *errMsg;
return false;
}
if (epoch.isSet() && shardVersion.epoch() != epoch) {
*errMsg = stream() << "could not merge chunks, collection " << nss.ns() << " has changed"
<< " since merge was sent"
<< "(sent epoch : " << epoch.toString()
<< ", current epoch : " << shardVersion.epoch().toString() << ")";
warning() << *errMsg;
return false;
}
shared_ptr<CollectionMetadata> metadata = shardingState->getCollectionMetadata(nss.ns());
if (!metadata || metadata->getKeyPattern().isEmpty()) {
*errMsg = stream() << "could not merge chunks, collection " << nss.ns()
<< " is not sharded";
warning() << *errMsg;
return false;
}
dassert(metadata->getShardVersion().equals(shardVersion));
if (!metadata->isValidKey(minKey) || !metadata->isValidKey(maxKey)) {
*errMsg = stream() << "could not merge chunks, the range " << rangeToString(minKey, maxKey)
<< " is not valid"
<< " for collection " << nss.ns() << " with key pattern "
<< metadata->getKeyPattern();
warning() << *errMsg;
return false;
}
//
// Get merged chunk information
//
ChunkVersion mergeVersion = metadata->getCollVersion();
mergeVersion.incMinor();
std::vector<ChunkType> chunksToMerge;
ChunkType itChunk;
itChunk.setMin(minKey);
itChunk.setMax(minKey);
itChunk.setNS(nss.ns());
itChunk.setShard(shardingState->getShardName());
while (itChunk.getMax().woCompare(maxKey) < 0 &&
metadata->getNextChunk(itChunk.getMax(), &itChunk)) {
chunksToMerge.push_back(itChunk);
}
if (chunksToMerge.empty()) {
*errMsg = stream() << "could not merge chunks, collection " << nss.ns()
<< " range starting at " << minKey << " and ending at " << maxKey
<< " does not belong to shard " << shardingState->getShardName();
warning() << *errMsg;
return false;
}
//
// Validate the range starts and ends at chunks and has no holes, error if not valid
//
BSONObj firstDocMin = chunksToMerge.front().getMin();
BSONObj firstDocMax = chunksToMerge.front().getMax();
// minKey is inclusive
bool minKeyInRange = rangeContains(firstDocMin, firstDocMax, minKey);
if (!minKeyInRange) {
*errMsg = stream() << "could not merge chunks, collection " << nss.ns()
<< " range starting at " << minKey << " does not belong to shard "
<< shardingState->getShardName();
warning() << *errMsg;
return false;
}
BSONObj lastDocMin = chunksToMerge.back().getMin();
BSONObj lastDocMax = chunksToMerge.back().getMax();
// maxKey is exclusive
bool maxKeyInRange = lastDocMin.woCompare(maxKey) < 0 && lastDocMax.woCompare(maxKey) >= 0;
if (!maxKeyInRange) {
*errMsg = stream() << "could not merge chunks, collection " << nss.ns()
<< " range ending at " << maxKey << " does not belong to shard "
<< shardingState->getShardName();
warning() << *errMsg;
return false;
}
bool validRangeStartKey = firstDocMin.woCompare(minKey) == 0;
bool validRangeEndKey = lastDocMax.woCompare(maxKey) == 0;
if (!validRangeStartKey || !validRangeEndKey) {
*errMsg = stream() << "could not merge chunks, collection " << nss.ns()
<< " does not contain a chunk "
<< (!validRangeStartKey ? "starting at " + minKey.toString() : "")
<< (!validRangeStartKey && !validRangeEndKey ? " or " : "")
<< (!validRangeEndKey ? "ending at " + maxKey.toString() : "");
warning() << *errMsg;
return false;
}
if (chunksToMerge.size() == 1) {
*errMsg = stream() << "could not merge chunks, collection " << nss.ns()
<< " already contains chunk for " << rangeToString(minKey, maxKey);
warning() << *errMsg;
return false;
}
// Look for hole in range
for (size_t i = 1; i < chunksToMerge.size(); ++i) {
if (chunksToMerge[i - 1].getMax().woCompare(chunksToMerge[i].getMin()) != 0) {
*errMsg =
stream() << "could not merge chunks, collection " << nss.ns()
<< " has a hole in the range " << rangeToString(minKey, maxKey) << " at "
<< rangeToString(chunksToMerge[i - 1].getMax(), chunksToMerge[i].getMin());
warning() << *errMsg;
return false;
}
}
//
// Run apply ops command
//
Status applyOpsStatus = runApplyOpsCmd(txn, chunksToMerge, shardVersion, mergeVersion);
if (!applyOpsStatus.isOK()) {
warning() << applyOpsStatus;
return false;
}
//
// Install merged chunk metadata
//
{
ScopedTransaction transaction(txn, MODE_IX);
Lock::DBLock writeLk(txn->lockState(), nss.db(), MODE_IX);
Lock::CollectionLock collLock(txn->lockState(), nss.ns(), MODE_X);
shardingState->mergeChunks(txn, nss.ns(), minKey, maxKey, mergeVersion);
}
//
// Log change
//
BSONObj mergeLogEntry = buildMergeLogEntry(chunksToMerge, shardVersion, mergeVersion);
grid.catalogManager(txn)->logChange(txn, "merge", nss.ns(), mergeLogEntry);
return true;
}
Status MetadataLoader::initChunks(CatalogManager* catalogManager,
const string& ns,
const string& shard,
const CollectionMetadata* oldMetadata,
CollectionMetadata* metadata) const
{
map<string, ChunkVersion> versionMap;
// Preserve the epoch
versionMap[shard] = metadata->_shardVersion;
OID epoch = metadata->getCollVersion().epoch();
bool fullReload = true;
// Check to see if we should use the old version or not.
if ( oldMetadata ) {
// If our epochs are compatible, it's useful to use the old metadata for diffs
if ( oldMetadata->getCollVersion().hasEqualEpoch( epoch ) ) {
fullReload = false;
invariant( oldMetadata->isValid() );
versionMap[shard] = oldMetadata->_shardVersion;
metadata->_collVersion = oldMetadata->_collVersion;
// TODO: This could be made more efficient if copying not required, but
// not as frequently reloaded as in mongos.
metadata->_chunksMap = oldMetadata->_chunksMap;
LOG( 2 ) << "loading new chunks for collection " << ns
<< " using old metadata w/ version " << oldMetadata->getShardVersion()
<< " and " << metadata->_chunksMap.size() << " chunks" << endl;
}
else {
warning() << "reloading collection metadata for " << ns << " with new epoch "
<< epoch.toString() << ", the current epoch is "
<< oldMetadata->getCollVersion().epoch().toString() << endl;
}
}
// Exposes the new metadata's range map and version to the "differ," who
// would ultimately be responsible of filling them up.
SCMConfigDiffTracker differ( shard );
differ.attach( ns, metadata->_chunksMap, metadata->_collVersion, versionMap );
try {
std::vector<ChunkType> chunks;
Status status = catalogManager->getChunks(differ.configDiffQuery(), 0, &chunks);
if (!status.isOK()) {
if (status == ErrorCodes::HostUnreachable) {
// Make our metadata invalid
metadata->_collVersion = ChunkVersion( 0, 0, OID() );
metadata->_chunksMap.clear();
}
return status;
}
//
// The diff tracker should always find at least one chunk (the highest chunk we saw
// last time). If not, something has changed on the config server (potentially between
// when we read the collection data and when we read the chunks data).
//
int diffsApplied = differ.calculateConfigDiff(chunks);
if ( diffsApplied > 0 ) {
// Chunks found, return ok
LOG(2) << "loaded " << diffsApplied << " chunks into new metadata for " << ns
<< " with version " << metadata->_collVersion;
metadata->_shardVersion = versionMap[shard];
metadata->fillRanges();
invariant( metadata->isValid() );
return Status::OK();
}
else if ( diffsApplied == 0 ) {
// No chunks found, the collection is dropping or we're confused
// If this is a full reload, assume it is a drop for backwards compatibility
// TODO: drop the config.collections entry *before* the chunks and eliminate this
// ambiguity
string errMsg =
str::stream() << "no chunks found when reloading " << ns
<< ", previous version was "
<< metadata->_collVersion.toString()
<< ( fullReload ? ", this is a drop" : "" );
warning() << errMsg << endl;
metadata->_collVersion = ChunkVersion( 0, 0, OID() );
metadata->_chunksMap.clear();
return fullReload ? Status( ErrorCodes::NamespaceNotFound, errMsg ) :
Status( ErrorCodes::RemoteChangeDetected, errMsg );
}
else {
// Invalid chunks found, our epoch may have changed because we dropped/recreated
// the collection.
string errMsg = str::stream() << "invalid chunks found when reloading " << ns
<< ", previous version was "
<< metadata->_collVersion.toString()
<< ", this should be rare";
warning() << errMsg;
metadata->_collVersion = ChunkVersion( 0, 0, OID() );
metadata->_chunksMap.clear();
return Status(ErrorCodes::RemoteChangeDetected, errMsg);
}
}
catch (const DBException& e) {
// We deliberately do not return connPtr to the pool, since it was involved with the
// error here.
return Status(ErrorCodes::HostUnreachable,
str::stream() << "problem querying chunks metadata" << causedBy(e));
}
}
bool mergeChunks( OperationContext* txn,
const NamespaceString& nss,
const BSONObj& minKey,
const BSONObj& maxKey,
const OID& epoch,
string* errMsg ) {
//
// Get sharding state up-to-date
//
ConnectionString configLoc = ConnectionString::parse( shardingState.getConfigServer(),
*errMsg );
if ( !configLoc.isValid() ){
warning() << *errMsg << endl;
return false;
}
//
// Get the distributed lock
//
ScopedDistributedLock collLock( configLoc, nss.ns() );
collLock.setLockMessage( stream() << "merging chunks in " << nss.ns() << " from "
<< minKey << " to " << maxKey );
Status acquisitionStatus = collLock.tryAcquire();
if (!acquisitionStatus.isOK()) {
*errMsg = stream() << "could not acquire collection lock for " << nss.ns()
<< " to merge chunks in [" << minKey << "," << maxKey << ")"
<< causedBy(acquisitionStatus);
warning() << *errMsg << endl;
return false;
}
//
// We now have the collection lock, refresh metadata to latest version and sanity check
//
ChunkVersion shardVersion;
Status status = shardingState.refreshMetadataNow(txn, nss.ns(), &shardVersion);
if ( !status.isOK() ) {
*errMsg = str::stream() << "could not merge chunks, failed to refresh metadata for "
<< nss.ns() << causedBy( status.reason() );
warning() << *errMsg << endl;
return false;
}
if ( epoch.isSet() && shardVersion.epoch() != epoch ) {
*errMsg = stream() << "could not merge chunks, collection " << nss.ns()
<< " has changed" << " since merge was sent" << "(sent epoch : "
<< epoch.toString()
<< ", current epoch : " << shardVersion.epoch().toString() << ")";
warning() << *errMsg << endl;
return false;
}
CollectionMetadataPtr metadata = shardingState.getCollectionMetadata( nss.ns() );
if ( !metadata || metadata->getKeyPattern().isEmpty() ) {
*errMsg = stream() << "could not merge chunks, collection " << nss.ns()
<< " is not sharded";
warning() << *errMsg << endl;
return false;
}
dassert( metadata->getShardVersion().equals( shardVersion ) );
if ( !metadata->isValidKey( minKey ) || !metadata->isValidKey( maxKey ) ) {
*errMsg = stream() << "could not merge chunks, the range "
<< rangeToString( minKey, maxKey ) << " is not valid"
<< " for collection " << nss.ns() << " with key pattern "
<< metadata->getKeyPattern();
warning() << *errMsg << endl;
return false;
}
//
// Get merged chunk information
//
ChunkVersion mergeVersion = metadata->getCollVersion();
mergeVersion.incMinor();
OwnedPointerVector<ChunkType> chunksToMerge;
ChunkType itChunk;
itChunk.setMin( minKey );
itChunk.setMax( minKey );
itChunk.setNS( nss.ns() );
itChunk.setShard( shardingState.getShardName() );
while ( itChunk.getMax().woCompare( maxKey ) < 0 &&
metadata->getNextChunk( itChunk.getMax(), &itChunk ) ) {
auto_ptr<ChunkType> saved( new ChunkType );
itChunk.cloneTo( saved.get() );
chunksToMerge.mutableVector().push_back( saved.release() );
}
if ( chunksToMerge.empty() ) {
*errMsg = stream() << "could not merge chunks, collection " << nss.ns()
<< " range starting at " << minKey
<< " and ending at " << maxKey
<< " does not belong to shard " << shardingState.getShardName();
warning() << *errMsg << endl;
return false;
}
//
// Validate the range starts and ends at chunks and has no holes, error if not valid
//
BSONObj firstDocMin = ( *chunksToMerge.begin() )->getMin();
BSONObj firstDocMax = ( *chunksToMerge.begin() )->getMax();
// minKey is inclusive
bool minKeyInRange = rangeContains( firstDocMin, firstDocMax, minKey );
if ( !minKeyInRange ) {
*errMsg = stream() << "could not merge chunks, collection " << nss.ns()
<< " range starting at " << minKey
<< " does not belong to shard " << shardingState.getShardName();
warning() << *errMsg << endl;
return false;
}
BSONObj lastDocMin = ( *chunksToMerge.rbegin() )->getMin();
BSONObj lastDocMax = ( *chunksToMerge.rbegin() )->getMax();
// maxKey is exclusive
bool maxKeyInRange = lastDocMin.woCompare( maxKey ) < 0 &&
lastDocMax.woCompare( maxKey ) >= 0;
if ( !maxKeyInRange ) {
*errMsg = stream() << "could not merge chunks, collection " << nss.ns()
<< " range ending at " << maxKey
<< " does not belong to shard " << shardingState.getShardName();
warning() << *errMsg << endl;
return false;
}
bool validRangeStartKey = firstDocMin.woCompare( minKey ) == 0;
bool validRangeEndKey = lastDocMax.woCompare( maxKey ) == 0;
if ( !validRangeStartKey || !validRangeEndKey ) {
*errMsg = stream() << "could not merge chunks, collection " << nss.ns()
<< " does not contain a chunk "
<< ( !validRangeStartKey ? "starting at " + minKey.toString() : "" )
<< ( !validRangeStartKey && !validRangeEndKey ? " or " : "" )
<< ( !validRangeEndKey ? "ending at " + maxKey.toString() : "" );
warning() << *errMsg << endl;
return false;
}
if ( chunksToMerge.size() == 1 ) {
*errMsg = stream() << "could not merge chunks, collection " << nss.ns()
<< " already contains chunk for " << rangeToString( minKey, maxKey );
warning() << *errMsg << endl;
return false;
}
bool holeInRange = false;
// Look for hole in range
ChunkType* prevChunk = *chunksToMerge.begin();
ChunkType* nextChunk = NULL;
for ( OwnedPointerVector<ChunkType>::const_iterator it = chunksToMerge.begin();
it != chunksToMerge.end(); ++it ) {
if ( it == chunksToMerge.begin() ) continue;
nextChunk = *it;
if ( prevChunk->getMax().woCompare( nextChunk->getMin() ) != 0 ) {
holeInRange = true;
break;
}
prevChunk = nextChunk;
}
if ( holeInRange ) {
dassert( NULL != nextChunk );
*errMsg = stream() << "could not merge chunks, collection " << nss.ns()
<< " has a hole in the range " << rangeToString( minKey, maxKey )
<< " at " << rangeToString( prevChunk->getMax(),
nextChunk->getMin() );
warning() << *errMsg << endl;
return false;
}
//
// Run apply ops command
//
BSONObj applyOpsCmd = buildApplyOpsCmd( chunksToMerge,
shardVersion,
mergeVersion );
bool ok;
BSONObj result;
try {
ScopedDbConnection conn( configLoc, 30.0 );
ok = conn->runCommand( "config", applyOpsCmd, result );
if ( !ok ) *errMsg = result.toString();
conn.done();
}
catch( const DBException& ex ) {
ok = false;
*errMsg = ex.toString();
}
if ( !ok ) {
*errMsg = stream() << "could not merge chunks for " << nss.ns()
<< ", writing to config failed" << causedBy( errMsg );
warning() << *errMsg << endl;
return false;
}
//
// Install merged chunk metadata
//
{
Lock::DBLock writeLk(txn->lockState(), nss.db(), newlm::MODE_X);
shardingState.mergeChunks(txn, nss.ns(), minKey, maxKey, mergeVersion);
}
//
// Log change
//
BSONObj mergeLogEntry = buildMergeLogEntry( chunksToMerge,
shardVersion,
mergeVersion );
configServer.logChange( "merge", nss.ns(), mergeLogEntry );
return true;
}
Status updateShardChunks(OperationContext* opCtx,
const NamespaceString& nss,
const std::vector<ChunkType>& chunks,
const OID& currEpoch) {
invariant(!chunks.empty());
NamespaceString chunkMetadataNss(ChunkType::ShardNSPrefix + nss.ns());
try {
DBDirectClient client(opCtx);
/**
* Here are examples of the operations that can happen on the config server to update
* the config.chunks collection. 'chunks' only includes the chunks that result from the
* operations, which can be read from the config server, not any that were removed, so
* we must delete any chunks that overlap with the new 'chunks'.
*
* CollectionVersion = 10.3
*
* moveChunk
* {_id: 3, max: 5, version: 10.1} --> {_id: 3, max: 5, version: 11.0}
*
* splitChunk
* {_id: 3, max: 9, version 10.3} --> {_id: 3, max: 5, version 10.4}
* {_id: 5, max: 8, version 10.5}
* {_id: 8, max: 9, version 10.6}
*
* mergeChunk
* {_id: 10, max: 14, version 4.3} --> {_id: 10, max: 22, version 10.4}
* {_id: 14, max: 19, version 7.1}
* {_id: 19, max: 22, version 2.0}
*
*/
for (auto& chunk : chunks) {
// Check for a different epoch.
if (!chunk.getVersion().hasEqualEpoch(currEpoch)) {
return Status{ErrorCodes::ConflictingOperationInProgress,
str::stream() << "Invalid chunks found when reloading '"
<< nss.toString()
<< "'. Previous collection epoch was '"
<< currEpoch.toString()
<< "', but unexpectedly found a new epoch '"
<< chunk.getVersion().epoch().toString()
<< "'. Collection was dropped and recreated."};
}
// Delete any overlapping chunk ranges. Overlapping chunks will have a min value
// ("_id") between (chunk.min, chunk.max].
//
// query: { "_id" : {"$gte": chunk.min, "$lt": chunk.max}}
auto deleteDocs(stdx::make_unique<BatchedDeleteDocument>());
deleteDocs->setQuery(BSON(ChunkType::minShardID << BSON(
"$gte" << chunk.getMin() << "$lt" << chunk.getMax())));
deleteDocs->setLimit(0);
auto deleteRequest(stdx::make_unique<BatchedDeleteRequest>());
deleteRequest->addToDeletes(deleteDocs.release());
BatchedCommandRequest batchedDeleteRequest(deleteRequest.release());
batchedDeleteRequest.setNS(chunkMetadataNss);
const BSONObj deleteCmdObj = batchedDeleteRequest.toBSON();
rpc::UniqueReply deleteCommandResponse =
client.runCommandWithMetadata(chunkMetadataNss.db().toString(),
deleteCmdObj.firstElementFieldName(),
rpc::makeEmptyMetadata(),
deleteCmdObj);
auto deleteStatus =
getStatusFromCommandResult(deleteCommandResponse->getCommandReply());
if (!deleteStatus.isOK()) {
return deleteStatus;
}
// Now the document can be expected to cleanly insert without overlap.
auto insert(stdx::make_unique<BatchedInsertRequest>());
insert->addToDocuments(chunk.toShardBSON());
BatchedCommandRequest insertRequest(insert.release());
insertRequest.setNS(chunkMetadataNss);
const BSONObj insertCmdObj = insertRequest.toBSON();
rpc::UniqueReply commandResponse =
client.runCommandWithMetadata(chunkMetadataNss.db().toString(),
insertCmdObj.firstElementFieldName(),
rpc::makeEmptyMetadata(),
insertCmdObj);
auto insertStatus = getStatusFromCommandResult(commandResponse->getCommandReply());
if (!insertStatus.isOK()) {
return insertStatus;
}
}
return Status::OK();
} catch (const DBException& ex) {
return ex.toStatus();
}
}
/* ****************************************************************************
*
* mongoRegisterContext -
*/
HttpStatusCode mongoRegisterContext(RegisterContextRequest* requestP, RegisterContextResponse* responseP, const std::string& tenant)
{
reqSemTake(__FUNCTION__, "ngsi9 register request");
LM_T(LmtMongo, ("Register Context Request"));
DBClientBase* connection = getMongoConnection();
/* Check if new registration */
if (requestP->registrationId.isEmpty()) {
HttpStatusCode result = processRegisterContext(requestP, responseP, NULL, tenant);
reqSemGive(__FUNCTION__, "ngsi9 register request");
return result;
}
/* It is not a new registration, so it should be an update */
BSONObj reg;
OID id;
try
{
id = OID(requestP->registrationId.get());
mongoSemTake(__FUNCTION__, "findOne from RegistrationsCollection");
reg = connection->findOne(getRegistrationsCollectionName(tenant).c_str(), BSON("_id" << id));
mongoSemGive(__FUNCTION__, "findOne from RegistrationsCollection");
LM_I(("Database Operation Successful (findOne _id: %s)", id.toString().c_str()));
}
catch (const AssertionException &e)
{
mongoSemGive(__FUNCTION__, "findOne from RegistrationsCollection (AssertionException)");
reqSemGive(__FUNCTION__, "ngsi9 register request");
/* This happens when OID format is wrong */
// FIXME: this checking should be done at parsing stage, without progressing to
// mongoBackend. By the moment we can live this here, but we should remove in the future
responseP->errorCode.fill(SccContextElementNotFound);
responseP->registrationId = requestP->registrationId;
++noOfRegistrationUpdateErrors;
LM_W(("Bad Input (invalid OID format)"));
return SccOk;
}
catch (const DBException &e)
{
mongoSemGive(__FUNCTION__, "findOne from RegistrationsCollection (DBException)");
reqSemGive(__FUNCTION__, "ngsi9 register request");
responseP->errorCode.fill(SccReceiverInternalError,
std::string("collection: ") + getRegistrationsCollectionName(tenant).c_str() +
" - findOne() _id: " + requestP->registrationId.get() +
" - exception: " + e.what());
++noOfRegistrationUpdateErrors;
LM_E(("Database Error (%s)", responseP->errorCode.details.c_str()));
return SccOk;
}
catch (...)
{
mongoSemGive(__FUNCTION__, "findOne from RegistrationsCollection (Generic Exception)");
reqSemGive(__FUNCTION__, "ngsi9 register request");
responseP->errorCode.fill(SccReceiverInternalError,
std::string("collection: ") + getRegistrationsCollectionName(tenant).c_str() +
" - findOne() _id: " + requestP->registrationId.get() +
" - exception: " + "generic");
++noOfRegistrationUpdateErrors;
LM_E(("Database Error (%s)", responseP->errorCode.details.c_str()));
return SccOk;
}
if (reg.isEmpty())
{
reqSemGive(__FUNCTION__, "ngsi9 register request (no registrations found)");
responseP->errorCode.fill(SccContextElementNotFound, std::string("registration id: /") + requestP->registrationId.get() + "/");
responseP->registrationId = requestP->registrationId;
++noOfRegistrationUpdateErrors;
return SccOk;
}
HttpStatusCode result = processRegisterContext(requestP, responseP, &id, tenant);
reqSemGive(__FUNCTION__, "ngsi9 register request");
return result;
}
void VStore::Object::copy(const OID &nobj) {
//OID nobj = OID::create();
OID res, value;
int flags = 0;
Attribute *attrib;
char *dbuf = new char [5000];
char *dbuf2;
DASM *dasm = (DASM*)(root.get("notations").get("dasm"));
for (int i=0; i<m_keys.size(); i++) {
if (m_keys[i] == This) continue;
attrib = Attribute::get(m_oid, m_keys[i]);
if (attrib == 0) {
res = Null;
value = Null;
} else {
flags = attrib->getFlags();
res = attrib->getDefinition();
}
/*if (flags & OID::FLAG_OUT_OF_DATE) {
attrib->setFlags(flags & (0xFF - OID::FLAG_OUT_OF_DATE));
flags = attrib->getFlags();
attrib->setValue(Definition(res).evaluate(m_oid, m_keys[i], false));
value = attrib->getValue();
}*/
if (res != Null) {
//Clone the definition
//value = res;
//res = s_alloc;
//s_alloc++;
//value.copy(res);
//std::cout << "CLONING DEF\n";
dbuf2 = dbuf;
Definition d = res;
nobj.toString(dbuf2, 100);
dbuf2 += strlen(dbuf2);
strcpy(dbuf2, " ");
dbuf2 += strlen(dbuf2);
m_keys[i].toString(dbuf2,100);
dbuf2 += strlen(dbuf2);
if (flags && 0x01) {
//nobj.define(m_keys[i], res, true);
strcpy(dbuf2, " is { ");
dbuf2 += strlen(dbuf2);
} else {
value = attrib->getValue();
//nobj.set(m_keys[i], value, true);
//nobj.definefuture(m_keys[i], res, true);
strcpy(dbuf2, " = ");
dbuf2 += strlen(dbuf2);
value.toString(dbuf2, 100);
dbuf2 += strlen(dbuf2);
strcpy(dbuf2, "\n");
dbuf2 += strlen(dbuf2);
m_keys[i].toString(dbuf2, 100);
dbuf2 += strlen(dbuf2);
strcpy(dbuf2, " := { ");
dbuf2 += strlen(dbuf2);
}
d.toString(dbuf2, 4000);
dbuf2 += strlen(dbuf2);
strcpy(dbuf2, " }\n");
//std::cout << "DBUF: " << dbuf << "\n";
//Execute
dasm->execute(dbuf);
} else {
if (flags && OID::FLAG_DEEP) {
//#ifdef LINUX
//SPINLOCK
//pthread_mutex_lock(&oid_lock);
//#endif
res = s_alloc;
s_alloc++;
//#ifdef LINUX
//SPINLOCK
//pthread_mutex_unlock(&oid_lock);
//#endif
value = attrib->getValue();
value.copy(res);
res.set(This, nobj, true);
nobj.set(m_keys[i], res, true);
} else {
value = attrib->getValue();
if (value != Null) nobj.set(m_keys[i], value, true);
}
nobj.flags(m_keys[i],flags, true);
}
//delete evt;
}
delete [] dbuf;
}
/* ****************************************************************************
*
* processRegisterContext -
*
* This function has a slightly different behaviour depending on whether the id
* parameter is null (new registration case) or not null (update case), in
* particular:
*
* - In the new registration case, the _id is generated and insert() is used to
* put the document in the DB.
* - In the update case, the _id is set according to the argument 'id' and update() is
* used to put the document in the DB.
*
*/
HttpStatusCode processRegisterContext
(
RegisterContextRequest* requestP,
RegisterContextResponse* responseP,
OID* id,
const std::string& tenant,
const std::string& servicePath,
const std::string& format
)
{
DBClientBase* connection = NULL;
/* If expiration is not present, then use a default one */
if (requestP->duration.isEmpty())
{
requestP->duration.set(DEFAULT_DURATION);
}
/* Calculate expiration (using the current time and the duration field in the request) */
long long expiration = getCurrentTime() + requestP->duration.parse();
LM_T(LmtMongo, ("Registration expiration: %lu", expiration));
/* Create the mongoDB registration document */
BSONObjBuilder reg;
OID oid;
if (id == NULL) {
oid.init();
}
else {
oid = *id;
}
reg.append("_id", oid);
reg.append(REG_EXPIRATION, expiration);
reg.append(REG_SERVICE_PATH, servicePath);
reg.append(REG_FORMAT, format);
/* We accumulate the subscriptions in a map. The key of the map is the string representing
* subscription id */
std::map<string, TriggeredSubscription*> subsToNotify;
/* This vector is used to define which entities to include in notifications */
EntityIdVector triggerEntitiesV;
BSONArrayBuilder contextRegistration;
for (unsigned int ix = 0; ix < requestP->contextRegistrationVector.size(); ++ix)
{
ContextRegistration* cr = requestP->contextRegistrationVector.get(ix);
BSONArrayBuilder entities;
for (unsigned int jx = 0; jx < cr->entityIdVector.size(); ++jx)
{
EntityId* en = cr->entityIdVector.get(jx);
triggerEntitiesV.push_back(en);
if (en->type == "")
{
entities.append(BSON(REG_ENTITY_ID << en->id));
LM_T(LmtMongo, ("Entity registration: {id: %s}", en->id.c_str()));
}
else
{
entities.append(BSON(REG_ENTITY_ID << en->id << REG_ENTITY_TYPE << en->type));
LM_T(LmtMongo, ("Entity registration: {id: %s, type: %s}", en->id.c_str(), en->type.c_str()));
}
}
BSONArrayBuilder attrs;
for (unsigned int jx = 0; jx < cr->contextRegistrationAttributeVector.size(); ++jx)
{
ContextRegistrationAttribute* cra = cr->contextRegistrationAttributeVector.get(jx);
attrs.append(BSON(REG_ATTRS_NAME << cra->name << REG_ATTRS_TYPE << cra->type << "isDomain" << cra->isDomain));
LM_T(LmtMongo, ("Attribute registration: {name: %s, type: %s, isDomain: %s}",
cra->name.c_str(),
cra->type.c_str(),
cra->isDomain.c_str()));
for (unsigned int kx = 0; kx < requestP->contextRegistrationVector.get(ix)->contextRegistrationAttributeVector.get(jx)->metadataVector.size(); ++kx)
{
// FIXME: metadata not supported at the moment
}
}
contextRegistration.append(BSON(
REG_ENTITIES << entities.arr() <<
REG_ATTRS << attrs.arr() <<
REG_PROVIDING_APPLICATION << requestP->contextRegistrationVector.get(ix)->providingApplication.get())
);
LM_T(LmtMongo, ("providingApplication registration: %s", requestP->contextRegistrationVector.get(ix)->providingApplication.c_str()));
std::string err;
if (!processAssociations(cr->registrationMetadataVector, &err, tenant))
{
responseP->errorCode.fill(SccReceiverInternalError);
return SccOk;
}
if (!addTriggeredSubscriptions(*cr, subsToNotify, err, tenant))
{
responseP->errorCode.fill(SccReceiverInternalError, err);
return SccOk;
}
}
reg.append(REG_CONTEXT_REGISTRATION, contextRegistration.arr());
BSONObj regDoc = reg.obj();
LM_T(LmtMongo, ("upsert update() in '%s' collection: '%s'", getRegistrationsCollectionName(tenant).c_str(), regDoc.toString().c_str()));
try
{
connection = getMongoConnection();
/* Note the fourth parameter is set to "true". This means "upsert", so if the document doesn't previously
* exist in the collection, it is created. Thus, this way is ok with both uses of
* registerContext (either new registration or updating an existing one) */
connection->update(getRegistrationsCollectionName(tenant).c_str(), BSON("_id" << oid), regDoc, true);
releaseMongoConnection(connection);
LM_I(("Database Operation Successful (_id: %s)", oid.toString().c_str()));
}
catch (const DBException& e)
{
releaseMongoConnection(connection);
responseP->errorCode.fill(SccReceiverInternalError,
std::string("collection: ") + getRegistrationsCollectionName(tenant).c_str() +
" - upsert update(): " + regDoc.toString() +
" - exception: " + e.what());
LM_E(("Database Error (%s)", responseP->errorCode.reasonPhrase.c_str()));
releaseTriggeredSubscriptions(subsToNotify);
return SccOk;
}
catch (...)
{
releaseMongoConnection(connection);
responseP->errorCode.fill(SccReceiverInternalError,
std::string("collection: ") + getRegistrationsCollectionName(tenant).c_str() +
" - upsert update(): " + regDoc.toString() +
" - exception: " + "generic");
LM_E(("Database Error (%s)", responseP->errorCode.reasonPhrase.c_str()));
releaseTriggeredSubscriptions(subsToNotify);
return SccOk;
}
/* Send notifications for each one of the subscriptions accumulated by
* previous addTriggeredSubscriptions() invocations */
std::string err;
processSubscriptions(triggerEntitiesV, subsToNotify, err, tenant);
/* Fill the response element */
responseP->duration = requestP->duration;
responseP->registrationId.set(oid.toString());
responseP->errorCode.fill(SccOk);
return SccOk;
}
/**
* Upgrade v3 to v4 described here.
*
* This upgrade takes a config server without collection epochs (potentially) and adds
* epochs to all mongo processes.
*
*/
bool doUpgradeV3ToV4(const ConnectionString& configLoc,
const VersionType& lastVersionInfo,
string* errMsg)
{
string dummy;
if (!errMsg) errMsg = &dummy;
verify(lastVersionInfo.getCurrentVersion() == UpgradeHistory_NoEpochVersion);
if (lastVersionInfo.isUpgradeIdSet() && lastVersionInfo.getUpgradeId().isSet()) {
//
// Another upgrade failed, so cleanup may be necessary
//
BSONObj lastUpgradeState = lastVersionInfo.getUpgradeState();
bool inCriticalSection;
if (!FieldParser::extract(lastUpgradeState,
inCriticalSectionField,
&inCriticalSection,
errMsg))
{
*errMsg = stream() << "problem reading previous upgrade state" << causedBy(errMsg);
return false;
}
if (inCriticalSection) {
// Manual intervention is needed here. Somehow our upgrade didn't get applied
// consistently across config servers.
*errMsg = cannotCleanupMessage;
return false;
}
if (!_cleanupUpgradeState(configLoc, lastVersionInfo.getUpgradeId(), errMsg)) {
// If we can't cleanup the old upgrade state, the user might have done it for us,
// not a fatal problem (we'll just end up with extra collections).
warning() << "could not cleanup previous upgrade state" << causedBy(errMsg) << endl;
*errMsg = "";
}
}
//
// Check the versions of other mongo processes in the cluster before upgrade.
// We can't upgrade if there are active pre-v2.2 processes in the cluster
//
Status mongoVersionStatus = checkClusterMongoVersions(configLoc,
string(minMongoProcessVersion));
if (!mongoVersionStatus.isOK()) {
*errMsg = stream() << "cannot upgrade with pre-v" << minMongoProcessVersion
<< " mongo processes active in the cluster"
<< causedBy(mongoVersionStatus);
return false;
}
VersionType newVersionInfo;
lastVersionInfo.cloneTo(&newVersionInfo);
// Set our upgrade id and state
OID upgradeId = OID::gen();
newVersionInfo.setUpgradeId(upgradeId);
newVersionInfo.setUpgradeState(BSONObj());
// Write our upgrade id and state
{
scoped_ptr<ScopedDbConnection> connPtr;
try {
connPtr.reset(ScopedDbConnection::getInternalScopedDbConnection(configLoc, 30));
ScopedDbConnection& conn = *connPtr;
verify(newVersionInfo.isValid(NULL));
conn->update(VersionType::ConfigNS,
BSON("_id" << 1 << VersionType::version_DEPRECATED(3)),
newVersionInfo.toBSON());
_checkGLE(conn);
}
catch (const DBException& e) {
*errMsg = stream() << "could not initialize version info for upgrade"
<< causedBy(e);
return false;
}
connPtr->done();
}
//
// First lock all collection namespaces that exist
//
OwnedPointerMap<string, CollectionType> ownedCollections;
const map<string, CollectionType*>& collections = ownedCollections.map();
Status findCollectionsStatus = findAllCollectionsV3(configLoc, &ownedCollections);
if (!findCollectionsStatus.isOK()) {
*errMsg = stream() << "could not read collections from config server"
<< causedBy(findCollectionsStatus);
return false;
}
//
// Acquire locks for all sharded collections
// Something that didn't involve getting thousands of locks would be better.
//
OwnedPointerVector<ScopedDistributedLock> collectionLocks;
log() << "acquiring locks for " << collections.size() << " sharded collections..." << endl;
// WARNING - this string is used programmatically when forcing locks, be careful when
// changing!
// TODO: Add programmatic "why" field to lock collection
string lockMessage = str::stream() << "ensuring epochs for config upgrade"
<< " (" << upgradeId.toString() << ")";
if (!_acquireAllCollectionLocks(configLoc,
collections,
lockMessage,
20 * 60 * 1000,
&collectionLocks,
errMsg))
{
*errMsg = stream() << "could not acquire all namespace locks for upgrade"
<< " (" << upgradeId.toString() << ")"
<< causedBy(errMsg);
return false;
}
// We are now preventing all splits and migrates for all sharded collections
// Get working and backup suffixes
string workingSuffix = genWorkingSuffix(upgradeId);
string backupSuffix = genBackupSuffix(upgradeId);
log() << "copying collection and chunk metadata to working and backup collections..."
<< endl;
// Get a backup and working copy of the config.collections and config.chunks collections
Status copyStatus = copyFrozenCollection(configLoc,
CollectionType::ConfigNS,
CollectionType::ConfigNS + workingSuffix);
if (!copyStatus.isOK()) {
*errMsg = stream() << "could not copy " << CollectionType::ConfigNS << " to "
<< (CollectionType::ConfigNS + workingSuffix)
<< causedBy(copyStatus);
return false;
}
copyStatus = copyFrozenCollection(configLoc,
CollectionType::ConfigNS,
CollectionType::ConfigNS + backupSuffix);
if (!copyStatus.isOK()) {
*errMsg = stream() << "could not copy " << CollectionType::ConfigNS << " to "
<< (CollectionType::ConfigNS + backupSuffix) << causedBy(copyStatus);
return false;
}
copyStatus = copyFrozenCollection(configLoc,
ChunkType::ConfigNS,
ChunkType::ConfigNS + workingSuffix);
if (!copyStatus.isOK()) {
*errMsg = stream() << "could not copy " << ChunkType::ConfigNS << " to "
<< (ChunkType::ConfigNS + workingSuffix) << causedBy(copyStatus);
return false;
}
copyStatus = copyFrozenCollection(configLoc,
ChunkType::ConfigNS,
ChunkType::ConfigNS + backupSuffix);
if (!copyStatus.isOK()) {
*errMsg = stream() << "could not copy " << ChunkType::ConfigNS << " to "
<< (ChunkType::ConfigNS + backupSuffix) << causedBy(copyStatus);
return false;
}
//
// Go through sharded collections one-by-one and add epochs where missing
//
for (map<string, CollectionType*>::const_iterator it = collections.begin();
it != collections.end(); ++it)
{
// Create a copy so that we can change the epoch later
CollectionType collection;
it->second->cloneTo(&collection);
log() << "checking epochs for " << collection.getNS() << " collection..." << endl;
OID epoch = collection.getEpoch();
//
// Go through chunks to find epoch if we haven't found it or to verify epoch is the same
//
OwnedPointerVector<ChunkType> ownedChunks;
const vector<ChunkType*>& chunks = ownedChunks.vector();
Status findChunksStatus = findAllChunks(configLoc, collection.getNS(), &ownedChunks);
if (!findChunksStatus.isOK()) {
*errMsg = stream() << "could not read chunks from config server"
<< causedBy(findChunksStatus);
return false;
}
for (vector<ChunkType*>::const_iterator chunkIt = chunks.begin();
chunkIt != chunks.end(); ++chunkIt)
{
const ChunkType& chunk = *(*chunkIt);
// If our chunk epoch is set and doesn't match
if (epoch.isSet() && chunk.getVersion().epoch().isSet()
&& chunk.getVersion().epoch() != epoch)
{
*errMsg = stream() << "chunk epoch for " << chunk.toString() << " in "
<< collection.getNS() << " does not match found epoch "
<< epoch;
return false;
}
else if (!epoch.isSet() && chunk.getVersion().epoch().isSet()) {
epoch = chunk.getVersion().epoch();
}
}
//
// Write collection epoch if needed
//
if (!collection.getEpoch().isSet()) {
OID newEpoch = OID::gen();
log() << "writing new epoch " << newEpoch << " for " << collection.getNS()
<< " collection..." << endl;
scoped_ptr<ScopedDbConnection> connPtr;
try {
connPtr.reset(ScopedDbConnection::getInternalScopedDbConnection(configLoc, 30));
ScopedDbConnection& conn = *connPtr;
conn->update(CollectionType::ConfigNS + workingSuffix,
BSON(CollectionType::ns(collection.getNS())),
BSON("$set" << BSON(CollectionType::DEPRECATED_lastmodEpoch(newEpoch))));
_checkGLE(conn);
}
catch (const DBException& e) {
*errMsg = stream() << "could not write a new epoch for " << collection.getNS()
<< causedBy(e);
return false;
}
connPtr->done();
collection.setEpoch(newEpoch);
}
epoch = collection.getEpoch();
verify(epoch.isSet());
//
// Now write verified epoch to all chunks
//
log() << "writing epoch " << epoch << " for " << chunks.size() << " chunks in "
<< collection.getNS() << " collection..." << endl;
{
scoped_ptr<ScopedDbConnection> connPtr;
try {
connPtr.reset(ScopedDbConnection::getInternalScopedDbConnection(configLoc, 30));
ScopedDbConnection& conn = *connPtr;
// Multi-update of all chunks
conn->update(ChunkType::ConfigNS + workingSuffix,
BSON(ChunkType::ns(collection.getNS())),
BSON("$set" << BSON(ChunkType::DEPRECATED_epoch(epoch))),
false,
true); // multi
_checkGLE(conn);
}
catch (const DBException& e) {
*errMsg = stream() << "could not write a new epoch " << epoch.toString()
<< " for chunks in " << collection.getNS() << causedBy(e);
return false;
}
connPtr->done();
}
}
//
// Paranoid verify the collection writes
//
{
scoped_ptr<ScopedDbConnection> connPtr;
try {
connPtr.reset(ScopedDbConnection::getInternalScopedDbConnection(configLoc, 30));
ScopedDbConnection& conn = *connPtr;
// Find collections with no epochs
BSONObj emptyDoc =
conn->findOne(CollectionType::ConfigNS + workingSuffix,
BSON("$unset" << BSON(CollectionType::DEPRECATED_lastmodEpoch() << 1)));
if (!emptyDoc.isEmpty()) {
*errMsg = stream() << "collection " << emptyDoc
<< " is still missing epoch after config upgrade";
connPtr->done();
return false;
}
// Find collections with empty epochs
emptyDoc = conn->findOne(CollectionType::ConfigNS + workingSuffix,
BSON(CollectionType::DEPRECATED_lastmodEpoch(OID())));
if (!emptyDoc.isEmpty()) {
*errMsg = stream() << "collection " << emptyDoc
<< " still has empty epoch after config upgrade";
connPtr->done();
return false;
}
// Find chunks with no epochs
emptyDoc =
conn->findOne(ChunkType::ConfigNS + workingSuffix,
BSON("$unset" << BSON(ChunkType::DEPRECATED_epoch() << 1)));
if (!emptyDoc.isEmpty()) {
*errMsg = stream() << "chunk " << emptyDoc
<< " is still missing epoch after config upgrade";
connPtr->done();
return false;
}
// Find chunks with empty epochs
emptyDoc = conn->findOne(ChunkType::ConfigNS + workingSuffix,
BSON(ChunkType::DEPRECATED_epoch(OID())));
if (!emptyDoc.isEmpty()) {
*errMsg = stream() << "chunk " << emptyDoc
<< " still has empty epoch after config upgrade";
connPtr->done();
return false;
}
}
catch (const DBException& e) {
*errMsg = stream() << "could not verify epoch writes" << causedBy(e);
return false;
}
connPtr->done();
}
//
// Double check that our collections haven't changed
//
Status idCheckStatus = checkIdsTheSame(configLoc,
CollectionType::ConfigNS,
CollectionType::ConfigNS + workingSuffix);
if (!idCheckStatus.isOK()) {
*errMsg = stream() << CollectionType::ConfigNS
<< " was modified while working on upgrade"
<< causedBy(idCheckStatus);
return false;
}
idCheckStatus = checkIdsTheSame(configLoc,
ChunkType::ConfigNS,
ChunkType::ConfigNS + workingSuffix);
if (!idCheckStatus.isOK()) {
*errMsg = stream() << ChunkType::ConfigNS << " was modified while working on upgrade"
<< causedBy(idCheckStatus);
return false;
}
//
// ENTER CRITICAL SECTION
//
newVersionInfo.setUpgradeState(BSON(inCriticalSectionField(true)));
{
scoped_ptr<ScopedDbConnection> connPtr;
try {
connPtr.reset(ScopedDbConnection::getInternalScopedDbConnection(configLoc, 30));
ScopedDbConnection& conn = *connPtr;
verify(newVersionInfo.isValid(NULL));
conn->update(VersionType::ConfigNS,
BSON("_id" << 1 << VersionType::version_DEPRECATED(3)),
newVersionInfo.toBSON());
_checkGLE(conn);
}
catch (const DBException& e) {
// No cleanup message here since we're not sure if we wrote or not, and
// not dangerous either way except to prevent further updates (at which point
// the message is printed)
*errMsg = stream()
<< "could not update version info to enter critical update section"
<< causedBy(e);
return false;
}
// AT THIS POINT ANY FAILURE REQUIRES MANUAL INTERVENTION!
connPtr->done();
}
log() << "entered critical section for config upgrade" << endl;
Status overwriteStatus = overwriteCollection(configLoc,
CollectionType::ConfigNS + workingSuffix,
CollectionType::ConfigNS);
if (!overwriteStatus.isOK()) {
error() << cleanupMessage << endl;
*errMsg = stream() << "could not overwrite collection " << CollectionType::ConfigNS
<< " with working collection "
<< (CollectionType::ConfigNS + workingSuffix)
<< causedBy(overwriteStatus);
return false;
}
overwriteStatus = overwriteCollection(configLoc,
ChunkType::ConfigNS + workingSuffix,
ChunkType::ConfigNS);
if (!overwriteStatus.isOK()) {
error() << cleanupMessage << endl;
*errMsg = stream() << "could not overwrite collection " << ChunkType::ConfigNS
<< " with working collection "
<< (ChunkType::ConfigNS + workingSuffix)
<< causedBy(overwriteStatus);
return false;
}
//
// Finally update the version to latest and add clusterId to version
//
OID newClusterId = OID::gen();
// Note: hardcoded versions, since this is a very particular upgrade
// Note: DO NOT CLEAR the config version unless bumping the minCompatibleVersion,
// we want to save the excludes that were set.
newVersionInfo.setMinCompatibleVersion(UpgradeHistory_NoEpochVersion);
newVersionInfo.setCurrentVersion(UpgradeHistory_MandatoryEpochVersion);
newVersionInfo.setClusterId(newClusterId);
// Leave critical section
newVersionInfo.unsetUpgradeId();
newVersionInfo.unsetUpgradeState();
log() << "writing new version info and clusterId " << newClusterId << "..." << endl;
{
scoped_ptr<ScopedDbConnection> connPtr;
try {
connPtr.reset(ScopedDbConnection::getInternalScopedDbConnection(configLoc, 30));
ScopedDbConnection& conn = *connPtr;
verify(newVersionInfo.isValid(NULL));
conn->update(VersionType::ConfigNS,
BSON("_id" << 1 << VersionType::version_DEPRECATED(UpgradeHistory_NoEpochVersion)),
newVersionInfo.toBSON());
_checkGLE(conn);
}
catch (const DBException& e) {
error() << cleanupMessage << endl;
*errMsg = stream() << "could not write new version info "
<< "and exit critical upgrade section" << causedBy(e);
return false;
}
connPtr->done();
}
//
// END CRITICAL SECTION
//
return true;
}
bool _cleanupUpgradeState(const ConnectionString& configLoc,
const OID& lastUpgradeId,
string* errMsg)
{
string dummy;
if (!errMsg) errMsg = &dummy;
scoped_ptr<ScopedDbConnection> connPtr;
string workingSuffix = genWorkingSuffix(lastUpgradeId);
try {
connPtr.reset(ScopedDbConnection::getInternalScopedDbConnection(configLoc, 30));
ScopedDbConnection& conn = *connPtr;
// Drop old upgrade collections on config server
bool resultOk;
BSONObj dropResult;
resultOk = conn->dropCollection(CollectionType::ConfigNS + workingSuffix, &dropResult);
if (!resultOk) {
*errMsg = stream() << "could not drop collection "
<< (CollectionType::ConfigNS + workingSuffix)
<< causedBy(dropResult.toString());
return false;
}
resultOk = conn->dropCollection(ChunkType::ConfigNS + workingSuffix, &dropResult);
if (!resultOk) {
*errMsg = stream() << "could not drop collection "
<< (ChunkType::ConfigNS + workingSuffix)
<< causedBy(dropResult.toString());
return false;
}
// Force old locks taken by previous upgrade process on config server
// This is safe because no previous upgrade process can be active while we hold the
// upgrade lock.
log() << "forcing upgrade locks of previous failed upgrade with id "
<< lastUpgradeId.toString() << endl;
// Explicit builder needed b/c of regex
BSONObjBuilder lockQueryB;
lockQueryB.appendRegex(LocksType::why(),
pcrecpp::RE::QuoteMeta("(" + lastUpgradeId.toString() + ")"));
conn->update(LocksType::ConfigNS,
lockQueryB.obj(),
BSON("$set" << BSON(LocksType::state(0))),
false, true); // multi
_checkGLE(conn);
}
catch (const DBException& e) {
*errMsg = stream() << "could not drop collections during cleanup of upgrade "
<< lastUpgradeId << causedBy(e);
return false;
}
connPtr->done();
return true;
}
string genWorkingSuffix(const OID& lastUpgradeId) {
return "-upgrade-" + lastUpgradeId.toString();
}
/* ****************************************************************************
*
* mongoUpdateContextSubscription -
*/
HttpStatusCode mongoUpdateContextSubscription(UpdateContextSubscriptionRequest* requestP, UpdateContextSubscriptionResponse* responseP, Format inFormat, const std::string& tenant)
{
reqSemTake(__FUNCTION__, "ngsi10 update subscription request");
LM_T(LmtMongo, ("Update Context Subscription"));
DBClientBase* connection = getMongoConnection();
/* Look for document */
BSONObj sub;
try
{
OID id = OID(requestP->subscriptionId.get());
mongoSemTake(__FUNCTION__, "findOne in SubscribeContextCollection");
sub = connection->findOne(getSubscribeContextCollectionName(tenant).c_str(), BSON("_id" << id));
mongoSemGive(__FUNCTION__, "findOne in SubscribeContextCollection");
LM_I(("Database Operation Successful (findOne _id: %s)", id.toString().c_str()));
}
catch (const AssertionException &e)
{
/* This happens when OID format is wrong */
// FIXME P4: this checking should be done at the parsing stage, without progressing to
// mongoBackend. For the moment we can leave this here, but we should remove it in the future
// (old issue #95)
mongoSemGive(__FUNCTION__, "findOne in SubscribeContextCollection (mongo assertion exception)");
reqSemGive(__FUNCTION__, "ngsi10 update subscription request (mongo assertion exception)");
responseP->subscribeError.errorCode.fill(SccContextElementNotFound);
LM_W(("Bad Input (invalid OID format)"));
return SccOk;
}
catch (const DBException &e)
{
mongoSemGive(__FUNCTION__, "findOne in SubscribeContextCollection (mongo db exception)");
reqSemGive(__FUNCTION__, "ngsi10 update subscription request (mongo db exception)");
responseP->subscribeError.errorCode.fill(SccReceiverInternalError,
std::string("collection: ") + getSubscribeContextCollectionName(tenant).c_str() +
" - findOne() _id: " + requestP->subscriptionId.get() +
" - exception: " + e.what());
LM_E(("Database Error (%s)", responseP->subscribeError.errorCode.details.c_str()));
return SccOk;
}
catch (...)
{
mongoSemGive(__FUNCTION__, "findOne in SubscribeContextCollection (mongo generic exception)");
reqSemGive(__FUNCTION__, "ngsi10 update subscription request (mongo generic exception)");
responseP->subscribeError.errorCode.fill(SccReceiverInternalError,
std::string("collection: ") + getSubscribeContextCollectionName(tenant).c_str() +
" - findOne() _id: " + requestP->subscriptionId.get() +
" - exception: " + "generic");
LM_E(("Database Error (%s)", responseP->subscribeError.errorCode.details.c_str()));
return SccOk;
}
if (sub.isEmpty()) {
responseP->subscribeError.errorCode.fill(SccContextElementNotFound);
reqSemGive(__FUNCTION__, "ngsi10 update subscription request (no subscriptions found)");
return SccOk;
}
/* We start with an empty BSONObjBuilder and process requestP for all the fields that can
* be updated. I don't like too much this strategy (I would have preferred to start with
* a copy of the original document, then modify as neded, but this doesn't seem to be easy
* using the API provide by the Mongo C++ driver)
*
* FIXME: a better implementation strategy could be doing an findAndModify() query to do the
* update, so detecting if the document was not found, instead of using findOne() + update()
* with $set operation. One operations to MongoDb. vs two operations.
*/
BSONObjBuilder newSub;
/* Entities, attribute list and reference are not updatable, so they are appended directly */
newSub.appendArray(CSUB_ENTITIES, sub.getField(CSUB_ENTITIES).Obj());
newSub.appendArray(CSUB_ATTRS, sub.getField(CSUB_ATTRS).Obj());
newSub.append(CSUB_REFERENCE, STR_FIELD(sub, CSUB_REFERENCE));
/* Duration update */
if (requestP->duration.isEmpty()) {
newSub.append(CSUB_EXPIRATION, sub.getField(CSUB_EXPIRATION).numberLong());
}
else {
long long expiration = getCurrentTime() + requestP->duration.parse();
newSub.append(CSUB_EXPIRATION, expiration);
LM_T(LmtMongo, ("New subscription expiration: %l", expiration));
}
/* Restriction update */
// FIXME: Restrictions not implemented yet
/* Throttling update */
if (!requestP->throttling.isEmpty()) {
/* Throttling equal to 0 removes throttling */
long long throttling = requestP->throttling.parse();
if (throttling != 0) {
newSub.append(CSUB_THROTTLING, throttling);
}
}
else {
/* The hasField check is needed due to Throttling could not be present in the original doc */
if (sub.hasField(CSUB_THROTTLING)) {
newSub.append(CSUB_THROTTLING, sub.getField(CSUB_THROTTLING).numberLong());
}
}
/* Notify conditions */
bool notificationDone = false;
if (requestP->notifyConditionVector.size() == 0) {
newSub.appendArray(CSUB_CONDITIONS, sub.getField(CSUB_CONDITIONS).embeddedObject());
}
else {
/* Destroy any previous ONTIMEINTERVAL thread */
getNotifier()->destroyOntimeIntervalThreads(requestP->subscriptionId.get());
/* Build conditions array (including side-effect notifications and threads creation)
* In order to do so, we have to create and EntityIdVector and AttributeList from sub
* document, given the processConditionVector() signature */
EntityIdVector enV = subToEntityIdVector(sub);
AttributeList attrL = subToAttributeList(sub);
BSONArray conds = processConditionVector(&requestP->notifyConditionVector,
enV,
attrL,
requestP->subscriptionId.get(),
C_STR_FIELD(sub, CSUB_REFERENCE),
¬ificationDone,
inFormat,
tenant);
newSub.appendArray(CSUB_CONDITIONS, conds);
/* Remove EntityIdVector and AttributeList dynamic memory */
enV.release();
attrL.release();
}
int count = sub.hasField(CSUB_COUNT) ? sub.getIntField(CSUB_COUNT) : 0;
/* Last notification */
if (notificationDone) {
newSub.append(CSUB_LASTNOTIFICATION, getCurrentTime());
newSub.append(CSUB_COUNT, count + 1);
}
else {
/* The hasField check is needed due to lastNotification/count could not be present in the original doc */
if (sub.hasField(CSUB_LASTNOTIFICATION)) {
newSub.append(CSUB_LASTNOTIFICATION, sub.getIntField(CSUB_LASTNOTIFICATION));
}
if (sub.hasField(CSUB_COUNT)) {
newSub.append(CSUB_COUNT, count);
}
}
/* Adding format to use in notifications */
newSub.append(CSUB_FORMAT, std::string(formatToString(inFormat)));
/* Update document in MongoDB */
BSONObj update = newSub.obj();
try
{
LM_T(LmtMongo, ("update() in '%s' collection _id '%s': %s}", getSubscribeContextCollectionName(tenant).c_str(),
requestP->subscriptionId.get().c_str(),
update.toString().c_str()));
mongoSemTake(__FUNCTION__, "update in SubscribeContextCollection");
connection->update(getSubscribeContextCollectionName(tenant).c_str(), BSON("_id" << OID(requestP->subscriptionId.get())), update);
mongoSemGive(__FUNCTION__, "update in SubscribeContextCollection");
LM_I(("Database Operation Successful (update _id: %s, %s)", requestP->subscriptionId.get().c_str(), update.toString().c_str()));
}
catch (const DBException &e)
{
mongoSemGive(__FUNCTION__, "update in SubscribeContextCollection (mongo db exception)");
reqSemGive(__FUNCTION__, "ngsi10 update subscription request (mongo db exception)");
responseP->subscribeError.errorCode.fill(SccReceiverInternalError,
std::string("collection: ") + getSubscribeContextCollectionName(tenant).c_str() +
" - update() _id: " + requestP->subscriptionId.get().c_str() +
" - update() doc: " + update.toString() +
" - exception: " + e.what());
LM_E(("Database Error (%s)", responseP->subscribeError.errorCode.details.c_str()));
return SccOk;
}
catch (...)
{
mongoSemGive(__FUNCTION__, "update in SubscribeContextCollection (mongo generic exception)");
reqSemGive(__FUNCTION__, "ngsi10 update subscription request (mongo generic exception)");
responseP->subscribeError.errorCode.fill(SccReceiverInternalError,
std::string("collection: ") + getSubscribeContextCollectionName(tenant).c_str() +
" - update() _id: " + requestP->subscriptionId.get().c_str() +
" - update() doc: " + update.toString() +
" - exception: " + "generic");
LM_E(("Database Error (%s)", responseP->subscribeError.errorCode.details.c_str()));
return SccOk;
}
/* Duration and throttling are optional parameters, they are only added in the case they
* was used for update */
if (!requestP->duration.isEmpty()) {
responseP->subscribeResponse.duration = requestP->duration;
}
if (!requestP->throttling.isEmpty()) {
responseP->subscribeResponse.throttling = requestP->throttling;
}
responseP->subscribeResponse.subscriptionId = requestP->subscriptionId;
reqSemGive(__FUNCTION__, "ngsi10 update subscription request");
return SccOk;
}
string genBackupSuffix(const OID& lastUpgradeId) {
return "-backup-" + lastUpgradeId.toString();
}
/* ****************************************************************************
*
* mongoUnsubscribeContext -
*/
HttpStatusCode mongoUnsubscribeContext(UnsubscribeContextRequest* requestP, UnsubscribeContextResponse* responseP, const std::string& tenant)
{
bool reqSemTaken;
BSONObj sub;
DBClientBase* connection = NULL;
reqSemTake(__FUNCTION__, "ngsi10 unsubscribe request", SemWriteOp, &reqSemTaken);
LM_T(LmtMongo, ("Unsubscribe Context"));
/* No matter if success or failure, the subscriptionId in the response is always the one
* in the request */
responseP->subscriptionId = requestP->subscriptionId;
if (responseP->subscriptionId.get() == "")
{
responseP->statusCode.fill(SccContextElementNotFound);
LM_W(("Bad Input (no subscriptionId)"));
return SccOk;
}
LM_T(LmtMongo, ("findOne() in '%s' collection _id '%s'}", getSubscribeContextCollectionName(tenant).c_str(),
requestP->subscriptionId.get().c_str()));
/* Look for document */
connection = getMongoConnection();
try
{
OID id = OID(requestP->subscriptionId.get());
sub = connection->findOne(getSubscribeContextCollectionName(tenant).c_str(), BSON("_id" << id));
releaseMongoConnection(connection);
LM_I(("Database Operation Successful (findOne _id: %s)", id.toString().c_str()));
}
catch (const AssertionException &e)
{
releaseMongoConnection(connection);
reqSemGive(__FUNCTION__, "ngsi10 unsubscribe request (mongo assertion exception)", reqSemTaken);
//
// This happens when OID format is wrong
// FIXME: this checking should be done at parsing stage, without progressing to
// mongoBackend. By the moment we can live this here, but we should remove in the future
// (old issue #95)
//
responseP->statusCode.fill(SccContextElementNotFound);
LM_W(("Bad Input (invalid OID format)"));
return SccOk;
}
catch (const DBException &e)
{
releaseMongoConnection(connection);
reqSemGive(__FUNCTION__, "ngsi10 unsubscribe request (mongo db exception)", reqSemTaken);
responseP->statusCode.fill(SccReceiverInternalError,
std::string("collection: ") + getSubscribeContextCollectionName(tenant).c_str() +
" - findOne() _id: " + requestP->subscriptionId.get() +
" - exception: " + e.what());
LM_E(("Database Error (%s)", responseP->statusCode.details.c_str()));
return SccOk;
}
catch (...)
{
releaseMongoConnection(connection);
reqSemGive(__FUNCTION__, "ngsi10 unsubscribe request (mongo generic exception)", reqSemTaken);
responseP->statusCode.fill(SccReceiverInternalError,
std::string("collection: ") + getSubscribeContextCollectionName(tenant).c_str() +
" - findOne() _id: " + requestP->subscriptionId.get() +
" - exception: " + "generic");
LM_E(("Database Error (%s)", responseP->statusCode.details.c_str()));
return SccOk;
}
if (sub.isEmpty())
{
reqSemGive(__FUNCTION__, "ngsi10 unsubscribe request (no subscriptions found)", reqSemTaken);
responseP->statusCode.fill(SccContextElementNotFound, std::string("subscriptionId: /") + requestP->subscriptionId.get() + "/");
return SccOk;
}
/* Remove document in MongoDB */
// FIXME: I will prefer to do the find and remove in a single operation. Is the some similar
// to findAndModify for this?
LM_T(LmtMongo, ("remove() in '%s' collection _id '%s'}", getSubscribeContextCollectionName(tenant).c_str(),
requestP->subscriptionId.get().c_str()));
connection = getMongoConnection();
try
{
connection->remove(getSubscribeContextCollectionName(tenant).c_str(), BSON("_id" << OID(requestP->subscriptionId.get())));
releaseMongoConnection(connection);
LM_I(("Database Operation Successful (remove _id: %s)", requestP->subscriptionId.get().c_str()));
}
catch (const DBException &e)
{
releaseMongoConnection(connection);
reqSemGive(__FUNCTION__, "ngsi10 unsubscribe request (mongo db exception)", reqSemTaken);
responseP->statusCode.fill(SccReceiverInternalError,
std::string("collection: ") + getSubscribeContextCollectionName(tenant).c_str() +
" - remove() _id: " + requestP->subscriptionId.get().c_str() +
" - exception: " + e.what());
LM_E(("Database Error (%s)", responseP->statusCode.details.c_str()));
return SccOk;
}
catch (...)
{
releaseMongoConnection(connection);
reqSemGive(__FUNCTION__, "ngsi10 unsubscribe request (mongo generic exception)", reqSemTaken);
responseP->statusCode.fill(SccReceiverInternalError,
std::string("collection: ") + getSubscribeContextCollectionName(tenant).c_str() +
" - remove() _id: " + requestP->subscriptionId.get().c_str() +
" - exception: " + "generic");
LM_E(("Database Error (%s)", responseP->statusCode.details.c_str()));
return SccOk;
}
/* Destroy any previous ONTIMEINTERVAL thread */
getNotifier()->destroyOntimeIntervalThreads(requestP->subscriptionId.get());
responseP->statusCode.fill(SccOk);
reqSemGive(__FUNCTION__, "ngsi10 unsubscribe request", reqSemTaken);
//
// Removing subscription from cache
//
subCache->remove(tenant, "", requestP->subscriptionId.get());
return SccOk;
}
/* ****************************************************************************
*
* mongoSubscribeContext -
*/
HttpStatusCode mongoSubscribeContext
(
SubscribeContextRequest* requestP,
SubscribeContextResponse* responseP,
const std::string& tenant,
std::map<std::string, std::string>& uriParam,
const std::string& xauthToken,
const std::vector<std::string>& servicePathV
)
{
const std::string notifyFormatAsString = uriParam[URI_PARAM_NOTIFY_FORMAT];
Format notifyFormat = stringToFormat(notifyFormatAsString);
std::string servicePath = (servicePathV.size() == 0)? "" : servicePathV[0];
bool reqSemTaken = false;
LM_T(LmtMongo, ("Subscribe Context Request: notifications sent in '%s' format", notifyFormatAsString.c_str()));
reqSemTake(__FUNCTION__, "ngsi10 subscribe request", SemWriteOp, &reqSemTaken);
//
// Calculate expiration (using the current time and the duration field in the request).
// If expiration is not present, use a default value
//
long long expiration = -1;
if (requestP->expires > 0)
{
expiration = requestP->expires;
}
else
{
if (requestP->duration.isEmpty())
{
requestP->duration.set(DEFAULT_DURATION);
}
expiration = getCurrentTime() + requestP->duration.parse();
}
LM_T(LmtMongo, ("Subscription expiration: %lu", expiration));
/* Create the mongoDB subscription document */
BSONObjBuilder sub;
OID oid;
oid.init();
sub.append("_id", oid);
sub.append(CSUB_EXPIRATION, expiration);
sub.append(CSUB_REFERENCE, requestP->reference.get());
/* Throttling */
long long throttling = 0;
if (requestP->throttling.seconds != -1)
{
throttling = (long long) requestP->throttling.seconds;
sub.append(CSUB_THROTTLING, throttling);
}
else if (!requestP->throttling.isEmpty())
{
throttling = (long long) requestP->throttling.parse();
sub.append(CSUB_THROTTLING, throttling);
}
if (servicePath != "")
{
sub.append(CSUB_SERVICE_PATH, servicePath);
}
/* Build entities array */
BSONArrayBuilder entities;
for (unsigned int ix = 0; ix < requestP->entityIdVector.size(); ++ix)
{
EntityId* en = requestP->entityIdVector[ix];
if (en->type == "")
{
entities.append(BSON(CSUB_ENTITY_ID << en->id <<
CSUB_ENTITY_ISPATTERN << en->isPattern));
}
else
{
entities.append(BSON(CSUB_ENTITY_ID << en->id <<
CSUB_ENTITY_TYPE << en->type <<
CSUB_ENTITY_ISPATTERN << en->isPattern));
}
}
sub.append(CSUB_ENTITIES, entities.arr());
/* Build attributes array */
BSONArrayBuilder attrs;
for (unsigned int ix = 0; ix < requestP->attributeList.size(); ++ix) {
attrs.append(requestP->attributeList[ix]);
}
sub.append(CSUB_ATTRS, attrs.arr());
/* Build conditions array (including side-effect notifications and threads creation) */
bool notificationDone = false;
BSONArray conds = processConditionVector(&requestP->notifyConditionVector,
requestP->entityIdVector,
requestP->attributeList, oid.toString(),
requestP->reference.get(),
¬ificationDone,
notifyFormat,
tenant,
xauthToken,
servicePathV,
requestP->expression.q);
sub.append(CSUB_CONDITIONS, conds);
/* Build expression */
BSONObjBuilder expression;
expression << CSUB_EXPR_Q << requestP->expression.q
<< CSUB_EXPR_GEOM << requestP->expression.geometry
<< CSUB_EXPR_COORDS << requestP->expression.coords
<< CSUB_EXPR_GEOREL << requestP->expression.georel;
sub.append(CSUB_EXPR, expression.obj());
/* Last notification */
long long lastNotificationTime = 0;
if (notificationDone)
{
lastNotificationTime = (long long) getCurrentTime();
sub.append(CSUB_LASTNOTIFICATION, lastNotificationTime);
sub.append(CSUB_COUNT, 1);
}
/* Adding format to use in notifications */
sub.append(CSUB_FORMAT, notifyFormatAsString);
/* Insert document in database */
std::string err;
if (!collectionInsert(getSubscribeContextCollectionName(tenant), sub.obj(), &err))
{
reqSemGive(__FUNCTION__, "ngsi10 subscribe request (mongo db exception)", reqSemTaken);
responseP->subscribeError.errorCode.fill(SccReceiverInternalError, err);
return SccOk;
}
//
// 3. Create Subscription for the cache
//
std::string oidString = oid.toString();
LM_T(LmtSubCache, ("inserting a new sub in cache (%s)", oidString.c_str()));
cacheSemTake(__FUNCTION__, "Inserting subscription in cache");
subCacheItemInsert(tenant.c_str(),
servicePath.c_str(),
requestP,
oidString.c_str(),
expiration,
throttling,
notifyFormat,
notificationDone,
lastNotificationTime,
requestP->expression.q,
requestP->expression.geometry,
requestP->expression.coords,
requestP->expression.georel);
cacheSemGive(__FUNCTION__, "Inserting subscription in cache");
reqSemGive(__FUNCTION__, "ngsi10 subscribe request", reqSemTaken);
/* Fill int the response element */
responseP->subscribeResponse.duration = requestP->duration;
responseP->subscribeResponse.subscriptionId.set(oid.toString());
responseP->subscribeResponse.throttling = requestP->throttling;
return SccOk;
}
