void
ReplicaCache::subscribe(const ReplicaObserverPrx& observer)
{
try
{
Lock sync(*this);
InternalRegistryPrxSeq replicas;
for(map<string, ReplicaEntryPtr>::const_iterator p = _entries.begin(); p != _entries.end(); ++p)
{
replicas.push_back(p->second->getProxy());
}
IceStorm::QoS qos;
qos["reliability"] = "ordered";
Ice::ObjectPrx publisher = _topic->subscribeAndGetPublisher(qos, observer->ice_twoway());
ReplicaObserverPrx::uncheckedCast(publisher)->replicaInit(replicas);
}
catch(const Ice::ConnectionRefusedException&)
{
// The replica is being shutdown.
}
catch(const Ice::LocalException& ex)
{
TraceLevelsPtr traceLevels = getTraceLevels();
if(traceLevels)
{
Ice::Warning out(traceLevels->logger);
out << "unexpected exception while subscribing observer from replica observer topic:\n" << ex;
}
}
}
InternalRegistryPrxSeq
InternalRegistryI::getReplicas(const Ice::Current&) const
{
InternalRegistryPrxSeq replicas;
Ice::ObjectProxySeq proxies = _database->getObjectsByType(InternalRegistry::ice_staticId());
for(Ice::ObjectProxySeq::const_iterator p = proxies.begin(); p != proxies.end(); ++p)
{
replicas.push_back(InternalRegistryPrx::uncheckedCast(*p));
}
return replicas;
}
void
NodeSessionManager::replicaInit(const InternalRegistryPrxSeq& replicas)
{
Lock sync(*this);
if(_destroyed)
{
return;
}
//
// Initialize the set of replicas known by the master.
//
_replicas.clear();
for(InternalRegistryPrxSeq::const_iterator p = replicas.begin(); p != replicas.end(); ++p)
{
_replicas.insert((*p)->ice_getIdentity());
addReplicaSession(*p)->tryCreateSession(false);
}
}
bool
RegistryI::start(bool nowarn)
{
assert(_communicator);
PropertiesPtr properties = _communicator->getProperties();
//
// Initialize the database environment.
//
string dbPath = properties->getProperty("IceGrid.Registry.Data");
if(dbPath.empty())
{
Error out(_communicator->getLogger());
out << "property `IceGrid.Registry.Data' is not set";
return false;
}
else
{
struct stat filestat;
if(stat(dbPath.c_str(), &filestat) != 0 || !S_ISDIR(filestat.st_mode))
{
Error out(_communicator->getLogger());
SyscallException ex(__FILE__, __LINE__);
ex.error = getSystemErrno();
out << "property `IceGrid.Registry.Data' is set to an invalid path:\n" << ex;
return false;
}
}
//
// Check that required properties are set and valid.
//
if(properties->getProperty("IceGrid.Registry.Client.Endpoints").empty())
{
Error out(_communicator->getLogger());
out << "property `IceGrid.Registry.Client.Endpoints' is not set";
return false;
}
if(properties->getProperty("IceGrid.Registry.Server.Endpoints").empty())
{
Error out(_communicator->getLogger());
out << "property `IceGrid.Registry.Server.Endpoints' is not set";
return false;
}
if(properties->getProperty("IceGrid.Registry.Internal.Endpoints").empty())
{
Error out(_communicator->getLogger());
out << "property `IceGrid.Registry.Internal.Endpoints' is not set";
return false;
}
if(!properties->getProperty("IceGrid.Registry.SessionManager.Endpoints").empty())
{
if(!nowarn)
{
Warning out(_communicator->getLogger());
out << "session manager endpoints `IceGrid.Registry.SessionManager.Endpoints' enabled";
}
}
properties->setProperty("Ice.PrintProcessId", "0");
properties->setProperty("Ice.ServerIdleTime", "0");
properties->setProperty("IceGrid.Registry.Client.AdapterId", "");
properties->setProperty("IceGrid.Registry.Server.AdapterId", "");
properties->setProperty("IceGrid.Registry.SessionManager.AdapterId", "");
properties->setProperty("IceGrid.Registry.Internal.AdapterId", "");
setupThreadPool(properties, "Ice.ThreadPool.Client", 1, 100);
setupThreadPool(properties, "IceGrid.Registry.Client.ThreadPool", 1, 10);
setupThreadPool(properties, "IceGrid.Registry.Server.ThreadPool", 1, 10);
setupThreadPool(properties, "IceGrid.Registry.SessionManager.ThreadPool", 1, 10);
setupThreadPool(properties, "IceGrid.Registry.Internal.ThreadPool", 1, 100);
_replicaName = properties->getPropertyWithDefault("IceGrid.Registry.ReplicaName", "Master");
_master = _replicaName == "Master";
_sessionTimeout = properties->getPropertyAsIntWithDefault("IceGrid.Registry.SessionTimeout", 30);
//
// Get the instance name
//
if(_master)
{
_instanceName = properties->getProperty("IceGrid.InstanceName");
if(_instanceName.empty())
{
if(_communicator->getDefaultLocator())
{
_instanceName = _communicator->getDefaultLocator()->ice_getIdentity().category;
}
else
{
_instanceName = "IceGrid";
}
}
}
else
{
if(properties->getProperty("Ice.Default.Locator").empty())
{
Error out(_communicator->getLogger());
out << "property `Ice.Default.Locator' is not set";
return false;
}
_instanceName = _communicator->getDefaultLocator()->ice_getIdentity().category;
}
//
// Ensure that nothing is running on this port. This is also
// useful to ensure that we don't run twice the same instance of
// the service too (which would cause the database environment of
// the already running instance to be "corrupted".)
//
try
{
string endpoints = properties->getProperty("IceGrid.Registry.Client.Endpoints");
string strPrx = _instanceName + "/Locator:" + endpoints;
_communicator->stringToProxy(strPrx)->ice_timeout(5000)->ice_ping();
Error out(_communicator->getLogger());
out << "an IceGrid registry is already running and listening on the client endpoints `" << endpoints << "'";
return false;
}
catch(const Ice::LocalException&)
{
}
properties->setProperty("Freeze.DbEnv.Registry.DbHome", dbPath);
properties->setProperty("Freeze.DbEnv.Registry.DbPrivate", "0");
//
// Create the reaper thread.
//
_reaper = new ReapThread();
_reaper->start();
//
// Create the internal registry object adapter.
//
ObjectAdapterPtr registryAdapter = _communicator->createObjectAdapter("IceGrid.Registry.Internal");
registryAdapter->activate();
//
// Create the internal IceStorm service.
//
Identity registryTopicManagerId;
registryTopicManagerId.category = _instanceName;
registryTopicManagerId.name = "RegistryTopicManager";
_iceStorm = IceStorm::Service::create(_communicator,
registryAdapter,
registryAdapter,
"IceGrid.Registry",
registryTopicManagerId,
"Registry");
const IceStorm::TopicManagerPrx topicManager = _iceStorm->getTopicManager();
//
// Create the registry database.
//
_database = new Database(registryAdapter, topicManager, _instanceName, _traceLevels, getInfo());
_wellKnownObjects = new WellKnownObjectsManager(_database);
//
// Get the saved replica/node proxies and remove them from the
// database.
//
Ice::ObjectProxySeq proxies;
Ice::ObjectProxySeq::const_iterator p;
NodePrxSeq nodes;
proxies = _database->getInternalObjectsByType(Node::ice_staticId());
for(p = proxies.begin(); p != proxies.end(); ++p)
{
nodes.push_back(NodePrx::uncheckedCast(*p));
}
InternalRegistryPrxSeq replicas;
proxies = _database->getObjectsByType(InternalRegistry::ice_staticId());
for(p = proxies.begin(); p != proxies.end(); ++p)
{
replicas.push_back(InternalRegistryPrx::uncheckedCast(*p));
}
//
// NOTE: The internal registry object must be added only once the
// node/replica proxies are retrieved and removed from the
// database. Otherwise, if some replica/node register as soon as
// the internal registry is setup we might clear valid proxies.
//
InternalRegistryPrx internalRegistry = setupInternalRegistry(registryAdapter);
if(_master)
{
nodes = registerReplicas(internalRegistry, replicas, nodes);
registerNodes(internalRegistry, nodes);
}
else
{
InternalReplicaInfoPtr info = _platform.getInternalReplicaInfo();
_session.create(_replicaName, info, _database, _wellKnownObjects, internalRegistry);
registerNodes(internalRegistry, _session.getNodes(nodes));
}
ObjectAdapterPtr serverAdapter = _communicator->createObjectAdapter("IceGrid.Registry.Server");
_clientAdapter = _communicator->createObjectAdapter("IceGrid.Registry.Client");
ObjectAdapterPtr sessionManagerAdapter;
if(!properties->getProperty("IceGrid.Registry.SessionManager.Endpoints").empty())
{
sessionManagerAdapter = _communicator->createObjectAdapter("IceGrid.Registry.SessionManager");
}
Ice::Identity dummy;
dummy.name = "dummy";
_wellKnownObjects->addEndpoint("Client", _clientAdapter->createDirectProxy(dummy));
_wellKnownObjects->addEndpoint("Server", serverAdapter->createDirectProxy(dummy));
if(sessionManagerAdapter)
{
_wellKnownObjects->addEndpoint("SessionManager", sessionManagerAdapter->createDirectProxy(dummy));
}
_wellKnownObjects->addEndpoint("Internal", registryAdapter->createDirectProxy(dummy));
setupNullPermissionsVerifier(registryAdapter);
if(!setupUserAccountMapper(registryAdapter))
{
return false;
}
QueryPrx query = setupQuery(_clientAdapter);
RegistryPrx registry = setupRegistry(_clientAdapter);
Ice::LocatorRegistryPrx locatorRegistry = setupLocatorRegistry(serverAdapter);
LocatorPrx internalLocator = setupLocator(_clientAdapter, registryAdapter, locatorRegistry, registry, query);
//
// Add a default servant locator to the client object adapter. The
// default servant ensure that request on session objects are from
// the same connection as the connection that created the session.
//
_sessionServantLocator = new SessionServantLocatorI(_clientAdapter, _instanceName);
_clientAdapter->addServantLocator(_sessionServantLocator, "");
setupClientSessionFactory(registryAdapter, sessionManagerAdapter, internalLocator, nowarn);
setupAdminSessionFactory(registryAdapter, sessionManagerAdapter, internalLocator, nowarn);
_wellKnownObjects->finish();
if(_master)
{
_wellKnownObjects->registerAll();
}
else
{
_session.registerAllWellKnownObjects();
}
//
// We are ready to go!
//
serverAdapter->activate();
_clientAdapter->activate();
if(sessionManagerAdapter)
{
sessionManagerAdapter->activate();
}
return true;
}
bool
RegistryI::startImpl()
{
assert(_communicator);
PropertiesPtr properties = _communicator->getProperties();
//
// Check that required properties are set and valid.
//
if(properties->getProperty("IceGrid.Registry.Client.Endpoints").empty())
{
Error out(_communicator->getLogger());
out << "property `IceGrid.Registry.Client.Endpoints' is not set";
return false;
}
if(properties->getProperty("IceGrid.Registry.Server.Endpoints").empty())
{
Error out(_communicator->getLogger());
out << "property `IceGrid.Registry.Server.Endpoints' is not set";
return false;
}
if(properties->getProperty("IceGrid.Registry.Internal.Endpoints").empty())
{
Error out(_communicator->getLogger());
out << "property `IceGrid.Registry.Internal.Endpoints' is not set";
return false;
}
if(!properties->getProperty("IceGrid.Registry.SessionManager.Endpoints").empty())
{
if(!_nowarn)
{
Warning out(_communicator->getLogger());
out << "session manager endpoints `IceGrid.Registry.SessionManager.Endpoints' enabled";
if(properties->getPropertyAsInt("IceGrid.Registry.SessionFilters") == 0)
{
out << " (with Glacier2 filters disabled)";
}
}
}
if(!properties->getProperty("IceGrid.Registry.AdminSessionManager.Endpoints").empty())
{
if(!_nowarn)
{
Warning out(_communicator->getLogger());
out << "administrative session manager endpoints `IceGrid.Registry.AdminSessionManager.Endpoints' enabled";
if(properties->getPropertyAsInt("IceGrid.Registry.AdminSessionFilters") == 0)
{
out << " (with Glacier2 filters disabled)";
}
}
}
properties->setProperty("Ice.PrintProcessId", "0");
properties->setProperty("Ice.ServerIdleTime", "0");
properties->setProperty("IceGrid.Registry.Client.AdapterId", "");
properties->setProperty("IceGrid.Registry.Server.AdapterId", "");
properties->setProperty("IceGrid.Registry.SessionManager.AdapterId", "");
properties->setProperty("IceGrid.Registry.Internal.AdapterId", "");
setupThreadPool(properties, "IceGrid.Registry.Client.ThreadPool", 1, 10);
setupThreadPool(properties, "IceGrid.Registry.Server.ThreadPool", 1, 10, true); // Serialize for admin callbacks
setupThreadPool(properties, "IceGrid.Registry.SessionManager.ThreadPool", 1, 10);
setupThreadPool(properties, "IceGrid.Registry.Internal.ThreadPool", 1, 100);
_replicaName = properties->getPropertyWithDefault("IceGrid.Registry.ReplicaName", "Master");
_master = _replicaName == "Master";
_sessionTimeout = properties->getPropertyAsIntWithDefault("IceGrid.Registry.SessionTimeout", 30);
if(!_master && properties->getProperty("Ice.Default.Locator").empty())
{
if(properties->getProperty("Ice.Default.Locator").empty())
{
Error out(_communicator->getLogger());
out << "property `Ice.Default.Locator' is not set";
return false;
}
}
//
// Get the instance name
//
if(_master)
{
_instanceName = properties->getProperty("IceGrid.InstanceName");
if(_instanceName.empty())
{
if(_communicator->getDefaultLocator())
{
_instanceName = _communicator->getDefaultLocator()->ice_getIdentity().category;
}
else
{
_instanceName = "IceGrid";
}
}
}
else
{
_instanceName = _communicator->getDefaultLocator()->ice_getIdentity().category;
}
//
// Ensure that nothing is running on this port. This is also
// useful to ensure that we don't run twice the same instance of
// the service too (which would cause the database environment of
// the already running instance to be "corrupted".)
//
try
{
string endpoints = properties->getProperty("IceGrid.Registry.Client.Endpoints");
string strPrx = _instanceName + "/Locator:" + endpoints;
_communicator->stringToProxy(strPrx)->ice_timeout(5000)->ice_ping();
Error out(_communicator->getLogger());
out << "an IceGrid registry is already running and listening on the client endpoints `" << endpoints << "'";
return false;
}
catch(const Ice::LocalException&)
{
}
//
// Create the reaper thread.
//
_reaper = new ReapThread();
_reaper->start();
//
// Create the internal registry object adapter.
//
ObjectAdapterPtr registryAdapter = _communicator->createObjectAdapter("IceGrid.Registry.Internal");
registryAdapter->activate();
//
// Create the internal IceStorm service.
//
Identity registryTopicManagerId;
registryTopicManagerId.category = _instanceName;
registryTopicManagerId.name = "RegistryTopicManager";
_iceStorm = IceStormInternal::Service::create(_communicator,
registryAdapter,
registryAdapter,
"IceGrid.Registry",
registryTopicManagerId,
"Registry");
const IceStorm::TopicManagerPrx topicManager = _iceStorm->getTopicManager();
//
// Create the registry database.
//
DatabasePluginPtr plugin;
try
{
plugin = DatabasePluginPtr::dynamicCast(_communicator->getPluginManager()->getPlugin("DB"));
}
catch(const NotRegisteredException&)
{
}
if(!plugin)
{
Error out(_communicator->getLogger());
out << "no database plugin configured with `Ice.Plugin.DB' or plugin is not a database plugin";
return false;
}
_database = new Database(registryAdapter, topicManager, _instanceName, _traceLevels, getInfo(), plugin, _readonly);
_wellKnownObjects = new WellKnownObjectsManager(_database);
//
// Get the saved replica/node proxies.
//
ObjectProxySeq proxies;
ObjectProxySeq::const_iterator p;
NodePrxSeq nodes;
proxies = _database->getInternalObjectsByType(Node::ice_staticId());
for(p = proxies.begin(); p != proxies.end(); ++p)
{
nodes.push_back(NodePrx::uncheckedCast(*p));
}
InternalRegistryPrxSeq replicas;
proxies = _database->getObjectsByType(InternalRegistry::ice_staticId());
for(p = proxies.begin(); p != proxies.end(); ++p)
{
replicas.push_back(InternalRegistryPrx::uncheckedCast(*p));
}
//
// NOTE: The internal registry object must be added only once the
// node/replica proxies are retrieved. Otherwise, if some
// replica/node register as soon as the internal registry is setup
// we might clear valid proxies.
//
InternalRegistryPrx internalRegistry = setupInternalRegistry(registryAdapter);
if(_master)
{
nodes = registerReplicas(internalRegistry, replicas, nodes);
registerNodes(internalRegistry, nodes);
}
else
{
InternalReplicaInfoPtr info = _platform.getInternalReplicaInfo();
_session.create(_replicaName, info, _database, _wellKnownObjects, internalRegistry);
registerNodes(internalRegistry, _session.getNodes(nodes));
}
_serverAdapter = _communicator->createObjectAdapter("IceGrid.Registry.Server");
_clientAdapter = _communicator->createObjectAdapter("IceGrid.Registry.Client");
Ice::Identity dummy;
dummy.name = "dummy";
_wellKnownObjects->addEndpoint("Client", _clientAdapter->createDirectProxy(dummy));
_wellKnownObjects->addEndpoint("Server", _serverAdapter->createDirectProxy(dummy));
_wellKnownObjects->addEndpoint("Internal", registryAdapter->createDirectProxy(dummy));
setupNullPermissionsVerifier(registryAdapter);
if(!setupUserAccountMapper(registryAdapter))
{
return false;
}
QueryPrx query = setupQuery(_clientAdapter);
RegistryPrx registry = setupRegistry(_clientAdapter);
Ice::LocatorRegistryPrx locatorRegistry = setupLocatorRegistry(_serverAdapter);
LocatorPrx internalLocator = setupLocator(_clientAdapter, registryAdapter, locatorRegistry, registry, query);
//
// Create the session servant manager. The session servant manager is responsible
// for managing sessions servants and to ensure that session servants are only
// accessed by the connection that created the session. The session servant manager
// also takes care of providing the router servant for server admin objects.
//
ObjectPtr serverAdminRouter = new RegistryServerAdminRouter(_database);
AdminCallbackRouterPtr adminCallbackRouter = new AdminCallbackRouter;
_servantManager = new SessionServantManager(_clientAdapter, _instanceName, true, getServerAdminCategory(),
serverAdminRouter, adminCallbackRouter);
_clientAdapter->addServantLocator(_servantManager, "");
_serverAdapter->addServantLocator(new DefaultServantLocator(adminCallbackRouter), "");
ObjectAdapterPtr sessionAdpt = setupClientSessionFactory(registryAdapter, internalLocator);
ObjectAdapterPtr admSessionAdpt = setupAdminSessionFactory(registryAdapter, serverAdminRouter, internalLocator);
_wellKnownObjects->finish();
if(_master)
{
_wellKnownObjects->registerAll();
}
else
{
_session.registerAllWellKnownObjects();
}
//
// We are ready to go!
//
_serverAdapter->activate();
_clientAdapter->activate();
if(sessionAdpt)
{
sessionAdpt->activate();
}
if(admSessionAdpt)
{
admSessionAdpt->activate();
}
return true;
}
void
NodeSessionManager::createdSession(const NodeSessionPrx& session)
{
bool activated;
{
Lock sync(*this);
activated = _activated;
}
//
// Synchronize the servers if the session is active and if the
// node adapter has been activated (otherwise, the servers will be
// synced after the node adapter activation, see activate()).
//
// We also set the replica observer to receive notifications of
// replica addition/removal.
//
if(session && activated)
{
try
{
session->setReplicaObserver(_node->getProxy());
syncServers(session);
}
catch(const Ice::LocalException&)
{
}
return;
}
//
// If there's no master session or if the node adapter isn't
// activated yet, we retrieve a list of the replicas either from
// the master or from the known replicas (the ones configured with
// Ice.Default.Locator) and we try to establish connections to
// each of the replicas.
//
InternalRegistryPrxSeq replicas;
if(session)
{
assert(!activated); // The node adapter isn't activated yet so
// we're not subscribed yet to the replica
// observer topic.
try
{
replicas = _thread->getRegistry()->getReplicas();
}
catch(const Ice::LocalException&)
{
}
}
else
{
vector<Ice::AsyncResultPtr> results;
for(vector<QueryPrx>::const_iterator q = _queryObjects.begin(); q != _queryObjects.end(); ++q)
{
results.push_back((*q)->begin_findAllObjectsByType(InternalRegistry::ice_staticId()));
}
map<Ice::Identity, Ice::ObjectPrx> proxies;
for(vector<Ice::AsyncResultPtr>::const_iterator p = results.begin(); p != results.end(); ++p)
{
QueryPrx query = QueryPrx::uncheckedCast((*p)->getProxy());
if(isDestroyed())
{
return;
}
try
{
Ice::ObjectProxySeq prxs = query->end_findAllObjectsByType(*p);
for(Ice::ObjectProxySeq::const_iterator q = prxs.begin(); q != prxs.end(); ++q)
{
//
// NOTE: We might override a good proxy here! We could improve this to make
// sure that we don't override the proxy for replica N if that proxy was
// obtained from replica N.
//
proxies[(*q)->ice_getIdentity()] = *q;
}
}
catch(const Ice::LocalException&)
{
// IGNORE
}
}
for(map<Ice::Identity, Ice::ObjectPrx>::const_iterator q = proxies.begin(); q != proxies.end(); ++q)
{
replicas.push_back(InternalRegistryPrx::uncheckedCast(q->second));
}
}
vector<NodeSessionKeepAliveThreadPtr> sessions;
{
Lock sync(*this);
if(_destroyed)
{
return;
}
//
// If the node adapter was activated since we last check, we don't need
// to initialize the replicas here, it will be done by replicaInit().
//
if(!session || !_activated)
{
_replicas.clear();
for(InternalRegistryPrxSeq::const_iterator p = replicas.begin(); p != replicas.end(); ++p)
{
if((*p)->ice_getIdentity() != _master->ice_getIdentity())
{
_replicas.insert((*p)->ice_getIdentity());
NodeSessionKeepAliveThreadPtr session = addReplicaSession(*p);
session->tryCreateSession(false);
sessions.push_back(session);
}
}
}
}
//
// Wait for the creation. It's important to wait to ensure that
// the replica sessions are created before the node adapter is
// activated.
//
IceUtil::Time before = IceUtil::Time::now();
for(vector<NodeSessionKeepAliveThreadPtr>::const_iterator p = sessions.begin(); p != sessions.end(); ++p)
{
if(isDestroyed())
{
return;
}
IceUtil::Time timeout = IceUtil::Time::seconds(5) - (IceUtil::Time::now() - before);
if(timeout <= IceUtil::Time())
{
break;
}
(*p)->tryCreateSession(true, timeout);
}
}