Пример #1
0
void
ServiceI::start(
    const string& name,
    const CommunicatorPtr& communicator,
    const StringSeq& /*args*/)
{
    PropertiesPtr properties = communicator->getProperties();

    validateProperties(name, properties, communicator->getLogger());

    int id = properties->getPropertyAsIntWithDefault(name + ".NodeId", -1);

    // If we are using a replicated deployment and if the topic
    // manager thread pool max size is not set then ensure it is set
    // to some suitably high number. This ensures no deadlocks in the
    // replicated case due to call forwarding from replicas to
    // coordinators.
    if(id != -1 && properties->getProperty(name + ".TopicManager.ThreadPool.SizeMax").empty())
    {
        properties->setProperty(name + ".TopicManager.ThreadPool.SizeMax", "100");
    }

    Ice::ObjectAdapterPtr topicAdapter = communicator->createObjectAdapter(name + ".TopicManager");
    Ice::ObjectAdapterPtr publishAdapter = communicator->createObjectAdapter(name + ".Publish");

    //
    // We use the name of the service for the name of the database environment.
    //
    string instanceName = properties->getPropertyWithDefault(name + ".InstanceName", "IceStorm");
    Identity topicManagerId;
    topicManagerId.category = instanceName;
    topicManagerId.name = "TopicManager";

    if(properties->getPropertyAsIntWithDefault(name+ ".Transient", 0) > 0)
    {
        _instance = new Instance(instanceName, name, communicator, publishAdapter, topicAdapter, 0);
        try
        {
            TransientTopicManagerImplPtr manager = new TransientTopicManagerImpl(_instance);
            _managerProxy = TopicManagerPrx::uncheckedCast(topicAdapter->add(manager, topicManagerId));
        }
        catch(const Ice::Exception& ex)
        {
            _instance = 0;

            LoggerOutputBase s;
            s << "exception while starting IceStorm service " << name << ":\n";
            s << ex;

            IceBox::FailureException e(__FILE__, __LINE__);
            e.reason = s.str();
            throw e;
        }
        topicAdapter->activate();
        publishAdapter->activate();
        return;
    }

    if(id == -1) // No replication.
    {
        _instance = new Instance(instanceName, name, communicator, publishAdapter, topicAdapter);

        try
        {
            _manager = new TopicManagerImpl(_instance);
            _managerProxy = TopicManagerPrx::uncheckedCast(topicAdapter->add(_manager->getServant(), topicManagerId));
        }
        catch(const Ice::Exception& ex)
        {
            _instance = 0;

            LoggerOutputBase s;
            s << "exception while starting IceStorm service " << name << ":\n";
            s << ex;

            IceBox::FailureException e(__FILE__, __LINE__);
            e.reason = s.str();
            throw e;
        }
    }
    else
    {
        // Here we want to create a map of id -> election node
        // proxies.
        map<int, NodePrx> nodes;

        string topicManagerAdapterId = properties->getProperty(name + ".TopicManager.AdapterId");

        // We support two possible deployments. The first is a manual
        // deployment, the second is IceGrid.
        //
        // Here we check for the manual deployment
        const string prefix = name + ".Nodes.";
        Ice::PropertyDict props = properties->getPropertiesForPrefix(prefix);
        if(!props.empty())
        {
            for(Ice::PropertyDict::const_iterator p = props.begin(); p != props.end(); ++p)
            {
                int nodeid = atoi(p->first.substr(prefix.size()).c_str());
                nodes[nodeid] = NodePrx::uncheckedCast(communicator->propertyToProxy(p->first));
            }
        }
        else
        {
            // If adapter id's are defined for the topic manager or
            // node adapters then we consider this an IceGrid based
            // deployment.
            string nodeAdapterId = properties->getProperty(name + ".Node.AdapterId");

            // Validate first that the adapter ids match for the node
            // and the topic manager otherwise some other deployment
            // is being used.
            const string suffix = ".TopicManager";
            if(topicManagerAdapterId.empty() || nodeAdapterId.empty() ||
               topicManagerAdapterId.replace(
                   topicManagerAdapterId.find(suffix), suffix.size(), ".Node") != nodeAdapterId)
            {
                Ice::Error error(communicator->getLogger());
                error << "deployment error: `" << topicManagerAdapterId << "' prefix does not match `"
                      << nodeAdapterId << "'";
                throw IceBox::FailureException(__FILE__, __LINE__, "IceGrid deployment is incorrect");
            }

            // Determine the set of node id and node proxies.
            //
            // This is determined by locating all topic manager
            // replicas, and then working out the node for that
            // replica.
            //
            // We work out the node id by removing the instance
            // name. The node id must follow.
            //
            IceGrid::LocatorPrx locator = IceGrid::LocatorPrx::checkedCast(communicator->getDefaultLocator());
            assert(locator);
            IceGrid::QueryPrx query = locator->getLocalQuery();
            Ice::ObjectProxySeq replicas = query->findAllReplicas(
                communicator->stringToProxy(instanceName + "/TopicManager"));

            for(Ice::ObjectProxySeq::const_iterator p = replicas.begin(); p != replicas.end(); ++p)
            {
                string adapterid = (*p)->ice_getAdapterId();

                // Replace TopicManager with the node endpoint.
                adapterid = adapterid.replace(adapterid.find(suffix), suffix.size(), ".Node");

                // The adapter id must start with the instance name.
                if(adapterid.find(instanceName) != 0)
                {
                    Ice::Error error(communicator->getLogger());
                    error << "deployment error: `" << adapterid << "' does not start with `" << instanceName << "'";
                    throw IceBox::FailureException(__FILE__, __LINE__, "IceGrid deployment is incorrect");
                }

                // The node id follows. We find the first digit (the
                // start of the node id, and then the end of the
                // digits).
                string::size_type start = instanceName.size();
                while(start < adapterid.size() && !IceUtilInternal::isDigit(adapterid[start]))
                {
                    ++start;
                }
                string::size_type end = start;
                while(end < adapterid.size() && IceUtilInternal::isDigit(adapterid[end]))
                {
                    ++end;
                }
                if(start == end)
                {
                    // We must have at least one digit, otherwise there is
                    // some sort of deployment error.
                    Ice::Error error(communicator->getLogger());
                    error << "deployment error: node id does not follow instance name. instance name:"
                          << instanceName << " adapter id: " << adapterid;
                    throw IceBox::FailureException(__FILE__, __LINE__, "IceGrid deployment is incorrect");
                }

                int nodeid = atoi(adapterid.substr(start, end-start).c_str());
                ostringstream os;
                os << "node" << nodeid;
                Ice::Identity id;
                id.category = instanceName;
                id.name = os.str();

                nodes[nodeid] = NodePrx::uncheckedCast((*p)->ice_adapterId(adapterid)->ice_identity(id));
            }
        }

        if(nodes.size() < 3)
        {
            Ice::Error error(communicator->getLogger());
            error << "Replication requires at least 3 Nodes";
            throw IceBox::FailureException(__FILE__, __LINE__, "Replication requires at least 3 Nodes");
        }

        try
        {
            // If the node thread pool size is not set then initialize
            // to the number of nodes + 1 and disable thread pool size
            // warnings.
            if(properties->getProperty(name + ".Node.ThreadPool.Size").empty())
            {
                ostringstream os;
                os << nodes.size() + 1;
                properties->setProperty(name + ".Node.ThreadPool.Size", os.str());
                properties->setProperty(name + ".Node.ThreadPool.SizeWarn", "0");
            }
            if(properties->getProperty(name + ".Node.MessageSizeMax").empty())
            {
                properties->setProperty(name + ".Node.MessageSizeMax", "0"); // No limit on data exchanged internally
            }

            Ice::ObjectAdapterPtr nodeAdapter = communicator->createObjectAdapter(name + ".Node");

            _instance = new Instance(instanceName, name, communicator, publishAdapter, topicAdapter,
                                     nodeAdapter, nodes[id]);
            _instance->observers()->setMajority(static_cast<unsigned int>(nodes.size())/2);

            // Trace replication information.
            TraceLevelsPtr traceLevels = _instance->traceLevels();
            if(traceLevels->election > 0)
            {
                Ice::Trace out(traceLevels->logger, traceLevels->electionCat);
                out << "I am node " << id << "\n";
                for(map<int, NodePrx>::const_iterator p = nodes.begin(); p != nodes.end(); ++p)
                {
                    out << "\tnode: " << p->first << " proxy: " << p->second->ice_toString() << "\n";
                }
            }

            if(topicManagerAdapterId.empty())
            {
                // We're not using an IceGrid deployment. Here we need
                // a proxy which is used to create proxies to the
                // replicas later.
                _managerProxy = TopicManagerPrx::uncheckedCast(topicAdapter->createProxy(topicManagerId));
            }
            else
            {
                // If we're using IceGrid deployment we need to create
                // indirect proxies.
                _managerProxy = TopicManagerPrx::uncheckedCast(topicAdapter->createIndirectProxy(topicManagerId));
            }

            _manager = new TopicManagerImpl(_instance);
            topicAdapter->add(_manager->getServant(), topicManagerId);

            ostringstream os; // The node object identity.
            os << "node" << id;
            Ice::Identity nodeid;
            nodeid.category = instanceName;
            nodeid.name = os.str();

            NodeIPtr node = new NodeI(_instance, _manager, _managerProxy, id, nodes);
            _instance->setNode(node);
            nodeAdapter->add(node, nodeid);
            nodeAdapter->activate();

            node->start();
        }
        catch(const Ice::Exception& ex)
        {
            _instance = 0;

            LoggerOutputBase s;
            s << "exception while starting IceStorm service " << name << ":\n";
            s << ex;

            IceBox::FailureException e(__FILE__, __LINE__);
            e.reason = s.str();
            throw e;
        }
    }

    topicAdapter->add(new FinderI(TopicManagerPrx::uncheckedCast(topicAdapter->createProxy(topicManagerId))),
                      communicator->stringToIdentity("IceStorm/Finder"));

    topicAdapter->activate();
    publishAdapter->activate();
}
Пример #2
0
void
allTests(const Ice::CommunicatorPtr& comm)
{
    IceGrid::RegistryPrx registry = IceGrid::RegistryPrx::checkedCast(
        comm->stringToProxy(comm->getDefaultLocator()->ice_getIdentity().category + "/Registry"));
    test(registry);
    IceGrid::QueryPrx query = IceGrid::QueryPrx::checkedCast(
        comm->stringToProxy(comm->getDefaultLocator()->ice_getIdentity().category + "/Query"));
    test(query);
    AdminSessionPrx session = registry->createAdminSession("foo", "bar");

    session->ice_getConnection()->setACM(registry->getACMTimeout(), IceUtil::None, Ice::HeartbeatAlways);

    AdminPrx admin = session->getAdmin();
    test(admin);

    set<string> serverReplicaIds;
    serverReplicaIds.insert("Server1.ReplicatedAdapter");
    serverReplicaIds.insert("Server2.ReplicatedAdapter");
    serverReplicaIds.insert("Server3.ReplicatedAdapter");
    set<string> svcReplicaIds;
    svcReplicaIds.insert("IceBox1.Service1.Service1");
    svcReplicaIds.insert("IceBox1.Service2.Service2");
    svcReplicaIds.insert("IceBox1.Service3.Service3");

    cout << "testing Query::findAllReplicas... " << flush;
    {
        map<string, string> params;
        params["replicaGroup"] = "RoundRobin";
        params["id"] = "Server1";
        instantiateServer(admin, "Server", "localnode", params);
        params["id"] = "Server2";
        instantiateServer(admin, "Server", "localnode", params);
        params["id"] = "Server3";
        instantiateServer(admin, "Server", "localnode", params);
        
        TestIntfPrx obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("dummy@RoundRobin"));
        Ice::ObjectProxySeq objs = query->findAllReplicas(obj);
        test(objs.size() == 3);
        test(serverReplicaIds.find(objs[0]->ice_getAdapterId()) != serverReplicaIds.end());
        test(serverReplicaIds.find(objs[1]->ice_getAdapterId()) != serverReplicaIds.end());
        test(serverReplicaIds.find(objs[2]->ice_getAdapterId()) != serverReplicaIds.end());

        obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("RoundRobin"));
        objs = query->findAllReplicas(obj);
        test(objs.size() == 3);
        test(serverReplicaIds.find(objs[0]->ice_getAdapterId()) != serverReplicaIds.end());
        test(serverReplicaIds.find(objs[1]->ice_getAdapterId()) != serverReplicaIds.end());
        test(serverReplicaIds.find(objs[2]->ice_getAdapterId()) != serverReplicaIds.end());
        
        obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("dummy@dummy"));
        objs = query->findAllReplicas(obj);
        test(objs.empty());

        obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("*****@*****.**"));
        objs = query->findAllReplicas(obj);
        test(objs.empty());

        obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("dummy:tcp"));
        objs = query->findAllReplicas(obj);
        test(objs.empty());

        obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("dummy@Ordered"));
        objs = query->findAllReplicas(obj);
        test(objs.empty());
        
        removeServer(admin, "Server1");
        removeServer(admin, "Server2");
        removeServer(admin, "Server3");
    }
    cout << "ok" << endl;

    cout << "testing replication with round-robin load balancing... " << flush;
    {
        map<string, string> params;
        params["replicaGroup"] = "RoundRobin";
        params["id"] = "Server1";
        instantiateServer(admin, "Server", "localnode", params);
        params["id"] = "Server2";
        instantiateServer(admin, "Server", "localnode", params);
        params["id"] = "Server3";
        instantiateServer(admin, "Server", "localnode", params);
        TestIntfPrx obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("RoundRobin"));
        obj = TestIntfPrx::uncheckedCast(obj->ice_locatorCacheTimeout(0));
        obj = TestIntfPrx::uncheckedCast(obj->ice_connectionCached(false));
        try
        {
            test(obj->getReplicaIdAndShutdown() == "Server1.ReplicatedAdapter");
            test(obj->getReplicaIdAndShutdown() == "Server2.ReplicatedAdapter");
            test(obj->getReplicaIdAndShutdown() == "Server3.ReplicatedAdapter");        

            admin->enableServer("Server1", false);
            admin->enableServer("Server2", false);
            admin->enableServer("Server3", false);

            try
            {
                obj->getReplicaId();
                test(false);
            }
            catch(const Ice::NoEndpointException&)
            {
            }

            admin->enableServer("Server1", true);
            admin->enableServer("Server2", true);
            admin->enableServer("Server3", true);

            set<string> adapterIds;
            string previousId;
            while(adapterIds.size() != 3)
            {
                string id = obj->getReplicaId();
                adapterIds.insert(id);

                if(adapterIds.size() == 1)
                {
                    previousId = id;
                }
                else
                {
                    test(previousId != id);
                    previousId = id;
                }
            }
            
            int i;
            for(i = 0; i < 3; i++)
            {
                if(obj->getReplicaId() == "Server3.ReplicatedAdapter")
                {
                    break;
                }
            }
            test(i != 3);

            test(obj->getReplicaId() == "Server1.ReplicatedAdapter");
            test(obj->getReplicaId() == "Server2.ReplicatedAdapter");
            test(obj->getReplicaId() == "Server3.ReplicatedAdapter");

            test(obj->getReplicaIdAndShutdown() == "Server1.ReplicatedAdapter");
            test(obj->getReplicaIdAndShutdown() == "Server2.ReplicatedAdapter");
            test(obj->getReplicaIdAndShutdown() == "Server3.ReplicatedAdapter");
        }
        catch(const Ice::LocalException& ex)
        {
            cerr << ex << endl;
            test(false);
        }

        removeServer(admin, "Server1");
        removeServer(admin, "Server2");
        removeServer(admin, "Server3");
    }
    {
        map<string, string> params;
        params["replicaGroup"] = "RoundRobin";
        params["id"] = "IceBox1";
        instantiateServer(admin, "IceBox", "localnode", params);
        TestIntfPrx obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("RoundRobin"));
        try
        {
            test(obj->getReplicaIdAndShutdown() == "IceBox1.Service1.Service1");
            test(obj->getReplicaIdAndShutdown() == "IceBox1.Service2.Service2");
            test(obj->getReplicaIdAndShutdown() == "IceBox1.Service3.Service3");
        }
        catch(const Ice::LocalException& ex)
        {
            cerr << ex << endl;
            test(false);
        }
        removeServer(admin, "IceBox1");
    }
    cout << "ok" << endl;

    cout << "testing replication with ordered load balancing... " << flush;
    {
        map<string, string> params;
        params["replicaGroup"] = "Ordered";
        params["id"] = "Server1";
        params["priority"] = "3";
        instantiateServer(admin, "Server", "localnode", params);
        params["id"] = "Server2";
        params["priority"] = "1";
        instantiateServer(admin, "Server", "localnode", params);
        params["id"] = "Server3";
        params["priority"] = "2";
        instantiateServer(admin, "Server", "localnode", params);
        TestIntfPrx obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("Ordered"));
        try
        {
            test(obj->getReplicaIdAndShutdown() == "Server2.ReplicatedAdapter");
            admin->enableServer("Server2", false);
            test(obj->getReplicaIdAndShutdown() == "Server3.ReplicatedAdapter");
            admin->enableServer("Server3", false);
            test(obj->getReplicaIdAndShutdown() == "Server1.ReplicatedAdapter");
        }
        catch(const Ice::LocalException& ex)
        {
            cerr << ex << endl;
            test(false);
        }
        removeServer(admin, "Server1");
        removeServer(admin, "Server2");
        removeServer(admin, "Server3");
    }
    {
        map<string, string> params;
        params["replicaGroup"] = "Ordered";
        params["id"] = "IceBox1";
        instantiateServer(admin, "IceBox", "localnode", params);
        TestIntfPrx obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("Ordered"));
        try
        {
            test(obj->getReplicaIdAndShutdown() == "IceBox1.Service3.Service3");
            test(obj->getReplicaIdAndShutdown() == "IceBox1.Service2.Service2");
            test(obj->getReplicaIdAndShutdown() == "IceBox1.Service1.Service1");
        }
        catch(const Ice::LocalException& ex)
        {
            cerr << ex << endl;
            test(false);
        }
        removeServer(admin, "IceBox1");
    }
    cout << "ok" << endl;

    cout << "testing replication with random load balancing... " << flush;
    {
        map<string, string> params;
        params["replicaGroup"] = "Random";
        params["id"] = "Server1";
        instantiateServer(admin, "Server", "localnode", params);
        params["id"] = "Server2";
        instantiateServer(admin, "Server", "localnode", params);
        params["id"] = "Server3";
        instantiateServer(admin, "Server", "localnode", params);
        TestIntfPrx obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("Random"));
        obj = TestIntfPrx::uncheckedCast(obj->ice_locatorCacheTimeout(0));
        obj = TestIntfPrx::uncheckedCast(obj->ice_connectionCached(false));
        set<string> replicaIds = serverReplicaIds;
        while(!replicaIds.empty())
        {
            try
            {
                replicaIds.erase(obj->getReplicaId());
            }
            catch(const Ice::LocalException& ex)
            {
                cerr << ex << endl;
                test(false);
            }
        }
        
        admin->stopServer("Server1");
        admin->stopServer("Server2");
        admin->stopServer("Server3");

        admin->enableServer("Server1", false);
        admin->enableServer("Server2", false);
        admin->enableServer("Server3", false);

        try
        {
            obj->getReplicaId();
            test(false);
        }
        catch(const Ice::NoEndpointException&)
        {
        }
        
        admin->enableServer("Server1", true);
        admin->enableServer("Server2", true);
        admin->enableServer("Server3", true);

        replicaIds = serverReplicaIds;
        while(!replicaIds.empty())
        {
            try
            {
                replicaIds.erase(obj->getReplicaIdAndShutdown());
            }
            catch(const Ice::LocalException& ex)
            {
                cerr << ex << endl;
                test(false);
            }
        }

        removeServer(admin, "Server1");
        removeServer(admin, "Server2");
        removeServer(admin, "Server3");
    }
    {
        map<string, string> params;
        params["replicaGroup"] = "Random";
        params["id"] = "IceBox1";
        instantiateServer(admin, "IceBox", "localnode", params);
        TestIntfPrx obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("Random"));
        obj = TestIntfPrx::uncheckedCast(obj->ice_locatorCacheTimeout(0));
        obj = TestIntfPrx::uncheckedCast(obj->ice_connectionCached(false));
        set<string> replicaIds = svcReplicaIds;
        while(!replicaIds.empty())
        {
            try
            {
                replicaIds.erase(obj->getReplicaId());
            }
            catch(const Ice::ConnectionRefusedException&)
            {
            }
            catch(const Ice::LocalException& ex)
            {
                cerr << ex << endl;
                test(false);
            }
        }
        removeServer(admin, "IceBox1");
    }
    cout << "ok" << endl;

    cout << "testing replication with adaptive load balancing... " << flush;
    {
        map<string, string> params;
        params["replicaGroup"] = "Adaptive";
        params["id"] = "Server1";
        instantiateServer(admin, "Server", "localnode", params);
        params["id"] = "Server2";
        instantiateServer(admin, "Server", "localnode", params);
        params["id"] = "Server3";
        instantiateServer(admin, "Server", "localnode", params);
        TestIntfPrx obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("Adaptive"));
        obj = TestIntfPrx::uncheckedCast(obj->ice_locatorCacheTimeout(0));
        obj = TestIntfPrx::uncheckedCast(obj->ice_connectionCached(false));
        set<string> replicaIds = serverReplicaIds;
        while(!replicaIds.empty())
        {
            try
            {
                replicaIds.erase(obj->getReplicaId());
            }
            catch(const Ice::LocalException& ex)
            {
                cerr << ex << endl;
                test(false);
            }
        }
        removeServer(admin, "Server1");
        removeServer(admin, "Server2");
        removeServer(admin, "Server3");
    }
    {
        map<string, string> params;
        params["replicaGroup"] = "Adaptive";
        params["id"] = "IceBox1";
        instantiateServer(admin, "IceBox", "localnode", params);
        TestIntfPrx obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("Adaptive"));
        obj = TestIntfPrx::uncheckedCast(obj->ice_locatorCacheTimeout(0));
        obj = TestIntfPrx::uncheckedCast(obj->ice_connectionCached(false));
        set<string> replicaIds = svcReplicaIds;
        while(!replicaIds.empty())
        {
            try
            {
                replicaIds.erase(obj->getReplicaId());
            }
            catch(const Ice::ConnectionRefusedException&)
            {
            }
            catch(const Ice::LocalException& ex)
            {
                cerr << ex << endl;
                test(false);
            }
        }
        removeServer(admin, "IceBox1");
    }
    cout << "ok" << endl;

    cout << "testing filters... " << flush;
    {
        map<string, string> params;
        params["replicaGroup"] = "Ordered-Filtered";
        params["id"] = "Server1";
        instantiateServer(admin, "Server", "localnode", params);
        params["id"] = "Server2";
        instantiateServer(admin, "Server", "localnode", params);
        params["id"] = "Server3";
        instantiateServer(admin, "IceBox1", "localnode", params);

        Ice::LocatorPrx locator = comm->getDefaultLocator();
        Ice::Context ctx;
        ctx["server"] = "Server3";
        locator->ice_context(ctx);

        TestIntfPrx obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("Ordered-Filtered"));
        obj = TestIntfPrx::uncheckedCast(obj->ice_locatorCacheTimeout(0));
        obj = TestIntfPrx::uncheckedCast(obj->ice_connectionCached(false));
        try
        {
            ctx["server"] = "Server3";
            test(obj->ice_locator(locator->ice_context(ctx))->getReplicaId() == "Server3.Service.Service");
            ctx["server"] = "Server1";
            test(obj->ice_locator(locator->ice_context(ctx))->getReplicaId() == "Server1.ReplicatedAdapter");
            ctx["server"] = "Server2";
            test(obj->ice_locator(locator->ice_context(ctx))->getReplicaId() == "Server2.ReplicatedAdapter");
        }
        catch(const Ice::LocalException& ex)
        {
            cerr << ex << endl;
            test(false);
        }

        try
        {
            ctx["server"] = "Server3";
            test(query->ice_context(ctx)->findAllReplicas(obj)[0]->ice_getAdapterId() == "Server3.Service.Service");
            ctx["server"] = "Server1";
            test(query->ice_context(ctx)->findAllReplicas(obj)[0]->ice_getAdapterId() == "Server1.ReplicatedAdapter");
            ctx["server"] = "Server2";
            test(query->ice_context(ctx)->findAllReplicas(obj)[0]->ice_getAdapterId() == "Server2.ReplicatedAdapter");
        }
        catch(const Ice::LocalException& ex)
        {
            cerr << ex << endl;
            test(false);
        }

        try
        {
            ctx["server"] = "Server3";
            test(query->ice_context(ctx)->findObjectByType("::Test::TestIntf2")->ice_getAdapterId() == 
                 "Server3.Service.Service");
            ctx["server"] = "Server1";
            test(query->ice_context(ctx)->findObjectByType("::Test::TestIntf2")->ice_getAdapterId() == 
                 "Server1.ReplicatedAdapter");
            ctx["server"] = "Server2";
            test(query->ice_context(ctx)->findObjectByType("::Test::TestIntf2")->ice_getAdapterId() == 
                 "Server2.ReplicatedAdapter");

            ctx["server"] = "Server3";
            test(query->ice_context(ctx)->findObjectByTypeOnLeastLoadedNode(
                     "::Test::TestIntf2", LoadSample5)->ice_getAdapterId() == "Server3.Service.Service");
            ctx["server"] = "Server1";
            test(query->ice_context(ctx)->findObjectByTypeOnLeastLoadedNode(
                     "::Test::TestIntf2", LoadSample5)->ice_getAdapterId() == "Server1.ReplicatedAdapter");
            ctx["server"] = "Server2";
            test(query->ice_context(ctx)->findObjectByTypeOnLeastLoadedNode(
                     "::Test::TestIntf2", LoadSample5)->ice_getAdapterId() == "Server2.ReplicatedAdapter");

            ctx["server"] = "Server3";
            test(query->ice_context(ctx)->findAllObjectsByType("::Test::TestIntf2")[0]->ice_getAdapterId() == 
                 "Server3.Service.Service");
            ctx["server"] = "Server1";
            test(query->ice_context(ctx)->findAllObjectsByType("::Test::TestIntf2")[0]->ice_getAdapterId() == 
                 "Server1.ReplicatedAdapter");
            ctx["server"] = "Server2";
            test(query->ice_context(ctx)->findAllObjectsByType("::Test::TestIntf2")[0]->ice_getAdapterId() == 
                 "Server2.ReplicatedAdapter");
        }
        catch(const Ice::LocalException& ex)
        {
            cerr << ex << endl;
            test(false);
        }

        removeServer(admin, "Server1");
        removeServer(admin, "Server2");
        removeServer(admin, "Server3");
    }
    {
        map<string, string> params;
        params["replicaGroup"] = "Unknown";
        params["id"] = "UnknownServer";
        instantiateServer(admin, "Server", "localnode", params);

        TestIntfPrx obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("Unknown"));
        obj->getReplicaId();

        removeServer(admin, "UnknownServer");
    }
    {
        map<string, string> params;
        params["replicaGroup"] = "Exclude";
        params["id"] = "ExcludeServer";
        instantiateServer(admin, "Server", "localnode", params);

        TestIntfPrx obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("Exclude"));
        obj = TestIntfPrx::uncheckedCast(obj->ice_locatorCacheTimeout(0));
        obj = TestIntfPrx::uncheckedCast(obj->ice_connectionCached(false));
        try
        {
            obj->getReplicaId();
            test(false);
        }
        catch(const Ice::NoEndpointException&)
        {
        }

        Ice::LocatorPrx locator = comm->getDefaultLocator();
        try
        {
            Ice::Context ctx;
            ctx["server"] = "Server2";
            obj->ice_locator(locator->ice_context(ctx))->getReplicaId();
            test(false);
        }
        catch(const Ice::NoEndpointException&)
        {
        }
        try
        {
            Ice::Context ctx;
            ctx["server"] = "Server3";
            obj->ice_locator(locator->ice_context(ctx))->getReplicaId();
            test(false);
        }
        catch(const Ice::NoEndpointException&)
        {
        }

        Ice::Context ctx;
        ctx["server"] = "Server1";
        string id = obj->ice_locator(locator->ice_context(ctx))->getReplicaId();

        removeServer(admin, "ExcludeServer");
    }
    cout << "ok" << endl;

    cout << "testing load balancing n-replicas... " << flush;
    {
        TestIntfPrx obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("RoundRobin-2"));
        obj = TestIntfPrx::uncheckedCast(obj->ice_locatorCacheTimeout(-1));
        obj = TestIntfPrx::uncheckedCast(obj->ice_connectionCached(false));
        try
        {
            obj->ice_ping();
            test(false);
        }
        catch(const Ice::NoEndpointException&)
        {
        }

        map<string, string> params;
        params["replicaGroup"] = "RoundRobin-2";
        params["id"] = "Server1";
        instantiateServer(admin, "Server", "localnode", params);
        params["id"] = "Server2";
        instantiateServer(admin, "Server", "localnode", params);
        params["id"] = "Server3";
        instantiateServer(admin, "Server", "localnode", params);
        try
        {
            set<string> replicaIds;
            set<string> expected;
            replicaIds.insert("Server1.ReplicatedAdapter");
            replicaIds.insert("Server2.ReplicatedAdapter");
            expected = replicaIds;
            while(!replicaIds.empty())
            {
                string replicaId = obj->getReplicaId();
                test(expected.find(replicaId) != expected.end());
                replicaIds.erase(replicaId);
            }
            obj->ice_locatorCacheTimeout(0)->ice_ping();
            replicaIds.insert("Server2.ReplicatedAdapter");
            replicaIds.insert("Server3.ReplicatedAdapter");
            expected = replicaIds;
            while(!replicaIds.empty())
            {
                string replicaId = obj->getReplicaId();
                test(expected.find(replicaId) != expected.end());
                replicaIds.erase(replicaId);
            }
            obj->ice_locatorCacheTimeout(0)->ice_ping();
            replicaIds.insert("Server3.ReplicatedAdapter");
            replicaIds.insert("Server1.ReplicatedAdapter");
            expected = replicaIds;
            while(!replicaIds.empty())
            {
                string replicaId = obj->getReplicaId();
                test(expected.find(replicaId) != expected.end());
                replicaIds.erase(replicaId);
            }

            admin->stopServer("Server1");
            admin->stopServer("Server2");
            admin->stopServer("Server3");

            obj->ice_locatorCacheTimeout(0)->ice_ping();
            int nRetry = 500; 
            while(replicaIds.size() != 2 && --nRetry > 0)
            {
                replicaIds.insert(obj->getReplicaId());
            }
            test(replicaIds.size() == 2);
        }
        catch(const Ice::LocalException& ex)
        {
            cerr << ex << endl;
            test(false);
        }
        removeServer(admin, "Server1");
        removeServer(admin, "Server2");
        removeServer(admin, "Server3");

    }
    {
        TestIntfPrx obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("RoundRobin-All"));
        obj = TestIntfPrx::uncheckedCast(obj->ice_locatorCacheTimeout(-1));
        obj = TestIntfPrx::uncheckedCast(obj->ice_connectionCached(false));
        try
        {
            obj->ice_ping();
            test(false);
        }
        catch(const Ice::NoEndpointException&)
        {
        }

        map<string, string> params;
        params["replicaGroup"] = "RoundRobin-All";
        params["id"] = "Server1";
        instantiateServer(admin, "Server", "localnode", params);
        params["id"] = "Server2";
        instantiateServer(admin, "Server", "localnode", params);
        params["id"] = "Server3";
        instantiateServer(admin, "Server", "localnode", params);
        try
        {
            set<string> replicaIds;
            set<string> expected;
            replicaIds.insert("Server1.ReplicatedAdapter");
            replicaIds.insert("Server2.ReplicatedAdapter");
            replicaIds.insert("Server3.ReplicatedAdapter");
            expected = replicaIds;
            while(!replicaIds.empty())
            {
                string replicaId = obj->getReplicaId();
                test(expected.find(replicaId) != expected.end());
                replicaIds.erase(replicaId);
            }
            obj->ice_locatorCacheTimeout(0)->ice_ping();
            replicaIds.insert("Server1.ReplicatedAdapter");
            replicaIds.insert("Server2.ReplicatedAdapter");
            replicaIds.insert("Server3.ReplicatedAdapter");
            expected = replicaIds;
            while(!replicaIds.empty())
            {
                string replicaId = obj->getReplicaId();
                test(expected.find(replicaId) != expected.end());
                replicaIds.erase(replicaId);
            }
        }
        catch(const Ice::LocalException& ex)
        {
            cerr << ex << endl;
            test(false);
        }
        removeServer(admin, "Server1");
        removeServer(admin, "Server2");
        removeServer(admin, "Server3");

    }
    cout << "ok" << endl;

    cout << "testing replication with inactive nodes... " << flush;
    {
        map<string, string> params;

        params["replicaGroup"] = "Random";
        params["id"] = "Server1";
        instantiateServer(admin, "Server", "inactivenode", params);
        params["id"] = "Server2";
        instantiateServer(admin, "Server", "localnode", params);
        TestIntfPrx obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("Random"));
        test(obj->getReplicaId() == "Server2.ReplicatedAdapter");

        //
        // Also make sure that findObjectByTypeOnLeastLoadedNode still work.
        //
        obj = TestIntfPrx::uncheckedCast(query->findObjectByTypeOnLeastLoadedNode("::Test::TestIntf", LoadSample1));
        test(obj->getReplicaId() == "Server2.ReplicatedAdapter");
        
        removeServer(admin, "Server1");
        removeServer(admin, "Server2");

        params["replicaGroup"] = "RoundRobin";
        params["id"] = "Server1";
        instantiateServer(admin, "Server", "inactivenode", params);
        params["id"] = "Server2";
        instantiateServer(admin, "Server", "localnode", params);
        obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("RoundRobin"));
        test(obj->getReplicaId() == "Server2.ReplicatedAdapter");
        removeServer(admin, "Server1");
        removeServer(admin, "Server2");

        params["replicaGroup"] = "Adaptive";
        params["id"] = "Server1";
        instantiateServer(admin, "Server", "inactivenode", params);
        params["id"] = "Server2";
        instantiateServer(admin, "Server", "localnode", params);
        obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("Adaptive"));
        test(obj->getReplicaId() == "Server2.ReplicatedAdapter");
        removeServer(admin, "Server1");
        removeServer(admin, "Server2");

        params["replicaGroup"] = "Random";
        params["id"] = "IceBox1";
        instantiateServer(admin, "IceBox", "localnode", params);
        params["id"] = "Server1";
        instantiateServer(admin, "Server", "inactivenode", params);
        obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("Random"));
        test(svcReplicaIds.find(obj->getReplicaId()) != svcReplicaIds.end());
        removeServer(admin, "IceBox1");
        removeServer(admin, "Server1");
    };
    cout << "ok" << endl;

    cout << "testing replica group from different applications... " << flush;
    {
        map<string, string> params;
        params["replicaGroup"] = "Random";
        params["id"] = "Server1";
        instantiateServer(admin, "Server", "localnode", params);

        ApplicationUpdateDescriptor update;
        update.name = "Test";
        update.removeReplicaGroups.push_back("Random");
        try
        {
            admin->updateApplication(update);
            test(false);
        }
        catch(const DeploymentException&)
        {
            // The Random replica goup is used by Server1!
        }

        //
        // Add an application Test1 without replica groups and a
        // server that uses the Random replica group.
        //
        ApplicationInfo app = admin->getApplicationInfo("Test");
        app.descriptor.name = "Test1";
        app.descriptor.replicaGroups.clear();
        app.descriptor.nodes.clear();
        try
        {
            admin->addApplication(app.descriptor);
        }
        catch(const DeploymentException& ex)
        {
            cerr << ex << endl;
            test(false);
        }
        params["id"] = "Server2";
        instantiateServer(admin, "Server", "localnode", params, "Test1");

        try
        {
            admin->removeApplication("Test");
            test(false);
        }
        catch(const DeploymentException&)
        {
            // Test has a replica group referenced by the Test1 application.
        }

        TestIntfPrx obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("Random"));
        obj = TestIntfPrx::uncheckedCast(obj->ice_locatorCacheTimeout(0));
        obj = TestIntfPrx::uncheckedCast(obj->ice_connectionCached(false));
        set<string> replicaIds;
        replicaIds.insert("Server1.ReplicatedAdapter");
        replicaIds.insert("Server2.ReplicatedAdapter");
        while(!replicaIds.empty())
        {
            try
            {
                replicaIds.erase(obj->getReplicaId());
            }
            catch(const Ice::LocalException& ex)
            {
                cerr << ex << endl;
                test(false);
            }
        }
        
        removeServer(admin, "Server2");
        removeServer(admin, "Server1");

        ReplicaGroupDescriptor replicaGroup;
        replicaGroup.id = "ReplicatedAdapterFromTest1";
        replicaGroup.loadBalancing = new RandomLoadBalancingPolicy();
        replicaGroup.loadBalancing->nReplicas = "0";
        update = ApplicationUpdateDescriptor();
        update.name = "Test1";
        update.replicaGroups.push_back(replicaGroup);

        try
        {
            admin->updateApplication(update);
        }
        catch(const DeploymentException& ex)
        {
            cerr << ex.reason << endl;
            test(false);
        }

        params["replicaGroup"] = "ReplicatedAdapterFromTest1";
        params["id"] = "Server1";
        instantiateServer(admin, "Server", "localnode", params);

        try
        {
            admin->removeApplication("Test1");
            test(false);
        }
        catch(const DeploymentException&)
        {
            // ReplicatedAdapterFromTest1 used by server from Test
        }

        update = ApplicationUpdateDescriptor();
        update.name = "Test1";
        update.removeReplicaGroups.push_back("ReplicatedAdapterFromTest1");
        try
        {
            admin->updateApplication(update);
            test(false);
        }
        catch(const DeploymentException&)
        {
            // ReplicatedAdapterFromTest1 used by server from Test
        }

        removeServer(admin, "Server1");

        try
        {
            admin->updateApplication(update);
        }
        catch(const DeploymentException& ex)
        {
            cerr << ex << endl;
            test(false);
        }

        admin->removeApplication("Test1");
    };
    cout << "ok" << endl;

    cout << "testing replica group with different server encoding support... " << flush;
    {
        vector<string> loadBalancings;
        loadBalancings.push_back("Random");
        loadBalancings.push_back("RoundRobin");
        loadBalancings.push_back("RoundRobin-All");
        for(vector<string>::const_iterator p = loadBalancings.begin(); p != loadBalancings.end(); ++p)
        {
            map<string, string> params;
            params["replicaGroup"] = *p;
            params["id"] = "Server1";
            params["encoding"] = "1.0";
            instantiateServer(admin, "Server", "localnode", params);
            params["id"] = "Server2";
            params["encoding"] = "1.1";
            instantiateServer(admin, "Server", "localnode", params);
            params["id"] = "Server3";
            params["encoding"] = "1.0";
            instantiateServer(admin, "Server", "localnode", params);
            
            TestIntfPrx obj = TestIntfPrx::uncheckedCast(comm->stringToProxy(*p));
            obj = obj->ice_locatorCacheTimeout(0);
            obj = obj->ice_connectionCached(false);
            
            for(int i = 0; i < 30; ++i)
            {
                test(obj->getReplicaId() == "Server2.ReplicatedAdapter");
            }
            
            obj = obj->ice_encodingVersion(Ice::Encoding_1_0);
            set<string> replicaIds = serverReplicaIds;
            while(!replicaIds.empty())
            {
                try
                {
                    replicaIds.erase(obj->getReplicaId());
                }
                catch(const Ice::LocalException& ex)
                {
                    cerr << ex << endl;
                    test(false);
                }
            }
            
            removeServer(admin, "Server1");
            removeServer(admin, "Server2");
            removeServer(admin, "Server3");
        }
    };
    cout << "ok" << endl;

    session->destroy();
}