Esempio n. 1
0
void 
allTests(const Ice::CommunicatorPtr& communicator)
{
    SessionKeepAliveThreadPtr keepAlive = new SessionKeepAliveThread(
        communicator->getLogger(), IceUtil::Time::seconds(5));
    keepAlive->start();

    RegistryPrx registry = IceGrid::RegistryPrx::checkedCast(communicator->stringToProxy("IceGrid/Registry"));
    test(registry);
    AdminSessionPrx session = registry->createAdminSession("foo", "bar");

    keepAlive->add(session);

    AdminPrx admin = session->getAdmin();
    test(admin);

    cout << "starting router... " << flush;
    try
    {
        admin->startServer("Glacier2");
    }
    catch(const ServerStartException& ex)
    {
        cerr << ex.reason << endl;
        test(false);
    }
    cout << "ok" << endl;

    const int allocationTimeout = 5000;

    Ice::ObjectPrx obj;
    Ice::ObjectPrx dummy;

    try
    {
        cout << "testing create session... " << flush;
        SessionPrx session1 = registry->createSession("Client1", "");
        SessionPrx session2 = registry->createSession("Client2", "");
        
        keepAlive->add(session1);
        keepAlive->add(session2);
    
        cout << "ok" << endl;

        cout << "testing allocate object by identity... " << flush;

        Ice::Identity allocatable = communicator->stringToIdentity("allocatable");
        Ice::Identity allocatablebis = communicator->stringToIdentity("allocatablebis");

        try
        {
            session1->allocateObjectById(communicator->stringToIdentity("dummy"));
        }
        catch(const ObjectNotRegisteredException&)
        {
        }
        try
        {
            session1->releaseObject(communicator->stringToIdentity("dummy"));
        }
        catch(const ObjectNotRegisteredException&)
        {
        }

        try
        {
            session1->allocateObjectById(communicator->stringToIdentity("nonallocatable"));
            test(false);
        }
        catch(const AllocationException&)
        {
            test(false);
        }
        catch(const ObjectNotRegisteredException&)
        {
        }

        try
        {
            session2->allocateObjectById(communicator->stringToIdentity("nonallocatable"));
            test(false);
        }
        catch(const AllocationException&)
        {
            test(false);
        }
        catch(const ObjectNotRegisteredException&)
        {
        }

        try
        {
            session1->releaseObject(communicator->stringToIdentity("nonallocatable"));
            test(false);
        }
        catch(const AllocationException&)
        {
            test(false);
        }
        catch(const ObjectNotRegisteredException&)
        {
        }

        try
        {
            session2->releaseObject(communicator->stringToIdentity("nonallocatable"));
            test(false);
        }
        catch(const AllocationException&)
        {
            test(false);
        }
        catch(const ObjectNotRegisteredException&)
        {
        }

        session1->allocateObjectById(allocatable);
        try
        {
            session1->allocateObjectById(allocatable);
            test(false);
        }
        catch(const AllocationException&)
        {
        }

        session1->setAllocationTimeout(0);
        session2->setAllocationTimeout(0);

        try
        {
            session2->allocateObjectById(allocatable);
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        try
        {
            session2->releaseObject(allocatable);
            test(false);
        }
        catch(const AllocationException&)
        {
        }

        session1->allocateObjectById(allocatablebis);
        try
        {
            session2->allocateObjectById(allocatablebis);
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        session1->releaseObject(allocatablebis);
        session2->allocateObjectById(allocatablebis);
        try
        {
            session1->allocateObjectById(allocatablebis);
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        session2->releaseObject(allocatablebis);
    
        session2->setAllocationTimeout(allocationTimeout);
        AllocateObjectByIdCallbackPtr cb1 = new AllocateObjectByIdCallback();
        session2->allocateObjectById_async(cb1, allocatable);
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
        test(!cb1->hasResponse(dummy));
        session1->releaseObject(allocatable);
        cb1->waitResponse(__FILE__, __LINE__);
        test(cb1->hasResponse(dummy));

        session1->setAllocationTimeout(0);
        try
        {
            session1->allocateObjectById(allocatable);
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        try
        {
            session1->releaseObject(allocatable);
            test(false);
        }
        catch(const AllocationException&)
        {
        }
        session1->setAllocationTimeout(allocationTimeout);
        cb1 = new AllocateObjectByIdCallback();
        session1->allocateObjectById_async(cb1, allocatable);
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
        test(!cb1->hasResponse(dummy));
        session2->releaseObject(allocatable);
        cb1->waitResponse(__FILE__, __LINE__);
        test(cb1->hasResponse(dummy));

        session1->releaseObject(allocatable);

        cout << "ok" << endl;

        cout << "testing allocate object by type... " << flush;
    
        session1->setAllocationTimeout(0);
        session2->setAllocationTimeout(0);


        try
        {
            obj = session1->allocateObjectByType("::Unknown");
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
            test(false);
        }
        catch(const AllocationException&)
        {
        }

        try
        {
            obj = session1->allocateObjectByType("::NotAllocatable");
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
            test(false);
        }
        catch(const AllocationException&)
        {
        }

        obj = session1->allocateObjectByType("::Test");
        test(obj && obj->ice_getIdentity().name == "allocatable");
        try
        {
            session1->allocateObjectByType("::Test");
            test(false);
        }
        catch(const AllocationException&)
        {
        }
        try
        {
            session2->allocateObjectByType("::Test");
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        try
        {
            session2->releaseObject(obj->ice_getIdentity());
        }
        catch(const AllocationException&)
        {
        }

        session1->releaseObject(obj->ice_getIdentity());
        try
        {
            session1->releaseObject(obj->ice_getIdentity());
        }
        catch(const AllocationException&)
        {
        }

        obj = session2->allocateObjectByType("::Test"); // Allocate the object
        test(obj && obj->ice_getIdentity().name == "allocatable");
        try
        {
            session2->allocateObjectByType("::Test");
            test(false);
        }
        catch(const AllocationException&)
        {
        }
        try
        {
            session1->allocateObjectByType("::Test");
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        session1->allocateObjectByType("::TestBis");
        try
        {
            session2->allocateObjectByType("::TestBis");
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        session1->releaseObject(allocatablebis);
        session2->allocateObjectByType("::TestBis");
        try
        {
            session1->allocateObjectByType("::TestBis");
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        session2->releaseObject(allocatablebis);

        session1->setAllocationTimeout(allocationTimeout);
        AllocateObjectByTypeCallbackPtr cb3 = new AllocateObjectByTypeCallback();
        session1->allocateObjectByType_async(cb3, "::Test");
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
        test(!cb3->hasResponse(dummy));
        session2->releaseObject(obj->ice_getIdentity());
        cb3->waitResponse(__FILE__, __LINE__);
        test(cb3->hasResponse(obj));

        session1->releaseObject(obj->ice_getIdentity());
    
        cout << "ok" << endl;

        cout << "testing object allocation timeout... " << flush;    

        session1->allocateObjectById(allocatable);
        IceUtil::Time time = IceUtil::Time::now();
        session2->setAllocationTimeout(500);
        try
        {
            session2->allocateObjectById(allocatable);
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
            test(time + IceUtil::Time::milliSeconds(100) < IceUtil::Time::now());
        }
        time = IceUtil::Time::now();
        try
        {
            session2->allocateObjectById(allocatable);
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        test(time + IceUtil::Time::milliSeconds(100) < IceUtil::Time::now());
        time = IceUtil::Time::now();
        try
        {
            session2->allocateObjectByType("::Test");
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        test(time + IceUtil::Time::milliSeconds(100) < IceUtil::Time::now());

        session1->releaseObject(allocatable);
        session2->setAllocationTimeout(0);

        cout << "ok" << endl;

        cout << "testing server allocation... " << flush;

        session1->setAllocationTimeout(0);
        session2->setAllocationTimeout(0);

        Ice::Identity allocatable3 = communicator->stringToIdentity("allocatable3");
        Ice::Identity allocatable4 = communicator->stringToIdentity("allocatable4");

        session1->allocateObjectById(allocatable3);
        try
        {
            session2->allocateObjectById(allocatable3);
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        try
        {
            session2->allocateObjectById(allocatable4);
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }

        session1->allocateObjectById(allocatable4);
        session1->releaseObject(allocatable3);
        try
        {
            session2->allocateObjectById(allocatable3);
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        session1->releaseObject(allocatable4);
        session2->allocateObjectById(allocatable3);
        try
        {
            session1->allocateObjectById(allocatable3);
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        try
        {
            session1->allocateObjectById(allocatable4);
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        session2->allocateObjectById(allocatable4);
        session2->releaseObject(allocatable3);
        try
        {
            session1->allocateObjectById(allocatable3);
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        try
        {
            session1->allocateObjectByType("::TestServer1");
            test(false);
        }
        catch(AllocationException&)
        {
        }
        try
        {
            session1->allocateObjectByType("::TestServer2");
            test(false);
        }
        catch(AllocationException&)
        {
        }
        test(session2->allocateObjectByType("::TestServer1"));
        try
        {
            session2->allocateObjectByType("::TestServer1");
            test(false);
        }
        catch(AllocationException&)
        {
        }
        try
        {
            session2->allocateObjectByType("::TestServer2");
            test(false);
        }
        catch(AllocationException&)
        {
        }
        session2->releaseObject(allocatable3);
        session2->releaseObject(allocatable4);

        session1->allocateObjectById(allocatable3);
        session1->allocateObjectById(allocatable4);

        session2->setAllocationTimeout(allocationTimeout);
        cb1 = new AllocateObjectByIdCallback();
        session2->allocateObjectById_async(cb1, allocatable3);
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
        test(!cb1->hasResponse(dummy));
        session1->releaseObject(allocatable3);
        test(!cb1->hasResponse(dummy));
        session1->releaseObject(allocatable4);
        cb1->waitResponse(__FILE__, __LINE__);
        test(cb1->hasResponse(dummy));
        session2->releaseObject(allocatable3);

        session1->setAllocationTimeout(allocationTimeout);
        test(session2->allocateObjectByType("::TestServer1"));
        cb3 = new AllocateObjectByTypeCallback();
        session1->allocateObjectByType_async(cb3, "::TestServer2");
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
        test(!cb3->hasResponse(dummy));
        session2->releaseObject(allocatable3);
        cb3->waitResponse(__FILE__, __LINE__);
        test(cb3->hasResponse(dummy));
        session1->releaseObject(allocatable4);    

        session1->setAllocationTimeout(0);
        session2->setAllocationTimeout(0);
        test(session1->allocateObjectByType("::TestMultipleByServer"));
        try
        {
            session2->allocateObjectByType("::TestMultipleByServer");
            test(false);
        }
        catch(AllocationException&)
        {
        }
        test(session1->allocateObjectByType("::TestMultipleByServer"));
        session1->releaseObject(communicator->stringToIdentity("allocatable31"));
        session1->releaseObject(communicator->stringToIdentity("allocatable41"));
        test(session2->allocateObjectByType("::TestMultipleByServer"));
        try
        {
            session1->allocateObjectByType("::TestMultipleByServer");
            test(false);
        }
        catch(AllocationException&)
        {
        }
        test(session2->allocateObjectByType("::TestMultipleByServer"));
        session2->releaseObject(communicator->stringToIdentity("allocatable31"));
        session2->releaseObject(communicator->stringToIdentity("allocatable41"));

        Ice::ObjectPrx obj1 = session1->allocateObjectByType("::TestMultipleServer");
        test(obj1);
        Ice::ObjectPrx obj2 = session2->allocateObjectByType("::TestMultipleServer");
        test(obj2);
        try
        {
            session1->allocateObjectByType("::TestMultipleServer");
            test(false);
        }
        catch(AllocationTimeoutException&)
        {
        }
        try
        {
            session2->allocateObjectByType("::TestMultipleServer");
            test(false);
        }
        catch(AllocationTimeoutException&)
        {
        }       
        session1->releaseObject(obj1->ice_getIdentity());
        obj1 = session2->allocateObjectByType("::TestMultipleServer");
        session2->releaseObject(obj1->ice_getIdentity());
        session2->releaseObject(obj2->ice_getIdentity());

        cout << "ok" << endl;

        cout << "testing concurrent allocations... " << flush;

        session1->setAllocationTimeout(allocationTimeout);
        session2->setAllocationTimeout(allocationTimeout);

        session2->allocateObjectById(allocatable);
        AllocateObjectByIdCallbackPtr cb11 = new AllocateObjectByIdCallback();
        AllocateObjectByIdCallbackPtr cb12 = new AllocateObjectByIdCallback();
        session1->allocateObjectById_async(cb11, allocatable);
        session1->allocateObjectById_async(cb12, allocatable);
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
        test(!cb11->hasResponse(dummy));
        test(!cb12->hasResponse(dummy));
        session2->releaseObject(allocatable);
        cb11->waitResponse(__FILE__, __LINE__);
        cb12->waitResponse(__FILE__, __LINE__);
        test(cb11->hasResponse(dummy) ? cb12->hasException() : cb12->hasResponse(dummy));
        test(cb12->hasResponse(dummy) ? cb11->hasException() : cb11->hasResponse(dummy));
        session1->releaseObject(allocatable);

        session2->allocateObjectById(allocatable);
        AllocateObjectByTypeCallbackPtr cb31 = new AllocateObjectByTypeCallback();
        AllocateObjectByTypeCallbackPtr cb32 = new AllocateObjectByTypeCallback();
        session1->allocateObjectByType_async(cb31, "::Test");
        session1->allocateObjectByType_async(cb32, "::Test");
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
        test(!cb31->hasResponse(dummy));
        test(!cb32->hasResponse(dummy));
        session2->releaseObject(allocatable);
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(300));
        do
        {
            IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(200));
        }
        while(!cb31->hasResponse(dummy) && !cb32->hasResponse(dummy));
        test(cb31->hasResponse(dummy) && dummy && !cb32->hasResponse(dummy) ||
             cb32->hasResponse(dummy) && dummy && !cb31->hasResponse(dummy));
        session1->releaseObject(allocatable);
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(300));
        AllocateObjectByTypeCallbackPtr cb33 = cb31->hasResponse(dummy) ? cb32 : cb31;
        cb33->waitResponse(__FILE__, __LINE__);
        test(cb33->hasResponse(dummy) && dummy);
        session1->releaseObject(allocatable);

        session2->allocateObjectById(allocatable3);
        cb11 = new AllocateObjectByIdCallback();
        cb12 = new AllocateObjectByIdCallback();
        session1->allocateObjectById_async(cb11, allocatable3);
        session1->allocateObjectById_async(cb12, allocatable3);
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
        test(!cb11->hasResponse(dummy));
        test(!cb12->hasResponse(dummy));
        session2->releaseObject(allocatable3);
        cb11->waitResponse(__FILE__, __LINE__);
        cb12->waitResponse(__FILE__, __LINE__);
        test(cb11->hasResponse(dummy) ? cb12->hasException() : cb12->hasResponse(dummy));
        test(cb12->hasResponse(dummy) ? cb11->hasException() : cb11->hasResponse(dummy));
        session1->releaseObject(allocatable3);

        session2->allocateObjectById(allocatable3);
        cb31 = new AllocateObjectByTypeCallback();
        cb32 = new AllocateObjectByTypeCallback();
        session1->allocateObjectByType_async(cb31, "::TestServer1");
        session1->allocateObjectByType_async(cb32, "::TestServer1");
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
        test(!cb31->hasResponse(dummy));
        test(!cb32->hasResponse(dummy));
        session2->releaseObject(allocatable3);
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(300));
        do
        {
            IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(200));
        }
        while(!cb31->hasResponse(dummy) && !cb32->hasResponse(dummy));
        test(cb31->hasResponse(dummy) && dummy && !cb32->hasResponse(dummy) ||
             cb32->hasResponse(dummy) && dummy && !cb31->hasResponse(dummy));
        session1->releaseObject(allocatable3);
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(300));
        cb33 = cb31->hasResponse(dummy) ? cb32 : cb31;
        cb33->waitResponse(__FILE__, __LINE__);
        test(cb33->hasResponse(dummy) && dummy);
        session1->releaseObject(allocatable3);

        session1->allocateObjectById(allocatable3);
        cb31 = new AllocateObjectByTypeCallback();
        cb32 = new AllocateObjectByTypeCallback();
        session1->allocateObjectByType_async(cb31, "::TestServer1");
        session1->allocateObjectByType_async(cb32, "::TestServer1");
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
        test(!cb31->hasResponse(dummy));
        test(!cb32->hasResponse(dummy));
        session1->releaseObject(allocatable3);
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(300));
        do
        {
            IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(200));
        }
        while(!cb31->hasResponse(dummy) && !cb32->hasResponse(dummy));
        test(cb31->hasResponse(dummy) && dummy && !cb32->hasResponse(dummy) ||
             cb32->hasResponse(dummy) && dummy && !cb31->hasResponse(dummy));
        session1->releaseObject(allocatable3);
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(300));
        cb33 = cb31->hasResponse(dummy) ? cb32 : cb31;
        cb33->waitResponse(__FILE__, __LINE__);
        test(cb33->hasResponse(dummy) && dummy);
        session1->releaseObject(allocatable3);

        cout << "ok" << endl;

        cout << "testing session destroy... " << flush;

        obj = session2->allocateObjectByType("::Test"); // Allocate the object
        test(obj && obj->ice_getIdentity().name == "allocatable");

        session1->setAllocationTimeout(allocationTimeout);
        cb3 = new AllocateObjectByTypeCallback();
        session1->allocateObjectByType_async(cb3, "::Test");
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
        test(!cb3->hasResponse(dummy));
        session2->destroy();    
        cb3->waitResponse(__FILE__, __LINE__);
        test(cb3->hasResponse(obj));
        session1->destroy();

        session2 = SessionPrx::uncheckedCast(registry->createSession("Client2", ""));
        session2->setAllocationTimeout(0);
        session2->allocateObjectById(allocatable);
        session2->destroy();

        cout << "ok" << endl;

        cout << "testing allocation with Glacier2 session... " << flush;
        Ice::ObjectPrx routerBase = communicator->stringToProxy("Glacier2/router:default -p 12347");
        Glacier2::RouterPrx router1 = Glacier2::RouterPrx::checkedCast(routerBase->ice_connectionId("client1"));
        test(router1);
        
        Glacier2::SessionPrx sessionBase = router1->createSession("test1", "abc123");
        try
        {
            session1 = IceGrid::SessionPrx::checkedCast(sessionBase->ice_connectionId("client1")->ice_router(router1));
            test(session1);
            session1->ice_ping();

            Ice::ObjectPrx obj;
            obj = session1->allocateObjectById(allocatable)->ice_connectionId("client1")->ice_router(router1);
            obj->ice_ping();
            session1->releaseObject(allocatable);
            try
            {
                obj->ice_ping();
            }
            catch(const Ice::ObjectNotExistException&)
            {
            }

            obj = session1->allocateObjectById(allocatable3)->ice_connectionId("client1")->ice_router(router1);
            obj->ice_ping();
            obj2 = communicator->stringToProxy("allocatable4")->ice_connectionId("client1")->ice_router(router1);
            obj2->ice_ping();
            session1->releaseObject(allocatable3);
            try
            {
                obj->ice_ping();
            }
            catch(const Ice::ObjectNotExistException&)
            {
            }
            try
            {
                obj2->ice_ping();
            }
            catch(const Ice::ObjectNotExistException&)
            {
            }
            session1->destroy();
        }
        catch(const Ice::LocalException& ex)
        {
            cerr << ex << endl;
            test(false);
        }
        cout << "ok" << endl;

        cout << "stress test... " << flush;

        SessionPrx stressSession = registry->createSession("StressSession", "");        
        keepAlive->add(stressSession);

        const int nClients = 10;
        int i;
        vector<StressClientPtr> clients;
        for(i = 0; i < nClients - 2; ++i)
        {
            if(IceUtil::random(2) == 1)
            {
                clients.push_back(new StressClient(i, registry, false));
            }
            else
            {
                clients.push_back(new StressClient(i, stressSession));
            }
            clients.back()->start();
        }
        clients.push_back(new StressClient(i++, registry, true));
        clients.back()->start();
        clients.push_back(new StressClient(i++, registry, true));
        clients.back()->start();
        
        for(vector<StressClientPtr>::const_iterator p = clients.begin(); p != clients.end(); ++p)
        {
            (*p)->notifyThread();
        }

        //
        // Let the stress client run for a bit.
        //
        IceUtil::ThreadControl::sleep(IceUtil::Time::seconds(8));

        //
        // Terminate the stress clients.
        //
        for(vector<StressClientPtr>::const_iterator q = clients.begin(); q != clients.end(); ++q)
        {
            (*q)->terminate();
            (*q)->getThreadControl().join();
        }

        stressSession->destroy();

        cout << "ok" << endl;
    }
    catch(const AllocationTimeoutException& ex)
    {
        cerr << ex << endl;
        test(false);
    }
    catch(const AllocationException& ex)
    {
        cerr << ex.reason << endl;
        test(false);
    }

    cout << "shutting down router... " << flush;
    admin->stopServer("Glacier2");
    cout << "ok" << endl;

    keepAlive->terminate();
    keepAlive->getThreadControl().join();
    keepAlive = 0;

    session->destroy();
}
Esempio n. 2
0
void
allTests(const Ice::CommunicatorPtr& comm)
{
    IceGrid::RegistryPrx registry = IceGrid::RegistryPrx::checkedCast(
        comm->stringToProxy(comm->getDefaultLocator()->ice_getIdentity().category + "/Registry"));

    AdminSessionPrx session = registry->createAdminSession("foo", "bar");

    session->ice_getConnection()->setACM(registry->getACMTimeout(), IceUtil::None, Ice::HeartbeatAlways);

    AdminPrx admin = session->getAdmin();
    test(admin);

    map<string, string> params;

    params.clear();
    params["id"] = "Master";
    params["replicaName"] = "";
    params["port"] = "12050";
    instantiateServer(admin, "IceGridRegistry", params);

    params.clear();
    params["id"] = "Slave1";
    params["replicaName"] = "Slave1";
    params["port"] = "12051";
    instantiateServer(admin, "IceGridRegistry", params);

    params.clear();
    params["id"] = "Slave2";
    params["replicaName"] = "Slave2";
    params["port"] = "12052";
    instantiateServer(admin, "IceGridRegistry", params);

    Ice::LocatorPrx masterLocator =
        Ice::LocatorPrx::uncheckedCast(comm->stringToProxy("RepTestIceGrid/Locator-Master:default -p 12050"));
    Ice::LocatorPrx slave1Locator =
        Ice::LocatorPrx::uncheckedCast(comm->stringToProxy("RepTestIceGrid/Locator-Slave1:default -p 12051"));
    Ice::LocatorPrx slave2Locator =
        Ice::LocatorPrx::uncheckedCast(comm->stringToProxy("RepTestIceGrid/Locator-Slave2:default -p 12052"));

    Ice::LocatorPrx replicatedLocator =
        Ice::LocatorPrx::uncheckedCast(comm->stringToProxy("RepTestIceGrid/Locator:default -p 12050:default -p 12051"));

    AdminPrx masterAdmin, slave1Admin, slave2Admin;

    admin->startServer("Master");
    masterAdmin = createAdminSession(masterLocator, "");

    admin->startServer("Slave1");
    slave1Admin = createAdminSession(slave1Locator, "Slave1");

    //
    // Test replication and well-known objects:
    //
    // - Locator interface
    // - Query interface
    //
    // - Registry object
    // - RegistryUserAccountMapper
    // - SessionManager/SSLSessionManager
    // - AdminSessionManager/AdminSSLSessionManager
    //
    cout << "testing replicated locator and query interface... " << flush;
    {
        Ice::EndpointSeq endpoints;
        ObjectInfo info;

        info = masterAdmin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Locator"));
        ObjectInfo info1 = slave1Admin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Locator"));
        test(slave1Admin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Locator")) == info);
        test(info.type == Ice::Locator::ice_staticId());
        endpoints = info.proxy->ice_getEndpoints();
        test(endpoints.size() == 2);
        test(endpoints[0]->toString().find("-p 12050") != string::npos);
        test(endpoints[1]->toString().find("-p 12051") != string::npos);

        info = masterAdmin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Query"));
        test(slave1Admin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Query")) == info);
        test(info.type == IceGrid::Query::ice_staticId());
        endpoints = info.proxy->ice_getEndpoints();
        test(endpoints.size() == 2);
        test(endpoints[0]->toString().find("-p 12050") != string::npos);
        test(endpoints[1]->toString().find("-p 12051") != string::npos);

        admin->startServer("Slave2");
        slave2Admin = createAdminSession(slave2Locator, "Slave2");

        info = masterAdmin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Locator"));
        // We eventually need to wait here for the update of the replicated objects to propagate to the replica.
        int nRetry = 0;
        while(slave1Admin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Locator")) != info && nRetry < maxRetry)
        {
            IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(sleepTime));
            ++nRetry;
        }
        test(slave2Admin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Locator")) == info);
        test(info.type == Ice::Locator::ice_staticId());
        endpoints = info.proxy->ice_getEndpoints();
        test(endpoints.size() == 3);
        test(endpoints[0]->toString().find("-p 12050") != string::npos);
        test(endpoints[1]->toString().find("-p 12051") != string::npos);
        test(endpoints[2]->toString().find("-p 12052") != string::npos);

        info = masterAdmin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Query"));
        // We eventually need to wait here for the update of the replicated objects to propagate to the replica.
        nRetry = 0;
        while(slave1Admin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Query")) != info && nRetry < maxRetry)
        {
            IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(sleepTime));
            ++nRetry;
        }
        test(slave2Admin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Query")) == info);
        test(info.type == IceGrid::Query::ice_staticId());
        endpoints = info.proxy->ice_getEndpoints();
        test(endpoints.size() == 3);
        test(endpoints[0]->toString().find("-p 12050") != string::npos);
        test(endpoints[1]->toString().find("-p 12051") != string::npos);
        test(endpoints[2]->toString().find("-p 12052") != string::npos);

        slave2Admin->shutdown();
        waitForServerState(admin, "Slave2", false);

        info = masterAdmin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Locator"));
        // We eventually need to wait here for the update of the replicated objects to propagate to the replica.
        nRetry = 0;
        while(slave1Admin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Locator")) != info && nRetry < maxRetry)
        {
            IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(sleepTime));
            ++nRetry;
        }
        test(slave1Admin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Locator")) == info);
        test(info.type == Ice::Locator::ice_staticId());
        endpoints = info.proxy->ice_getEndpoints();
        test(endpoints.size() == 2);
        test(endpoints[0]->toString().find("-p 12050") != string::npos);
        test(endpoints[1]->toString().find("-p 12051") != string::npos);

        info = masterAdmin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Query"));
        nRetry = 0;
        while(slave1Admin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Query")) != info && nRetry < maxRetry)
        {
            IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(sleepTime));
            ++nRetry;
        }
        test(slave1Admin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Query")) == info);
        test(info.type == IceGrid::Query::ice_staticId());
        endpoints = info.proxy->ice_getEndpoints();
        test(endpoints.size() == 2);
        test(endpoints[0]->toString().find("-p 12050") != string::npos);
        test(endpoints[1]->toString().find("-p 12051") != string::npos);

        QueryPrx query;
        query = QueryPrx::uncheckedCast(comm->stringToProxy("RepTestIceGrid/Query:" + endpoints[0]->toString()));
        Ice::ObjectProxySeq objs = query->findAllObjectsByType("::IceGrid::Registry");
        test(objs.size() == 2);
        query = QueryPrx::uncheckedCast(comm->stringToProxy("RepTestIceGrid/Query:" + endpoints[1]->toString()));
        test(objs == query->findAllObjectsByType("::IceGrid::Registry"));
    }
    cout << "ok" << endl;

    cout << "testing well-known IceGrid objects... " << flush;
    {
        //
        // Test Registry well-known object (we have already tested
        // admin session creation for the creation of the admin
        // session above!)
        //
        RegistryPrx masterRegistry = RegistryPrx::checkedCast(
            comm->stringToProxy("RepTestIceGrid/Registry")->ice_locator(replicatedLocator));
        RegistryPrx slave1Registry = RegistryPrx::checkedCast(
            comm->stringToProxy("RepTestIceGrid/Registry-Slave1")->ice_locator(replicatedLocator));

        SessionPrx session = masterRegistry->createSession("dummy", "dummy");
        session->destroy();
        if(comm->getProperties()->getProperty("Ice.Default.Protocol") == "ssl")
        {
            session = masterRegistry->createSessionFromSecureConnection();
            session->destroy();
        }
        else
        {
            try
            {
                masterRegistry->createSessionFromSecureConnection();
            }
            catch(const PermissionDeniedException&)
            {
            }
        }

        try
        {
            slave1Registry->createSession("dummy", "");
        }
        catch(const PermissionDeniedException&)
        {
        }
        try
        {
            slave1Registry->createSessionFromSecureConnection();
        }
        catch(const PermissionDeniedException&)
        {
        }

        //
        // Test registry user-account mapper.
        //
        UserAccountMapperPrx masterMapper = UserAccountMapperPrx::checkedCast(
            comm->stringToProxy("RepTestIceGrid/RegistryUserAccountMapper")->ice_locator(replicatedLocator));
        UserAccountMapperPrx slave1Mapper = UserAccountMapperPrx::checkedCast(
            comm->stringToProxy("RepTestIceGrid/RegistryUserAccountMapper-Slave1")->ice_locator(replicatedLocator));

        test(masterMapper->getUserAccount("Dummy User Account1") == "dummy1");
        test(masterMapper->getUserAccount("Dummy User Account2") == "dummy2");
        test(slave1Mapper->getUserAccount("Dummy User Account1") == "dummy1");
        test(slave1Mapper->getUserAccount("Dummy User Account2") == "dummy2");
        try
        {
            masterMapper->getUserAccount("unknown");
            test(false);
        }
        catch(const UserAccountNotFoundException&)
        {
        }
        try
        {
            slave1Mapper->getUserAccount("unknown");
            test(false);
        }
        catch(const UserAccountNotFoundException&)
        {
        }

        //
        // Test SessionManager, SSLSessionManager,
        // AdminSessionManager, AdminSSLSessionManager
        //
        comm->stringToProxy("RepTestIceGrid/SessionManager")->ice_locator(replicatedLocator)->ice_ping();
        comm->stringToProxy("RepTestIceGrid/SSLSessionManager")->ice_locator(replicatedLocator)->ice_ping();
        try
        {
            comm->stringToProxy("RepTestIceGrid/SessionManager-Slave1")->ice_locator(replicatedLocator)->ice_ping();
            test(false);
        }
        catch(const Ice::NotRegisteredException&)
        {
        }
        try
        {
            comm->stringToProxy("RepTestIceGrid/SSLSessionManager-Slave1")->ice_locator(replicatedLocator)->ice_ping();
            test(false);
        }
        catch(const Ice::NotRegisteredException&)
        {
        }

        comm->stringToProxy("RepTestIceGrid/AdminSessionManager")->ice_locator(replicatedLocator)->ice_ping();
        comm->stringToProxy("RepTestIceGrid/AdminSSLSessionManager")->ice_locator(replicatedLocator)->ice_ping();
        comm->stringToProxy("RepTestIceGrid/AdminSessionManager-Slave1")->ice_locator(replicatedLocator)->ice_ping();
        comm->stringToProxy("RepTestIceGrid/AdminSSLSessionManager-Slave1")->ice_locator(replicatedLocator)->ice_ping();
    }
    cout << "ok" << endl;

    //
    // Registry update test:
    //
    // - start master
    // - start slave1: keep slave1 up for each update
    // - start slave2: shutdown slave2 for each update
    // - ensure updates are correctly replicated
    // - updates to test: application/adapter/object
    //
    cout << "testing registry updates... " << flush;
    {
        ApplicationDescriptor app;
        app.name = "TestApp";
        app.description = "added application";

        AdapterInfo adpt;
        adpt.id = "TestAdpt";
        adpt.proxy = comm->stringToProxy("dummy:tcp -p 12345 -h 127.0.0.1");

        ObjectInfo obj;
        obj.proxy = comm->stringToProxy("dummy:tcp -p 12345 -h 127.0.0.1");
        obj.type = "::Hello";

        //
        // We use the locator registry from Slave1 to ensure that the
        // forwarding to the master work (the slave locator registry
        // forwards everything to the master).
        //
        Ice::LocatorRegistryPrx locatorRegistry = slave1Locator->getRegistry();

        //
        // Test addition of application, adapter, object.
        //

        try
        {
            slave1Admin->addApplication(app);
            test(false);
        }
        catch(const DeploymentException&)
        {
            // Slave can't modify the database.
        }
        masterAdmin->addApplication(app);

        locatorRegistry->setAdapterDirectProxy(adpt.id, adpt.proxy);

        try
        {
            slave1Admin->addObjectWithType(obj.proxy, obj.type);
            test(false);
        }
        catch(const DeploymentException&)
        {
            // Slave can't modify the database
        }
        masterAdmin->addObjectWithType(obj.proxy, obj.type);

        admin->startServer("Slave2");
        slave2Admin = createAdminSession(slave2Locator, "Slave2");

        test(masterAdmin->getApplicationInfo("TestApp").descriptor.description == "added application");
        test(slave1Admin->getApplicationInfo("TestApp").descriptor.description == "added application");
        test(slave2Admin->getApplicationInfo("TestApp").descriptor.description == "added application");

        test(masterAdmin->getAdapterInfo("TestAdpt")[0] == adpt);
        test(slave1Admin->getAdapterInfo("TestAdpt")[0] == adpt);
        test(slave2Admin->getAdapterInfo("TestAdpt")[0] == adpt);

        test(masterAdmin->getObjectInfo(obj.proxy->ice_getIdentity()) == obj);
        test(slave1Admin->getObjectInfo(obj.proxy->ice_getIdentity()) == obj);
        test(slave2Admin->getObjectInfo(obj.proxy->ice_getIdentity()) == obj);

        slave2Admin->shutdown();
        waitForServerState(admin, "Slave2", false);

        //
        // Test sync of application.
        //

        app.description = "updated1 application";
        try
        {
            slave1Admin->syncApplication(app);
            test(false);
        }
        catch(const DeploymentException&)
        {
            // Slave can't modify the database.
        }
        masterAdmin->syncApplication(app);

        admin->startServer("Slave2");
        slave2Admin = createAdminSession(slave2Locator, "Slave2");
        test(masterAdmin->getApplicationInfo("TestApp").descriptor.description == "updated1 application");
        test(slave1Admin->getApplicationInfo("TestApp").descriptor.description == "updated1 application");
        test(slave2Admin->getApplicationInfo("TestApp").descriptor.description == "updated1 application");
        slave2Admin->shutdown();
        waitForServerState(admin, "Slave2", false);

        //
        // Test update of application, adapter, object.
        //

        ApplicationUpdateDescriptor appUpdate;
        appUpdate.name = "TestApp";
        appUpdate.description = new BoxedString("updated2 application");
        try
        {
            slave1Admin->updateApplication(appUpdate);
            test(false);
        }
        catch(const DeploymentException&)
        {
            // Slave can't modify the database.
        }
        masterAdmin->updateApplication(appUpdate);

        adpt.replicaGroupId = "TestReplicaGroup";
        locatorRegistry->setReplicatedAdapterDirectProxy(adpt.id, adpt.replicaGroupId, adpt.proxy);

        obj.proxy = comm->stringToProxy("dummy:tcp -p 12346 -h 127.0.0.1");
        try
        {
            slave1Admin->updateObject(obj.proxy);
            test(false);
        }
        catch(const DeploymentException&)
        {
            // Slave can't modify the database
        }
        masterAdmin->updateObject(obj.proxy);

        admin->startServer("Slave2");
        slave2Admin = createAdminSession(slave2Locator, "Slave2");

        test(masterAdmin->getApplicationInfo("TestApp").descriptor.description == "updated2 application");
        test(slave1Admin->getApplicationInfo("TestApp").descriptor.description == "updated2 application");
        test(slave2Admin->getApplicationInfo("TestApp").descriptor.description == "updated2 application");

        test(masterAdmin->getAdapterInfo("TestAdpt")[0] == adpt);
        test(slave1Admin->getAdapterInfo("TestAdpt")[0] == adpt);
        test(slave2Admin->getAdapterInfo("TestAdpt")[0] == adpt);

        test(masterAdmin->getObjectInfo(obj.proxy->ice_getIdentity()) == obj);
        test(slave1Admin->getObjectInfo(obj.proxy->ice_getIdentity()) == obj);
        test(slave2Admin->getObjectInfo(obj.proxy->ice_getIdentity()) == obj);

        slave2Admin->shutdown();
        waitForServerState(admin, "Slave2", false);

        //
        // Test removal of application, adapter and object.

        try
        {
            slave1Admin->removeApplication("TestApp");
            test(false);
        }
        catch(const DeploymentException&)
        {
            // Slave can't modify the database.
        }
        masterAdmin->removeApplication("TestApp");

        try
        {
            slave1Admin->removeAdapter("TestAdpt");
            test(false);
        }
        catch(const DeploymentException&)
        {
            // Slave can't modify the database.
        }
        masterAdmin->removeAdapter("TestAdpt");
        try
        {
            slave1Admin->removeObject(obj.proxy->ice_getIdentity());
        }
        catch(const DeploymentException&)
        {
            // Slave can't modify the database.
        }
        masterAdmin->removeObject(obj.proxy->ice_getIdentity());

        admin->startServer("Slave2");
        slave2Admin = createAdminSession(slave2Locator, "Slave2");
        try
        {
            masterAdmin->getApplicationInfo("TestApp");
            test(false);
        }
        catch(const ApplicationNotExistException&)
        {
        }
        try
        {
            slave1Admin->getApplicationInfo("TestApp");
            test(false);
        }
        catch(const ApplicationNotExistException&)
        {
        }
        try
        {
            slave2Admin->getApplicationInfo("TestApp");
            test(false);
        }
        catch(const ApplicationNotExistException&)
        {
        }
        try
        {
            masterAdmin->getAdapterInfo("TestAdpt");
            test(false);
        }
        catch(const AdapterNotExistException&)
        {
        }
        try
        {
            slave1Admin->getAdapterInfo("TestAdpt");
            test(false);
        }
        catch(const AdapterNotExistException&)
        {
        }
        try
        {
            slave2Admin->getAdapterInfo("TestAdpt");
            test(false);
        }
        catch(const AdapterNotExistException&)
        {
        }
        try
        {
            masterAdmin->getObjectInfo(obj.proxy->ice_getIdentity());
            test(false);
        }
        catch(const ObjectNotRegisteredException&)
        {
        }
        try
        {
            slave1Admin->getObjectInfo(obj.proxy->ice_getIdentity());
            test(false);
        }
        catch(const ObjectNotRegisteredException&)
        {
        }
        try
        {
            slave2Admin->getObjectInfo(obj.proxy->ice_getIdentity());
            test(false);
        }
        catch(const ObjectNotRegisteredException&)
        {
        }

        slave2Admin->shutdown();
        waitForServerState(admin, "Slave2", false);
    }
    cout << "ok" << endl;

    params.clear();
    params["id"] = "Node1";
    instantiateServer(admin, "IceGridNode", params);

    //
    // Add an application which is using Node1. Otherwise, when a
    // registry restarts it would throw aways the proxy of the nodes
    // because the node isn't used by any application.
    //
    ApplicationDescriptor app;
    app.name = "DummyApp";
    app.nodes["Node1"].description = "dummy node";
    try
    {
        masterAdmin->addApplication(app);
    }
    catch(const Ice::Exception& ex)
    {
        cerr << ex << endl;
        test(false);
    }

    //
    // Test node session establishment.
    //
    // - start master, start slave1, start node, start slave2
    // - shutdown slave1, start slave1 -> node should re-connect
    // - shutdown master
    // - shutdown slave2, start slave2 -> node should re-connect
    // - shutdown slave1
    // - start master -> node connects to master
    // - start slave1 -> node connects to slave1
    //
    cout << "testing node session establishment... " << flush;
    {
        admin->startServer("Node1");

        waitForNodeState(masterAdmin, "Node1", true);
        waitForNodeState(slave1Admin, "Node1", true);

        admin->startServer("Slave2");
        slave2Admin = createAdminSession(slave2Locator, "Slave2");

        waitForNodeState(slave2Admin, "Node1", true); // Node should connect.

        slave1Admin->shutdown();
        waitForServerState(admin, "Slave1", false);
        admin->startServer("Slave1");
        slave1Admin = createAdminSession(slave1Locator, "Slave1");

        try
        {
            test(slave1Admin->pingNode("Node1")); // Node should be re-connected.
        }
        catch(const NodeNotExistException&)
        {
            test(false);
        }

        masterAdmin->shutdown();
        waitForServerState(admin, "Master", false);

        slave2Admin->shutdown();
        waitForServerState(admin, "Slave2", false);
        admin->startServer("Slave2");
        slave2Admin = createAdminSession(slave2Locator, "Slave2");

        try
        {
            //
            // On slow environments, it can take a bit for the node to
            // re-establish the connection so we ping it twice. The
            // second should succeed.
            //
            slave2Admin->pingNode("Node1");
            test(slave2Admin->pingNode("Node1")); // Node should be re-connected even if the master is down.
        }
        catch(const NodeNotExistException&)
        {
            test(false);
        }

        slave1Admin->shutdown();
        waitForServerState(admin, "Slave1", false);

        admin->startServer("Master");
        masterAdmin = createAdminSession(masterLocator, "");

        try
        {
            test(masterAdmin->pingNode("Node1")); // Node should be re-connected.
        }
        catch(const NodeNotExistException&)
        {
            test(false);
        }

        admin->startServer("Slave1");
        slave1Admin = createAdminSession(slave1Locator, "Slave1");

        try
        {
            test(slave1Admin->pingNode("Node1")); // Node should be re-connected.
        }
        catch(const NodeNotExistException&)
        {
            test(false);
        }

        try
        {
            test(masterAdmin->pingNode("Node1"));
        }
        catch(const NodeNotExistException&)
        {
            test(false);
        }

        try
        {
            test(slave2Admin->pingNode("Node1"));
        }
        catch(const NodeNotExistException&)
        {
            test(false);
        }

        slave2Admin->shutdown();
        waitForServerState(admin, "Slave2", false);
        admin->startServer("Slave2");
        slave2Admin = createAdminSession(slave2Locator, "Slave2");
        try
        {
            test(slave2Admin->pingNode("Node1"));
        }
        catch(const NodeNotExistException&)
        {
            test(false);
        }
    }
    cout << "ok" << endl;

    //
    // Testing updates with out-of-date replicas.
    //
    cout << "testing out-of-date replicas... " << flush;
    {
        ApplicationDescriptor app;
        app.name = "TestApp";
        app.description = "added application";

        ServerDescriptorPtr server = new ServerDescriptor();
        server->id = "Server";
        server->exe = comm->getProperties()->getProperty("ServerDir") + "/server";
        server->pwd = ".";
        server->applicationDistrib = false;
        server->allocatable = false;
        addProperty(server, "Ice.Admin.Endpoints", "tcp -h 127.0.0.1");
        server->activation = "on-demand";
        AdapterDescriptor adapter;
        adapter.name = "TestAdapter";
        adapter.id = "TestAdapter.Server";
        adapter.registerProcess = false;
        adapter.serverLifetime = true;
        PropertyDescriptor property;
        property.name = "TestAdapter.Endpoints";
        property.value = "default";
        server->propertySet.properties.push_back(property);
        property.name = "Identity";
        property.value = "test";
        server->propertySet.properties.push_back(property);
        ObjectDescriptor object;
        object.id = Ice::stringToIdentity("test");
        object.type = "::Test::TestIntf";
        adapter.objects.push_back(object);
        server->adapters.push_back(adapter);
        app.nodes["Node1"].servers.push_back(server);

        masterAdmin->addApplication(app);

        try
        {
            comm->stringToProxy("test")->ice_locator(masterLocator)->ice_locatorCacheTimeout(0)->ice_ping();
            comm->stringToProxy("test")->ice_locator(slave1Locator)->ice_locatorCacheTimeout(0)->ice_ping();
            comm->stringToProxy("test")->ice_locator(slave2Locator)->ice_locatorCacheTimeout(0)->ice_ping();
        }
        catch(const Ice::LocalException& ex)
        {
            cerr << ex << endl;
            test(false);
        }

        masterAdmin->stopServer("Server");

        //
        // Shutdown Slave2 and update application.
        //
        slave2Admin->shutdown();
        waitForServerState(admin, "Slave2", false);

        ApplicationUpdateDescriptor update;
        update.name = "TestApp";
        NodeUpdateDescriptor node;
        node.name = "Node1";
        node.servers.push_back(server);
        update.nodes.push_back(node);
        property.name = "Dummy";
        property.value = "val";
        server->propertySet.properties.push_back(property);
        masterAdmin->updateApplication(update);

        try
        {
            comm->stringToProxy("test")->ice_locator(masterLocator)->ice_locatorCacheTimeout(0)->ice_ping();
            comm->stringToProxy("test")->ice_locator(slave1Locator)->ice_locatorCacheTimeout(0)->ice_ping();
        }
        catch(const Ice::LocalException& ex)
        {
            cerr << ex << endl;
            test(false);
        }

        masterAdmin->shutdown();
        waitForServerState(admin, "Master", false);

        admin->startServer("Slave2");
        slave2Admin = createAdminSession(slave2Locator, "Slave2");
        waitForNodeState(slave2Admin, "Node1", true); // Node should connect.

        try
        {
            slave2Admin->startServer("Server");
            test(false);
        }
        catch(const DeploymentException&)
        {
        }
        try
        {
            comm->stringToProxy("test")->ice_locator(slave2Locator)->ice_locatorCacheTimeout(0)->ice_ping();
            test(false);
        }
        catch(const Ice::NoEndpointException&)
        {
        }

        admin->startServer("Master");
        masterAdmin = createAdminSession(masterLocator, "");

        slave2Admin->shutdown();
        waitForServerState(admin, "Slave2", false);
        admin->startServer("Slave2");
        slave2Admin = createAdminSession(slave2Locator, "Slave2");

        try
        {
            comm->stringToProxy("test")->ice_locator(slave2Locator)->ice_locatorCacheTimeout(0)->ice_ping();
        }
        catch(const Ice::LocalException& ex)
        {
            cerr << ex << endl;
            test(false);
        }

        //
        // Shutdown Node1 and update the application, then, shutdown
        // the master.
        //
        slave1Admin->shutdownNode("Node1");
        waitForServerState(admin, "Node1", false);

        slave2Admin->shutdown();
        waitForServerState(admin, "Slave2", false);

        property.name = "Dummy2";
        property.value = "val";
        server->propertySet.properties.push_back(property);
        masterAdmin->updateApplication(update);

        masterAdmin->shutdown();
        waitForServerState(admin, "Master", false);

        //
        // Restart Node1 and Slave2, Slave2 still has the old version
        // of the server so it should be able to load it. Slave1 has
        // a more recent version, so it can't load it.
        //
        admin->startServer("Slave2");
        slave2Admin = createAdminSession(slave2Locator, "Slave2");

        admin->startServer("Node1");

        waitForNodeState(slave2Admin, "Node1", true);

        slave1Admin->shutdown();
        waitForServerState(admin, "Slave1", false);

        try
        {
            comm->stringToProxy("test")->ice_locator(slave2Locator)->ice_locatorCacheTimeout(0)->ice_ping();
        }
        catch(const Ice::LocalException& ex)
        {
            cerr << ex << endl;
            test(false);
        }

        admin->startServer("Slave1");
        slave1Admin = createAdminSession(slave1Locator, "Slave1");

        try
        {
            comm->stringToProxy("test")->ice_locator(slave1Locator)->ice_locatorCacheTimeout(0)->ice_ping();
        }
        catch(const Ice::NoEndpointException&)
        {
        }

        try
        {
            comm->stringToProxy("test")->ice_locator(slave2Locator)->ice_locatorCacheTimeout(0)->ice_ping();
        }
        catch(const Ice::LocalException& ex)
        {
            cerr << ex << endl;
            test(false);
        }
        slave2Admin->stopServer("Server");

        //
        // Start the master. This will re-load the server on the node
        // and update the out-of-date replicas.
        //
        admin->startServer("Master");
        masterAdmin = createAdminSession(masterLocator, "");

        slave1Admin->shutdown();
        waitForServerState(admin, "Slave1", false);
        admin->startServer("Slave1");
        slave1Admin = createAdminSession(slave1Locator, "Slave1");

        slave2Admin->shutdownNode("Node1");
        waitForServerState(admin, "Node1", false);
        admin->startServer("Node1");

        slave2Admin->shutdown();
        waitForServerState(admin, "Slave2", false);
        admin->startServer("Slave2");
        slave2Admin = createAdminSession(slave2Locator, "Slave2");

        waitForNodeState(masterAdmin, "Node1", true);
        waitForNodeState(slave1Admin, "Node1", true);
        waitForNodeState(slave2Admin, "Node1", true);

        try
        {
            comm->stringToProxy("test")->ice_locator(masterLocator)->ice_locatorCacheTimeout(0)->ice_ping();
            comm->stringToProxy("test")->ice_locator(slave1Locator)->ice_locatorCacheTimeout(0)->ice_ping();
            comm->stringToProxy("test")->ice_locator(slave2Locator)->ice_locatorCacheTimeout(0)->ice_ping();
        }
        catch(const Ice::LocalException& ex)
        {
            cerr << ex << endl;
            test(false);
        }

        slave2Admin->stopServer("Server");

        masterAdmin->removeApplication("TestApp");
    }
    cout << "ok" << endl;

    cout << "testing master upgrade... " << flush;
    {
        ApplicationDescriptor app;
        app.name = "TestApp";
        app.description = "added application";

        ServerDescriptorPtr server = new ServerDescriptor();
        server->id = "Server";
        server->exe = comm->getProperties()->getProperty("ServerDir") + "/server";
        server->pwd = ".";
        server->applicationDistrib = false;
        server->allocatable = false;
        addProperty(server, "Ice.Admin.Endpoints", "tcp -h 127.0.0.1");
        server->activation = "on-demand";
        AdapterDescriptor adapter;
        adapter.name = "TestAdapter";
        adapter.id = "TestAdapter.Server";
        adapter.serverLifetime = true;
        adapter.registerProcess = false;
        PropertyDescriptor property;
        property.name = "TestAdapter.Endpoints";
        property.value = "default";
        server->propertySet.properties.push_back(property);
        property.name = "Identity";
        property.value = "test";
        server->propertySet.properties.push_back(property);
        ObjectDescriptor object;
        object.id = Ice::stringToIdentity("test");
        object.type = "::Test::TestIntf";
        adapter.objects.push_back(object);
        server->adapters.push_back(adapter);
        app.nodes["Node1"].servers.push_back(server);

        masterAdmin->addApplication(app);

        comm->stringToProxy("test")->ice_locator(masterLocator)->ice_locatorCacheTimeout(0)->ice_ping();
        comm->stringToProxy("test")->ice_locator(slave1Locator)->ice_locatorCacheTimeout(0)->ice_ping();
        comm->stringToProxy("test")->ice_locator(slave2Locator)->ice_locatorCacheTimeout(0)->ice_ping();
        masterAdmin->stopServer("Server");

        //
        // Shutdown the Master, update Slave1 to be the Master.
        //
        masterAdmin->shutdown();
        waitForServerState(admin, "Master", false);
        slave1Admin->shutdown();
        waitForServerState(admin, "Slave1", false);

        params.clear();
        params["id"] = "Slave1";
        params["port"] = "12051";
        params["replicaName"] = "Master";
        instantiateServer(admin, "IceGridRegistry", params);

        admin->startServer("Slave1");
        slave1Locator =
            Ice::LocatorPrx::uncheckedCast(comm->stringToProxy("RepTestIceGrid/Locator-Master:default -p 12051"));
        slave1Admin = createAdminSession(slave1Locator, "");

        waitForReplicaState(slave1Admin, "Slave2", true);

        ApplicationUpdateDescriptor update;
        update.name = "TestApp";
        NodeUpdateDescriptor node;
        node.name = "Node1";
        node.servers.push_back(server);
        update.nodes.push_back(node);
        property.name = "Dummy";
        property.value = "val";
        server->propertySet.properties.push_back(property);
        slave1Admin->updateApplication(update);

        comm->stringToProxy("test")->ice_locator(slave1Locator)->ice_locatorCacheTimeout(0)->ice_ping();
        comm->stringToProxy("test")->ice_locator(slave2Locator)->ice_locatorCacheTimeout(0)->ice_ping();

        slave1Admin->shutdown();
        waitForServerState(admin, "Slave1", false);

        params.clear();
        params["id"] = "Slave1";
        params["replicaName"] = "Slave1";
        params["port"] = "12051";
        instantiateServer(admin, "IceGridRegistry", params);

        params.clear();
        params["id"] = "Master";
        params["replicaName"] = "";
        params["port"] = "12050";
        params["arg"] = "--initdb-from-replica=Slave2";
        instantiateServer(admin, "IceGridRegistry", params);

        admin->startServer("Master");
        masterAdmin = createAdminSession(masterLocator, "");

        admin->startServer("Slave1");
        slave1Locator =
            Ice::LocatorPrx::uncheckedCast(comm->stringToProxy("RepTestIceGrid/Locator-Slave1:default -p 12051"));
        slave1Admin = createAdminSession(slave1Locator, "Slave1");

        comm->stringToProxy("test")->ice_locator(masterLocator)->ice_locatorCacheTimeout(0)->ice_ping();
        comm->stringToProxy("test")->ice_locator(slave1Locator)->ice_locatorCacheTimeout(0)->ice_ping();
        comm->stringToProxy("test")->ice_locator(slave2Locator)->ice_locatorCacheTimeout(0)->ice_ping();

        masterAdmin->stopServer("Server");

        waitForReplicaState(masterAdmin, "Slave1", true);
        waitForReplicaState(masterAdmin, "Slave2", true);

        ApplicationInfo info = masterAdmin->getApplicationInfo("TestApp");
        test(info.revision == 2);

        masterAdmin->removeApplication("TestApp");
    }
    cout << "ok" << endl;

    cout << "testing interop with registry and node using the 1.0 encoding... " << flush;
    {
        params.clear();
        params["id"] = "Slave3";
        params["replicaName"] = "Slave3";
        params["port"] = "12053";
        params["encoding"] = "1.0";
        instantiateServer(admin, "IceGridRegistry", params);

        params.clear();
        params["id"] = "Node2";
        params["encoding"] = "1.0";
        instantiateServer(admin, "IceGridNode", params);

        admin->startServer("Slave3");
        waitForServerState(admin, "Slave3", true);
        waitForReplicaState(masterAdmin, "Slave3", true);

        admin->startServer("Node2");
        waitForNodeState(masterAdmin, "Node2", true);

        Ice::LocatorPrx slave3Locator =
            Ice::LocatorPrx::uncheckedCast(
                comm->stringToProxy("RepTestIceGrid/Locator-Slave3 -e 1.0:default -p 12053"));
        IceGrid::AdminPrx slave3Admin = createAdminSession(slave3Locator, "Slave3");
        waitForNodeState(slave3Admin, "Node2", true);

        ApplicationDescriptor app;
        app.name = "TestApp";
        app.description = "added application";

        ServerDescriptorPtr server = new ServerDescriptor();
        server->id = "Server";
        server->exe = comm->getProperties()->getProperty("ServerDir") + "/server";
        server->pwd = ".";
        server->applicationDistrib = false;
        server->allocatable = false;
        addProperty(server, "Ice.Admin.Endpoints", "tcp -h 127.0.0.1");
        server->activation = "on-demand";
        AdapterDescriptor adapter;
        adapter.name = "TestAdapter";
        adapter.id = "TestAdapter.Server";
        adapter.serverLifetime = true;
        adapter.registerProcess = false;
        PropertyDescriptor property;
        property.name = "TestAdapter.Endpoints";
        property.value = "default";
        server->propertySet.properties.push_back(property);
        property.name = "Identity";
        property.value = "test";
        server->propertySet.properties.push_back(property);
        ObjectDescriptor object;
        object.id = Ice::stringToIdentity("test");
        object.type = "::Test::TestIntf";
        adapter.objects.push_back(object);
        server->adapters.push_back(adapter);
        app.nodes["Node2"].servers.push_back(server);

        masterAdmin->addApplication(app);

        comm->stringToProxy("test -e 1.0")->ice_locator(
            masterLocator->ice_encodingVersion(Ice::Encoding_1_0))->ice_locatorCacheTimeout(0)->ice_ping();
        comm->stringToProxy("test -e 1.0")->ice_locator(
            slave1Locator->ice_encodingVersion(Ice::Encoding_1_0))->ice_locatorCacheTimeout(0)->ice_ping();
        comm->stringToProxy("test -e 1.0")->ice_locator(slave3Locator)->ice_locatorCacheTimeout(0)->ice_ping();
        masterAdmin->stopServer("Server");

    }
    cout << "ok" << endl;


    slave1Admin->shutdownNode("Node1");
    removeServer(admin, "Node1");

    removeServer(admin, "Slave2");

    slave1Admin->shutdown();
    removeServer(admin, "Slave1");
    masterAdmin->shutdown();
    removeServer(admin, "Master");
}