Example #1
0
void
allTestsWithTarget(const Ice::CommunicatorPtr& comm)
{
    RegistryPrx registry = IceGrid::RegistryPrx::checkedCast(
        comm->stringToProxy(comm->getDefaultLocator()->ice_getIdentity().category + "/Registry"));
    test(registry);
    AdminSessionPrx session = registry->createAdminSession("foo", "bar");

    session->ice_getConnection()->setACM(registry->getACMTimeout(), IceUtil::None, Ice::HeartbeatOnIdle);

    AdminPrx admin = session->getAdmin();
    test(admin);

    cout << "testing targets... " << flush;

    TestIntfPrx obj = TestIntfPrx::checkedCast(comm->stringToProxy("*****@*****.**"));
    obj = TestIntfPrx::checkedCast(comm->stringToProxy("[email protected]"));
    obj = TestIntfPrx::checkedCast(comm->stringToProxy("[email protected]"));

    obj = TestIntfPrx::checkedCast(comm->stringToProxy("[email protected]"));

    obj = TestIntfPrx::checkedCast(comm->stringToProxy("*****@*****.**"));
    test(obj->getProperty("TargetProp") == "1");

    cout << "ok" << endl;

    session->destroy();
}
Example #2
0
 StressClient(int id, const RegistryPrx& registry, bool destroySession) :
     _communicator(registry->ice_getCommunicator()),
     _id(id),
     _registry(registry),
     _notified(false),
     _terminated(false),
     _destroySession(destroySession)
 {
 }
Example #3
0
void 
allTests(const Ice::CommunicatorPtr& communicator)
{
    RegistryPrx registry = IceGrid::RegistryPrx::checkedCast(communicator->stringToProxy("IceGrid/Registry"));
    test(registry);
    AdminSessionPrx session = registry->createAdminSession("foo", "bar");

    SessionKeepAliveThreadPtr keepAlive = new SessionKeepAliveThread(session, registry->getSessionTimeout()/2);
    keepAlive->start();

    AdminPrx admin = session->getAdmin();
    test(admin);
    
    cout << "testing distributions... " << flush;
    {
        admin->startServer("Test.IcePatch2");
        admin->startServer("IcePatch2-Direct");

        TestIntfPrx test;
        test = TestIntfPrx::uncheckedCast(communicator->stringToProxy("server-all"));
        test(test->getServerFile("rootfile") == "");

        try
        {
            admin->patchServer("server-all", true);
        }
        catch(const PatchException& ex)
        {
            copy(ex.reasons.begin(), ex.reasons.end(), ostream_iterator<string>(cerr, "\n"));
            test(false);
        }

        test(test->getServerFile("rootfile") == "rootfile");
        test(test->getServerFile("dir1/file1") == "dummy-file1");
        test(test->getServerFile("dir1/file2") == "dummy-file2");
        test(test->getServerFile("dir2/file3") == "dummy-file3");

        test(test->getApplicationFile("rootfile") == "");
        test(test->getApplicationFile("dir1/file1") == "");
        test(test->getApplicationFile("dir1/file2") == "");
        test(test->getApplicationFile("dir2/file3") == "dummy-file3");

        test = TestIntfPrx::uncheckedCast(communicator->stringToProxy("server-all-direct"));

        test(test->getServerFile("rootfile") == "");
        test(test->getServerFile("dir1/file1") == "");
        test(test->getServerFile("dir1/file2") == "");
        test(test->getServerFile("dir2/file3") == "");

        test(test->getApplicationFile("rootfile") == "");
        test(test->getApplicationFile("dir1/file1") == "");
        test(test->getApplicationFile("dir1/file2") == "");
        test(test->getApplicationFile("dir2/file3") == "dummy-file3");

        try
        {
            admin->patchServer("server-all-direct", true);
        }
        catch(const PatchException& ex)
        {
            copy(ex.reasons.begin(), ex.reasons.end(), ostream_iterator<string>(cerr, "\n"));
            test(false);
        }

        test(test->getServerFile("rootfile") == "rootfile");
        test(test->getServerFile("dir1/file1") == "dummy-file1");
        test(test->getServerFile("dir1/file2") == "dummy-file2");
        test(test->getServerFile("dir2/file3") == "dummy-file3");

        test(test->getApplicationFile("rootfile") == "");
        test(test->getApplicationFile("dir1/file1") == "");
        test(test->getApplicationFile("dir1/file2") == "");
        test(test->getApplicationFile("dir2/file3") == "dummy-file3");

        try
        {
            admin->patchApplication("Test", true);
        }
        catch(const PatchException& ex)
        {
            copy(ex.reasons.begin(), ex.reasons.end(), ostream_iterator<string>(cerr, "\n"));
            test(false);
        }
        test = TestIntfPrx::uncheckedCast(communicator->stringToProxy("server-dir1"));

        test(test->getServerFile("rootfile") == "");
        test(test->getServerFile("dir1/file1") == "dummy-file1");
        test(test->getServerFile("dir1/file2") == "dummy-file2");
        test(test->getServerFile("dir2/file3") == "");

        test(test->getApplicationFile("rootfile") == "");
        test(test->getApplicationFile("dir1/file1") == "");
        test(test->getApplicationFile("dir1/file2") == "");
        test(test->getApplicationFile("dir2/file3") == "dummy-file3");

        admin->stopServer("Test.IcePatch2");
        admin->stopServer("IcePatch2-Direct");
    }
    cout << "ok" << endl;

    cout << "testing distributions after update... " << flush;
    {
        ApplicationUpdateDescriptor update;
        update.name = "Test";
        update.variables["icepatch.directory"] = "${test.dir}/data/updated";
        admin->updateApplication(update);
        
        admin->startServer("Test.IcePatch2");
        admin->startServer("IcePatch2-Direct");

        TestIntfPrx test;
        test = TestIntfPrx::uncheckedCast(communicator->stringToProxy("server-all"));
        test(test->getServerFile("rootfile") == "rootfile");

        try
        {
            admin->patchServer("server-all", true);
        }
        catch(const PatchException& ex)
        {
            copy(ex.reasons.begin(), ex.reasons.end(), ostream_iterator<string>(cerr, "\n"));
            test(false);
        }
 
        test(test->getServerFile("rootfile") == "rootfile-updated!");
        test(test->getServerFile("dir1/file1") == "");
        test(test->getServerFile("dir1/file2") == "dummy-file2-updated!");
        test(test->getServerFile("dir2/file3") == "dummy-file3");
        test(test->getServerFile("dir2/file4") == "dummy-file4");

        test(test->getApplicationFile("rootfile") == "");
        test(test->getApplicationFile("dir1/file1") == "");
        test(test->getApplicationFile("dir1/file2") == "");
        test(test->getApplicationFile("dir2/file3") == "dummy-file3");
        test(test->getApplicationFile("dir2/file4") == "dummy-file4");

        try
        {
            admin->patchServer("server-all-direct", true);
        }
        catch(const PatchException& ex)
        {
            copy(ex.reasons.begin(), ex.reasons.end(), ostream_iterator<string>(cerr, "\n"));
            test(false);
        }
        test = TestIntfPrx::uncheckedCast(communicator->stringToProxy("server-all-direct"));

        test(test->getServerFile("rootfile") == "rootfile-updated!");
        test(test->getServerFile("dir1/file1") == "");
        test(test->getServerFile("dir1/file2") == "dummy-file2-updated!");
        test(test->getServerFile("dir2/file3") == "dummy-file3");
        test(test->getServerFile("dir2/file4") == "dummy-file4");

        test(test->getApplicationFile("rootfile") == "");
        test(test->getApplicationFile("dir1/file1") == "");
        test(test->getApplicationFile("dir1/file2") == "");
        test(test->getApplicationFile("dir2/file3") == "dummy-file3");
        test(test->getApplicationFile("dir2/file4") == "dummy-file4");

        try
        {
            admin->patchApplication("Test", true);
        }
        catch(const PatchException& ex)
        {
            copy(ex.reasons.begin(), ex.reasons.end(), ostream_iterator<string>(cerr, "\n"));
            test(false);
        }
        test = TestIntfPrx::uncheckedCast(communicator->stringToProxy("server-dir1"));

        test(test->getServerFile("rootfile") == "");
        test(test->getServerFile("dir1/file1") == "");
        test(test->getServerFile("dir1/file2") == "dummy-file2-updated!");
        test(test->getServerFile("dir2/file3") == "");
        test(test->getServerFile("dir2/file4") == "");

        test(test->getApplicationFile("rootfile") == "");
        test(test->getApplicationFile("dir1/file1") == "");
        test(test->getApplicationFile("dir1/file2") == "");
        test(test->getApplicationFile("dir2/file3") == "dummy-file3");
        test(test->getApplicationFile("dir2/file4") == "dummy-file4");

        admin->stopServer("Test.IcePatch2");
        admin->stopServer("IcePatch2-Direct");
    }
    cout << "ok" << endl;

    cout << "testing application distrib configuration... " << flush;
    try
    {
        ApplicationDescriptor app = admin->getApplicationInfo("Test").descriptor;
        admin->removeApplication("Test");
        
        app.variables["icepatch.directory"] = "${test.dir}/data/original";
        test(app.nodes["localnode"].servers[2]->id == "server-dir1");
        app.nodes["localnode"].servers[2]->applicationDistrib = false;
        
        admin->addApplication(app);
        admin->startServer("Test.IcePatch2");

        try
        {
            admin->patchServer("server-dir1", true);
        }
        catch(const PatchException& ex)
        {
            copy(ex.reasons.begin(), ex.reasons.end(), ostream_iterator<string>(cerr, "\n"));
            test(false);
        }

        TestIntfPrx test = TestIntfPrx::uncheckedCast(communicator->stringToProxy("server-dir1"));

        test(test->getServerFile("rootfile") == "");
        test(test->getServerFile("dir1/file1") == "dummy-file1");
        test(test->getServerFile("dir1/file2") == "dummy-file2");
        test(test->getServerFile("dir2/file3") == "");

        test(test->getApplicationFile("rootfile") == "");
        test(test->getApplicationFile("dir1/file1") == "");
        test(test->getApplicationFile("dir1/file2") == "");
        test(test->getApplicationFile("dir2/file3") == "");

        admin->removeApplication("Test");

        admin->addApplication(app);
        admin->startServer("Test.IcePatch2");
        admin->startServer("IcePatch2-Direct");
        
        try
        {
            admin->patchApplication("Test", true);
        }
        catch(const PatchException& ex)
        {
            copy(ex.reasons.begin(), ex.reasons.end(), ostream_iterator<string>(cerr, "\n"));
            test(false);
        }

        test = TestIntfPrx::uncheckedCast(communicator->stringToProxy("server-dir1"));

        test(test->getServerFile("rootfile") == "");
        test(test->getServerFile("dir1/file1") == "dummy-file1");
        test(test->getServerFile("dir1/file2") == "dummy-file2");
        test(test->getServerFile("dir2/file3") == "");

        test(test->getApplicationFile("rootfile") == "");
        test(test->getApplicationFile("dir1/file1") == "");
        test(test->getApplicationFile("dir1/file2") == "");
        test(test->getApplicationFile("dir2/file3") == "dummy-file3");

        admin->removeApplication("Test");

        app.distrib.icepatch = "";

        admin->addApplication(app);
        admin->startServer("Test.IcePatch2");
        admin->startServer("IcePatch2-Direct");
        
        try
        {
            admin->patchServer("server-dir1", true);
        }
        catch(const PatchException& ex)
        {
            copy(ex.reasons.begin(), ex.reasons.end(), ostream_iterator<string>(cerr, "\n"));
            test(false);
        }

        test = TestIntfPrx::uncheckedCast(communicator->stringToProxy("server-dir1"));

        test(test->getServerFile("rootfile") == "");
        test(test->getServerFile("dir1/file1") == "dummy-file1");
        test(test->getServerFile("dir1/file2") == "dummy-file2");
        test(test->getServerFile("dir2/file3") == "");

        test(test->getApplicationFile("rootfile") == "");
        test(test->getApplicationFile("dir1/file1") == "");
        test(test->getApplicationFile("dir1/file2") == "");
        test(test->getApplicationFile("dir2/file3") == "");

        test = TestIntfPrx::uncheckedCast(communicator->stringToProxy("server-all"));

        test(test->getServerFile("rootfile") == "");
        test(test->getServerFile("dir1/file1") == "");
        test(test->getServerFile("dir1/file2") == "");
        test(test->getServerFile("dir2/file3") == "");

        try
        {
            admin->patchApplication("Test", true);
        }
        catch(const PatchException& ex)
        {
            copy(ex.reasons.begin(), ex.reasons.end(), ostream_iterator<string>(cerr, "\n"));
            test(false);
        }

        test(test->getServerFile("rootfile") == "rootfile");
        test(test->getServerFile("dir1/file1") == "dummy-file1");
        test(test->getServerFile("dir1/file2") == "dummy-file2");
        test(test->getServerFile("dir2/file3") == "dummy-file3");
    }
    catch(const DeploymentException& ex)
    {
        cerr << ex << ":\n" << ex.reason << endl;
        test(false);
    }
    cout << "ok" << endl;

    keepAlive->destroy();
    keepAlive->getThreadControl().join();
    keepAlive = 0;

    session->destroy();
}
Example #4
0
void 
allTests(const Ice::CommunicatorPtr& communicator)
{
    SessionKeepAliveThreadPtr keepAlive = new SessionKeepAliveThread(
        communicator->getLogger(), IceUtil::Time::seconds(5));
    keepAlive->start();

    RegistryPrx registry = IceGrid::RegistryPrx::checkedCast(communicator->stringToProxy("IceGrid/Registry"));
    test(registry);
    AdminSessionPrx session = registry->createAdminSession("foo", "bar");

    keepAlive->add(session);

    AdminPrx admin = session->getAdmin();
    test(admin);

    cout << "starting router... " << flush;
    try
    {
        admin->startServer("Glacier2");
    }
    catch(const ServerStartException& ex)
    {
        cerr << ex.reason << endl;
        test(false);
    }
    cout << "ok" << endl;

    const int allocationTimeout = 5000;

    Ice::ObjectPrx obj;
    Ice::ObjectPrx dummy;

    try
    {
        cout << "testing create session... " << flush;
        SessionPrx session1 = registry->createSession("Client1", "");
        SessionPrx session2 = registry->createSession("Client2", "");
        
        keepAlive->add(session1);
        keepAlive->add(session2);
    
        cout << "ok" << endl;

        cout << "testing allocate object by identity... " << flush;

        Ice::Identity allocatable = communicator->stringToIdentity("allocatable");
        Ice::Identity allocatablebis = communicator->stringToIdentity("allocatablebis");

        try
        {
            session1->allocateObjectById(communicator->stringToIdentity("dummy"));
        }
        catch(const ObjectNotRegisteredException&)
        {
        }
        try
        {
            session1->releaseObject(communicator->stringToIdentity("dummy"));
        }
        catch(const ObjectNotRegisteredException&)
        {
        }

        try
        {
            session1->allocateObjectById(communicator->stringToIdentity("nonallocatable"));
            test(false);
        }
        catch(const AllocationException&)
        {
            test(false);
        }
        catch(const ObjectNotRegisteredException&)
        {
        }

        try
        {
            session2->allocateObjectById(communicator->stringToIdentity("nonallocatable"));
            test(false);
        }
        catch(const AllocationException&)
        {
            test(false);
        }
        catch(const ObjectNotRegisteredException&)
        {
        }

        try
        {
            session1->releaseObject(communicator->stringToIdentity("nonallocatable"));
            test(false);
        }
        catch(const AllocationException&)
        {
            test(false);
        }
        catch(const ObjectNotRegisteredException&)
        {
        }

        try
        {
            session2->releaseObject(communicator->stringToIdentity("nonallocatable"));
            test(false);
        }
        catch(const AllocationException&)
        {
            test(false);
        }
        catch(const ObjectNotRegisteredException&)
        {
        }

        session1->allocateObjectById(allocatable);
        try
        {
            session1->allocateObjectById(allocatable);
            test(false);
        }
        catch(const AllocationException&)
        {
        }

        session1->setAllocationTimeout(0);
        session2->setAllocationTimeout(0);

        try
        {
            session2->allocateObjectById(allocatable);
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        try
        {
            session2->releaseObject(allocatable);
            test(false);
        }
        catch(const AllocationException&)
        {
        }

        session1->allocateObjectById(allocatablebis);
        try
        {
            session2->allocateObjectById(allocatablebis);
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        session1->releaseObject(allocatablebis);
        session2->allocateObjectById(allocatablebis);
        try
        {
            session1->allocateObjectById(allocatablebis);
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        session2->releaseObject(allocatablebis);
    
        session2->setAllocationTimeout(allocationTimeout);
        AllocateObjectByIdCallbackPtr cb1 = new AllocateObjectByIdCallback();
        session2->allocateObjectById_async(cb1, allocatable);
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
        test(!cb1->hasResponse(dummy));
        session1->releaseObject(allocatable);
        cb1->waitResponse(__FILE__, __LINE__);
        test(cb1->hasResponse(dummy));

        session1->setAllocationTimeout(0);
        try
        {
            session1->allocateObjectById(allocatable);
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        try
        {
            session1->releaseObject(allocatable);
            test(false);
        }
        catch(const AllocationException&)
        {
        }
        session1->setAllocationTimeout(allocationTimeout);
        cb1 = new AllocateObjectByIdCallback();
        session1->allocateObjectById_async(cb1, allocatable);
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
        test(!cb1->hasResponse(dummy));
        session2->releaseObject(allocatable);
        cb1->waitResponse(__FILE__, __LINE__);
        test(cb1->hasResponse(dummy));

        session1->releaseObject(allocatable);

        cout << "ok" << endl;

        cout << "testing allocate object by type... " << flush;
    
        session1->setAllocationTimeout(0);
        session2->setAllocationTimeout(0);


        try
        {
            obj = session1->allocateObjectByType("::Unknown");
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
            test(false);
        }
        catch(const AllocationException&)
        {
        }

        try
        {
            obj = session1->allocateObjectByType("::NotAllocatable");
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
            test(false);
        }
        catch(const AllocationException&)
        {
        }

        obj = session1->allocateObjectByType("::Test");
        test(obj && obj->ice_getIdentity().name == "allocatable");
        try
        {
            session1->allocateObjectByType("::Test");
            test(false);
        }
        catch(const AllocationException&)
        {
        }
        try
        {
            session2->allocateObjectByType("::Test");
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        try
        {
            session2->releaseObject(obj->ice_getIdentity());
        }
        catch(const AllocationException&)
        {
        }

        session1->releaseObject(obj->ice_getIdentity());
        try
        {
            session1->releaseObject(obj->ice_getIdentity());
        }
        catch(const AllocationException&)
        {
        }

        obj = session2->allocateObjectByType("::Test"); // Allocate the object
        test(obj && obj->ice_getIdentity().name == "allocatable");
        try
        {
            session2->allocateObjectByType("::Test");
            test(false);
        }
        catch(const AllocationException&)
        {
        }
        try
        {
            session1->allocateObjectByType("::Test");
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        session1->allocateObjectByType("::TestBis");
        try
        {
            session2->allocateObjectByType("::TestBis");
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        session1->releaseObject(allocatablebis);
        session2->allocateObjectByType("::TestBis");
        try
        {
            session1->allocateObjectByType("::TestBis");
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        session2->releaseObject(allocatablebis);

        session1->setAllocationTimeout(allocationTimeout);
        AllocateObjectByTypeCallbackPtr cb3 = new AllocateObjectByTypeCallback();
        session1->allocateObjectByType_async(cb3, "::Test");
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
        test(!cb3->hasResponse(dummy));
        session2->releaseObject(obj->ice_getIdentity());
        cb3->waitResponse(__FILE__, __LINE__);
        test(cb3->hasResponse(obj));

        session1->releaseObject(obj->ice_getIdentity());
    
        cout << "ok" << endl;

        cout << "testing object allocation timeout... " << flush;    

        session1->allocateObjectById(allocatable);
        IceUtil::Time time = IceUtil::Time::now();
        session2->setAllocationTimeout(500);
        try
        {
            session2->allocateObjectById(allocatable);
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
            test(time + IceUtil::Time::milliSeconds(100) < IceUtil::Time::now());
        }
        time = IceUtil::Time::now();
        try
        {
            session2->allocateObjectById(allocatable);
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        test(time + IceUtil::Time::milliSeconds(100) < IceUtil::Time::now());
        time = IceUtil::Time::now();
        try
        {
            session2->allocateObjectByType("::Test");
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        test(time + IceUtil::Time::milliSeconds(100) < IceUtil::Time::now());

        session1->releaseObject(allocatable);
        session2->setAllocationTimeout(0);

        cout << "ok" << endl;

        cout << "testing server allocation... " << flush;

        session1->setAllocationTimeout(0);
        session2->setAllocationTimeout(0);

        Ice::Identity allocatable3 = communicator->stringToIdentity("allocatable3");
        Ice::Identity allocatable4 = communicator->stringToIdentity("allocatable4");

        session1->allocateObjectById(allocatable3);
        try
        {
            session2->allocateObjectById(allocatable3);
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        try
        {
            session2->allocateObjectById(allocatable4);
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }

        session1->allocateObjectById(allocatable4);
        session1->releaseObject(allocatable3);
        try
        {
            session2->allocateObjectById(allocatable3);
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        session1->releaseObject(allocatable4);
        session2->allocateObjectById(allocatable3);
        try
        {
            session1->allocateObjectById(allocatable3);
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        try
        {
            session1->allocateObjectById(allocatable4);
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        session2->allocateObjectById(allocatable4);
        session2->releaseObject(allocatable3);
        try
        {
            session1->allocateObjectById(allocatable3);
            test(false);
        }
        catch(const AllocationTimeoutException&)
        {
        }
        try
        {
            session1->allocateObjectByType("::TestServer1");
            test(false);
        }
        catch(AllocationException&)
        {
        }
        try
        {
            session1->allocateObjectByType("::TestServer2");
            test(false);
        }
        catch(AllocationException&)
        {
        }
        test(session2->allocateObjectByType("::TestServer1"));
        try
        {
            session2->allocateObjectByType("::TestServer1");
            test(false);
        }
        catch(AllocationException&)
        {
        }
        try
        {
            session2->allocateObjectByType("::TestServer2");
            test(false);
        }
        catch(AllocationException&)
        {
        }
        session2->releaseObject(allocatable3);
        session2->releaseObject(allocatable4);

        session1->allocateObjectById(allocatable3);
        session1->allocateObjectById(allocatable4);

        session2->setAllocationTimeout(allocationTimeout);
        cb1 = new AllocateObjectByIdCallback();
        session2->allocateObjectById_async(cb1, allocatable3);
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
        test(!cb1->hasResponse(dummy));
        session1->releaseObject(allocatable3);
        test(!cb1->hasResponse(dummy));
        session1->releaseObject(allocatable4);
        cb1->waitResponse(__FILE__, __LINE__);
        test(cb1->hasResponse(dummy));
        session2->releaseObject(allocatable3);

        session1->setAllocationTimeout(allocationTimeout);
        test(session2->allocateObjectByType("::TestServer1"));
        cb3 = new AllocateObjectByTypeCallback();
        session1->allocateObjectByType_async(cb3, "::TestServer2");
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
        test(!cb3->hasResponse(dummy));
        session2->releaseObject(allocatable3);
        cb3->waitResponse(__FILE__, __LINE__);
        test(cb3->hasResponse(dummy));
        session1->releaseObject(allocatable4);    

        session1->setAllocationTimeout(0);
        session2->setAllocationTimeout(0);
        test(session1->allocateObjectByType("::TestMultipleByServer"));
        try
        {
            session2->allocateObjectByType("::TestMultipleByServer");
            test(false);
        }
        catch(AllocationException&)
        {
        }
        test(session1->allocateObjectByType("::TestMultipleByServer"));
        session1->releaseObject(communicator->stringToIdentity("allocatable31"));
        session1->releaseObject(communicator->stringToIdentity("allocatable41"));
        test(session2->allocateObjectByType("::TestMultipleByServer"));
        try
        {
            session1->allocateObjectByType("::TestMultipleByServer");
            test(false);
        }
        catch(AllocationException&)
        {
        }
        test(session2->allocateObjectByType("::TestMultipleByServer"));
        session2->releaseObject(communicator->stringToIdentity("allocatable31"));
        session2->releaseObject(communicator->stringToIdentity("allocatable41"));

        Ice::ObjectPrx obj1 = session1->allocateObjectByType("::TestMultipleServer");
        test(obj1);
        Ice::ObjectPrx obj2 = session2->allocateObjectByType("::TestMultipleServer");
        test(obj2);
        try
        {
            session1->allocateObjectByType("::TestMultipleServer");
            test(false);
        }
        catch(AllocationTimeoutException&)
        {
        }
        try
        {
            session2->allocateObjectByType("::TestMultipleServer");
            test(false);
        }
        catch(AllocationTimeoutException&)
        {
        }       
        session1->releaseObject(obj1->ice_getIdentity());
        obj1 = session2->allocateObjectByType("::TestMultipleServer");
        session2->releaseObject(obj1->ice_getIdentity());
        session2->releaseObject(obj2->ice_getIdentity());

        cout << "ok" << endl;

        cout << "testing concurrent allocations... " << flush;

        session1->setAllocationTimeout(allocationTimeout);
        session2->setAllocationTimeout(allocationTimeout);

        session2->allocateObjectById(allocatable);
        AllocateObjectByIdCallbackPtr cb11 = new AllocateObjectByIdCallback();
        AllocateObjectByIdCallbackPtr cb12 = new AllocateObjectByIdCallback();
        session1->allocateObjectById_async(cb11, allocatable);
        session1->allocateObjectById_async(cb12, allocatable);
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
        test(!cb11->hasResponse(dummy));
        test(!cb12->hasResponse(dummy));
        session2->releaseObject(allocatable);
        cb11->waitResponse(__FILE__, __LINE__);
        cb12->waitResponse(__FILE__, __LINE__);
        test(cb11->hasResponse(dummy) ? cb12->hasException() : cb12->hasResponse(dummy));
        test(cb12->hasResponse(dummy) ? cb11->hasException() : cb11->hasResponse(dummy));
        session1->releaseObject(allocatable);

        session2->allocateObjectById(allocatable);
        AllocateObjectByTypeCallbackPtr cb31 = new AllocateObjectByTypeCallback();
        AllocateObjectByTypeCallbackPtr cb32 = new AllocateObjectByTypeCallback();
        session1->allocateObjectByType_async(cb31, "::Test");
        session1->allocateObjectByType_async(cb32, "::Test");
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
        test(!cb31->hasResponse(dummy));
        test(!cb32->hasResponse(dummy));
        session2->releaseObject(allocatable);
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(300));
        do
        {
            IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(200));
        }
        while(!cb31->hasResponse(dummy) && !cb32->hasResponse(dummy));
        test(cb31->hasResponse(dummy) && dummy && !cb32->hasResponse(dummy) ||
             cb32->hasResponse(dummy) && dummy && !cb31->hasResponse(dummy));
        session1->releaseObject(allocatable);
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(300));
        AllocateObjectByTypeCallbackPtr cb33 = cb31->hasResponse(dummy) ? cb32 : cb31;
        cb33->waitResponse(__FILE__, __LINE__);
        test(cb33->hasResponse(dummy) && dummy);
        session1->releaseObject(allocatable);

        session2->allocateObjectById(allocatable3);
        cb11 = new AllocateObjectByIdCallback();
        cb12 = new AllocateObjectByIdCallback();
        session1->allocateObjectById_async(cb11, allocatable3);
        session1->allocateObjectById_async(cb12, allocatable3);
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
        test(!cb11->hasResponse(dummy));
        test(!cb12->hasResponse(dummy));
        session2->releaseObject(allocatable3);
        cb11->waitResponse(__FILE__, __LINE__);
        cb12->waitResponse(__FILE__, __LINE__);
        test(cb11->hasResponse(dummy) ? cb12->hasException() : cb12->hasResponse(dummy));
        test(cb12->hasResponse(dummy) ? cb11->hasException() : cb11->hasResponse(dummy));
        session1->releaseObject(allocatable3);

        session2->allocateObjectById(allocatable3);
        cb31 = new AllocateObjectByTypeCallback();
        cb32 = new AllocateObjectByTypeCallback();
        session1->allocateObjectByType_async(cb31, "::TestServer1");
        session1->allocateObjectByType_async(cb32, "::TestServer1");
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
        test(!cb31->hasResponse(dummy));
        test(!cb32->hasResponse(dummy));
        session2->releaseObject(allocatable3);
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(300));
        do
        {
            IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(200));
        }
        while(!cb31->hasResponse(dummy) && !cb32->hasResponse(dummy));
        test(cb31->hasResponse(dummy) && dummy && !cb32->hasResponse(dummy) ||
             cb32->hasResponse(dummy) && dummy && !cb31->hasResponse(dummy));
        session1->releaseObject(allocatable3);
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(300));
        cb33 = cb31->hasResponse(dummy) ? cb32 : cb31;
        cb33->waitResponse(__FILE__, __LINE__);
        test(cb33->hasResponse(dummy) && dummy);
        session1->releaseObject(allocatable3);

        session1->allocateObjectById(allocatable3);
        cb31 = new AllocateObjectByTypeCallback();
        cb32 = new AllocateObjectByTypeCallback();
        session1->allocateObjectByType_async(cb31, "::TestServer1");
        session1->allocateObjectByType_async(cb32, "::TestServer1");
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
        test(!cb31->hasResponse(dummy));
        test(!cb32->hasResponse(dummy));
        session1->releaseObject(allocatable3);
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(300));
        do
        {
            IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(200));
        }
        while(!cb31->hasResponse(dummy) && !cb32->hasResponse(dummy));
        test(cb31->hasResponse(dummy) && dummy && !cb32->hasResponse(dummy) ||
             cb32->hasResponse(dummy) && dummy && !cb31->hasResponse(dummy));
        session1->releaseObject(allocatable3);
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(300));
        cb33 = cb31->hasResponse(dummy) ? cb32 : cb31;
        cb33->waitResponse(__FILE__, __LINE__);
        test(cb33->hasResponse(dummy) && dummy);
        session1->releaseObject(allocatable3);

        cout << "ok" << endl;

        cout << "testing session destroy... " << flush;

        obj = session2->allocateObjectByType("::Test"); // Allocate the object
        test(obj && obj->ice_getIdentity().name == "allocatable");

        session1->setAllocationTimeout(allocationTimeout);
        cb3 = new AllocateObjectByTypeCallback();
        session1->allocateObjectByType_async(cb3, "::Test");
        IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
        test(!cb3->hasResponse(dummy));
        session2->destroy();    
        cb3->waitResponse(__FILE__, __LINE__);
        test(cb3->hasResponse(obj));
        session1->destroy();

        session2 = SessionPrx::uncheckedCast(registry->createSession("Client2", ""));
        session2->setAllocationTimeout(0);
        session2->allocateObjectById(allocatable);
        session2->destroy();

        cout << "ok" << endl;

        cout << "testing allocation with Glacier2 session... " << flush;
        Ice::ObjectPrx routerBase = communicator->stringToProxy("Glacier2/router:default -p 12347");
        Glacier2::RouterPrx router1 = Glacier2::RouterPrx::checkedCast(routerBase->ice_connectionId("client1"));
        test(router1);
        
        Glacier2::SessionPrx sessionBase = router1->createSession("test1", "abc123");
        try
        {
            session1 = IceGrid::SessionPrx::checkedCast(sessionBase->ice_connectionId("client1")->ice_router(router1));
            test(session1);
            session1->ice_ping();

            Ice::ObjectPrx obj;
            obj = session1->allocateObjectById(allocatable)->ice_connectionId("client1")->ice_router(router1);
            obj->ice_ping();
            session1->releaseObject(allocatable);
            try
            {
                obj->ice_ping();
            }
            catch(const Ice::ObjectNotExistException&)
            {
            }

            obj = session1->allocateObjectById(allocatable3)->ice_connectionId("client1")->ice_router(router1);
            obj->ice_ping();
            obj2 = communicator->stringToProxy("allocatable4")->ice_connectionId("client1")->ice_router(router1);
            obj2->ice_ping();
            session1->releaseObject(allocatable3);
            try
            {
                obj->ice_ping();
            }
            catch(const Ice::ObjectNotExistException&)
            {
            }
            try
            {
                obj2->ice_ping();
            }
            catch(const Ice::ObjectNotExistException&)
            {
            }
            session1->destroy();
        }
        catch(const Ice::LocalException& ex)
        {
            cerr << ex << endl;
            test(false);
        }
        cout << "ok" << endl;

        cout << "stress test... " << flush;

        SessionPrx stressSession = registry->createSession("StressSession", "");        
        keepAlive->add(stressSession);

        const int nClients = 10;
        int i;
        vector<StressClientPtr> clients;
        for(i = 0; i < nClients - 2; ++i)
        {
            if(IceUtil::random(2) == 1)
            {
                clients.push_back(new StressClient(i, registry, false));
            }
            else
            {
                clients.push_back(new StressClient(i, stressSession));
            }
            clients.back()->start();
        }
        clients.push_back(new StressClient(i++, registry, true));
        clients.back()->start();
        clients.push_back(new StressClient(i++, registry, true));
        clients.back()->start();
        
        for(vector<StressClientPtr>::const_iterator p = clients.begin(); p != clients.end(); ++p)
        {
            (*p)->notifyThread();
        }

        //
        // Let the stress client run for a bit.
        //
        IceUtil::ThreadControl::sleep(IceUtil::Time::seconds(8));

        //
        // Terminate the stress clients.
        //
        for(vector<StressClientPtr>::const_iterator q = clients.begin(); q != clients.end(); ++q)
        {
            (*q)->terminate();
            (*q)->getThreadControl().join();
        }

        stressSession->destroy();

        cout << "ok" << endl;
    }
    catch(const AllocationTimeoutException& ex)
    {
        cerr << ex << endl;
        test(false);
    }
    catch(const AllocationException& ex)
    {
        cerr << ex.reason << endl;
        test(false);
    }

    cout << "shutting down router... " << flush;
    admin->stopServer("Glacier2");
    cout << "ok" << endl;

    keepAlive->terminate();
    keepAlive->getThreadControl().join();
    keepAlive = 0;

    session->destroy();
}
Example #5
0
bool
NodeService::startImpl(int argc, char* argv[], int& status)
{
    bool nowarn = false;
    bool readonly = false;
    string initFromReplica;
    string desc;
    vector<string> targets;


    for(int i = 1; i < argc; ++i)
    {
        if(strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0)
        {
            usage(argv[0]);
            status = EXIT_SUCCESS;
            return false;
        }
        else if(strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--version") == 0)
        {
            print(ICE_STRING_VERSION);
            status = EXIT_SUCCESS;
            return false;
        }
        else if(strcmp(argv[i], "--nowarn") == 0)
        {
            nowarn = true;
        }
        else if(strcmp(argv[i], "--readonly") == 0)
        {
            readonly = true;
        }
        else if(strcmp(argv[i], "--initdb-from-replica") == 0)
        {
            if(i + 1 >= argc)
            {
                error("missing replica argument for option `" + string(argv[i]) + "'");
                usage(argv[0]);
                return false;
            }

            initFromReplica = argv[++i];
        } 
        else if(strcmp(argv[i], "--deploy") == 0)
        {
            if(i + 1 >= argc)
            {
                error("missing descriptor argument for option `" + string(argv[i]) + "'");
                usage(argv[0]);
                return false;
            }

            desc = argv[++i];

            while(i + 1 < argc && argv[++i][0] != '-')
            {
                targets.push_back(argv[i]);
            }
        }
        else
        {
            error("invalid option: `" + string(argv[i]) + "'");
            usage(argv[0]);
            return false;
        }
    }

    PropertiesPtr properties = communicator()->getProperties();

    //
    // Disable server idle time. Otherwise, the adapter would be
    // shutdown prematurely and the deactivation would fail.
    // Deactivation of the node relies on the object adapter
    // to be active since it needs to terminate servers.
    //
    // TODO: implement Ice.ServerIdleTime in the activator
    // termination listener instead?
    //
    properties->setProperty("Ice.ServerIdleTime", "0");

    //
    // Warn the user that setting Ice.ThreadPool.Server isn't useful.
    //
    if(!nowarn && properties->getPropertyAsIntWithDefault("Ice.ThreadPool.Server.Size", 0) > 0)
    {
        Warning out(communicator()->getLogger());
        out << "setting `Ice.ThreadPool.Server.Size' is not useful, ";
        out << "you should set individual adapter thread pools instead.";
    }
    
    setupThreadPool(properties, "IceGrid.Node.ThreadPool", 1, 100);

    //
    // Create the activator.
    //
    TraceLevelsPtr traceLevels = new TraceLevels(communicator(), "IceGrid.Node");
    _activator = new Activator(traceLevels);

    //
    // Collocate the IceGrid registry if we need to.
    //
    if(properties->getPropertyAsInt("IceGrid.Node.CollocateRegistry") > 0)
    {
        _registry = new CollocatedRegistry(communicator(), _activator, nowarn, readonly, initFromReplica);
        if(!_registry->start())
        {
            return false;
        }

        //
        // Set the default locator property to point to the collocated
        // locator (this property is passed by the activator to each
        // activated server). The default locator is also needed by
        // the node session manager.
        //
        if(properties->getProperty("Ice.Default.Locator").empty())
        {
            Identity locatorId;
            locatorId.category = properties->getPropertyWithDefault("IceGrid.InstanceName", "IceGrid");
            locatorId.name = "Locator";
            string endpoints = properties->getProperty("IceGrid.Registry.Client.Endpoints");
            string locPrx = "\"" + communicator()->identityToString(locatorId) + "\" :" + endpoints;
            communicator()->setDefaultLocator(Ice::LocatorPrx::uncheckedCast(communicator()->stringToProxy(locPrx)));
            properties->setProperty("Ice.Default.Locator", locPrx);
        }
    }
    else if(properties->getProperty("Ice.Default.Locator").empty())
    {
        error("property `Ice.Default.Locator' is not set");
        return false;
    }

    //
    // Initialize the database environment (first setup the directory structure if needed).
    //
    string dataPath = properties->getProperty("IceGrid.Node.Data");
    string dbPath;
    if(dataPath.empty())
    {
        error("property `IceGrid.Node.Data' is not set");
        return false;
    }
    else
    {
        if(!IceUtilInternal::directoryExists(dataPath))
        {
            FileException ex(__FILE__, __LINE__);
            ex.path = dataPath;
            ex.error = IceInternal::getSystemErrno();
      
            ServiceError err(this);
            err << "property `IceGrid.Node.Data' is set to an invalid path:\n" << ex;
            return false;
        }

        //
        // Creates subdirectories.
        //
        if(dataPath[dataPath.length() - 1] != '/')
        {
            dataPath += "/"; 
        }

        IcePatch2::createDirectory(dataPath + "servers");
        IcePatch2::createDirectory(dataPath + "tmp");
        IcePatch2::createDirectory(dataPath + "distrib");

#ifdef _WIN32
        //
        // Make sure these directories are not indexed by the Windows
        // indexing service (which can cause random "Access Denied"
        // errors if indexing runs at the same time as the node is
        // creating/deleting files).
        //
        try
        {
            setNoIndexingAttribute(dataPath + "servers");
            setNoIndexingAttribute(dataPath + "tmp");
            setNoIndexingAttribute(dataPath + "distrib");
        }
        catch(const FileException& ex)
        {
            if(!nowarn)
            {
                Warning out(communicator()->getLogger());
                out << "couldn't disable file indexing:\n" << ex;
            }
        }
#endif
    }

    //
    // Check that required properties are set and valid.
    //
    if(properties->getProperty("IceGrid.Node.Endpoints").empty())
    {
        error("property `IceGrid.Node.Endpoints' is not set");
        return false;
    }

    string name = properties->getProperty("IceGrid.Node.Name");
    if(name.empty())
    {
        error("property `IceGrid.Node.Name' is not set");
        return false;
    }

    //
    // Setup the Freeze database environment home directory. The name of the database
    // environment for the IceGrid node is the name of the node.
    //
    properties->setProperty("Freeze.DbEnv." + name + ".DbHome", dbPath);

    //
    // Create the node object adapter.
    //
    _adapter = communicator()->createObjectAdapter("IceGrid.Node");

    //
    // Setup the user account mapper if configured.
    //
    string mapperProperty = "IceGrid.Node.UserAccountMapper";
    string mapperPropertyValue = properties->getProperty(mapperProperty);
    UserAccountMapperPrx mapper;
    if(!mapperPropertyValue.empty())
    {
        try
        {
            mapper = UserAccountMapperPrx::uncheckedCast(communicator()->propertyToProxy(mapperProperty));
        }
        catch(const std::exception& ex)
        {
            ServiceError err(this);
            err << "user account mapper `" << mapperProperty << "' is invalid:\n" << ex;
            return false;
        }
    }
    else
    {
        string userAccountFileProperty = properties->getProperty("IceGrid.Node.UserAccounts");
        if(!userAccountFileProperty.empty())
        {
            try
            {
                Ice::ObjectPrx object = _adapter->addWithUUID(new FileUserAccountMapperI(userAccountFileProperty));
                object = object->ice_collocationOptimized(true);
                mapper = UserAccountMapperPrx::uncheckedCast(object);
            }
            catch(const std::string& msg)
            {
                error(msg);
                return false;
            }
        }
    }

    //
    // Create a new timer to handle server activation/deactivation timeouts.
    //
    _timer = new IceUtil::Timer();

    //
    // The IceGrid instance name.
    //
    const string instanceName = communicator()->getDefaultLocator()->ice_getIdentity().category;

    _sessions.reset(new NodeSessionManager(communicator()));

    //
    // Create the server factory. The server factory creates persistent objects
    // for the server and server adapter. It also takes care of installing the
    // evictors and object factories necessary to store these objects.
    //
    Identity id = communicator()->stringToIdentity(instanceName + "/Node-" + name);
    NodePrx nodeProxy = NodePrx::uncheckedCast(_adapter->createProxy(id));
    _node = new NodeI(_adapter, *_sessions, _activator, _timer, traceLevels, nodeProxy, name, mapper);
    _adapter->add(_node, nodeProxy->ice_getIdentity());

    _adapter->addServantLocator(new DefaultServantLocator(new NodeServerAdminRouter(_node)), 
                                _node->getServerAdminCategory());

    //
    // Start the platform info thread if needed.
    //
    _node->getPlatformInfo().start();

    //
    // Ensures that the locator is reachable.
    // 
    if(!nowarn)
    {
        try
        {
            communicator()->getDefaultLocator()->ice_timeout(1000)->ice_ping();
        }
        catch(const Ice::LocalException& ex)
        {
            Warning out(communicator()->getLogger());
            out << "couldn't reach the IceGrid registry (this is expected ";
            out << "if it's down, otherwise please check the value of the ";
            out << "Ice.Default.Locator property):\n" << ex;
        }
    }

    //
    // Create the node sessions with the registries.
    //
    _sessions->create(_node);

    //
    // In some tests, we deploy icegridnodes using IceGrid:
    //
    if(properties->getProperty("Ice.Admin.Endpoints") != "")
    {
        //
        // Replace Process facet and create Admin object
        //
        try
        {
            ProcessPtr origProcess = ProcessPtr::dynamicCast(communicator()->removeAdminFacet("Process"));
            communicator()->addAdminFacet(new ProcessI(_activator, origProcess), "Process");
            communicator()->getAdmin();
        }
        catch(const Ice::NotRegisteredException&)
        {
            //
            // Some plug-in removed the Process facet, so we don't replace it.
            // (unlikely error though)
            // 
        }
    }

    //
    // Start the activator.
    //
    _activator->start();

    //
    // Activate the adapter.
    //
    _adapter->activate();

    //
    // Notify the node session manager that the node can start
    // accepting incoming connections.
    //
    _sessions->activate();

    string bundleName = properties->getProperty("IceGrid.Node.PrintServersReady");
    if(!bundleName.empty() || !desc.empty())
    {
        enableInterrupt();
        if(!_sessions->waitForCreate())
        {
            //
            // Create was interrupted, return true as if the service was
            // correctly initiliazed to make sure it's properly stopped.
            //
            return true;
        }
        disableInterrupt();
    }

    //
    // Deploy application if a descriptor is passed as a command-line option.
    //
    if(!desc.empty())
    {
        try
        {
            Ice::Identity regId;
            regId.category = instanceName;
            regId.name = "Registry";
            
            RegistryPrx registry = RegistryPrx::checkedCast(communicator()->getDefaultLocator()->findObjectById(regId));
            if(!registry)
            {
                throw "invalid registry";
            }
        
            registry = registry->ice_preferSecure(true); // Use SSL if available.
            
            IceGrid::AdminSessionPrx session;
            if(communicator()->getProperties()->getPropertyAsInt("IceGridAdmin.AuthenticateUsingSSL"))
            {
                session = registry->createAdminSessionFromSecureConnection();
            }
            else
            {
                string id = communicator()->getProperties()->getProperty("IceGridAdmin.Username");
                string password = communicator()->getProperties()->getProperty("IceGridAdmin.Password");
                while(id.empty())
                {
                    cout << "user id: " << flush;
                    getline(cin, id);
                    id = IceUtilInternal::trim(id);
                }
                
                if(password.empty())
                {
                    cout << "password: "******"failed to deploy application `" << desc << "':\n" << ex;
        }
        catch(const AccessDeniedException& ex)
        {
            ServiceWarning warn(this);
            warn << "failed to deploy application `" << desc << "':\n" 
                 << "registry database is locked by `" << ex.lockUserId << "'";
        }
        catch(const std::exception& ex)
        {
            ServiceWarning warn(this);
            warn << "failed to deploy application `" << desc << "':\n" << ex;
        }
        catch(const string& reason)
        {
            ServiceWarning warn(this);
            warn << "failed to deploy application `" << desc << "':\n" << reason;
        }
    }

    if(!bundleName.empty())
    {
        print(bundleName + " ready");
    }

    return true;
}
Example #6
0
void
allTests(const Ice::CommunicatorPtr& comm)
{
    IceGrid::RegistryPrx registry = IceGrid::RegistryPrx::checkedCast(
        comm->stringToProxy(comm->getDefaultLocator()->ice_getIdentity().category + "/Registry"));

    AdminSessionPrx session = registry->createAdminSession("foo", "bar");

    session->ice_getConnection()->setACM(registry->getACMTimeout(), IceUtil::None, Ice::HeartbeatAlways);

    AdminPrx admin = session->getAdmin();
    test(admin);

    map<string, string> params;

    params.clear();
    params["id"] = "Master";
    params["replicaName"] = "";
    params["port"] = "12050";
    instantiateServer(admin, "IceGridRegistry", params);

    params.clear();
    params["id"] = "Slave1";
    params["replicaName"] = "Slave1";
    params["port"] = "12051";
    instantiateServer(admin, "IceGridRegistry", params);

    params.clear();
    params["id"] = "Slave2";
    params["replicaName"] = "Slave2";
    params["port"] = "12052";
    instantiateServer(admin, "IceGridRegistry", params);

    Ice::LocatorPrx masterLocator =
        Ice::LocatorPrx::uncheckedCast(comm->stringToProxy("RepTestIceGrid/Locator-Master:default -p 12050"));
    Ice::LocatorPrx slave1Locator =
        Ice::LocatorPrx::uncheckedCast(comm->stringToProxy("RepTestIceGrid/Locator-Slave1:default -p 12051"));
    Ice::LocatorPrx slave2Locator =
        Ice::LocatorPrx::uncheckedCast(comm->stringToProxy("RepTestIceGrid/Locator-Slave2:default -p 12052"));

    Ice::LocatorPrx replicatedLocator =
        Ice::LocatorPrx::uncheckedCast(comm->stringToProxy("RepTestIceGrid/Locator:default -p 12050:default -p 12051"));

    AdminPrx masterAdmin, slave1Admin, slave2Admin;

    admin->startServer("Master");
    masterAdmin = createAdminSession(masterLocator, "");

    admin->startServer("Slave1");
    slave1Admin = createAdminSession(slave1Locator, "Slave1");

    //
    // Test replication and well-known objects:
    //
    // - Locator interface
    // - Query interface
    //
    // - Registry object
    // - RegistryUserAccountMapper
    // - SessionManager/SSLSessionManager
    // - AdminSessionManager/AdminSSLSessionManager
    //
    cout << "testing replicated locator and query interface... " << flush;
    {
        Ice::EndpointSeq endpoints;
        ObjectInfo info;

        info = masterAdmin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Locator"));
        ObjectInfo info1 = slave1Admin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Locator"));
        test(slave1Admin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Locator")) == info);
        test(info.type == Ice::Locator::ice_staticId());
        endpoints = info.proxy->ice_getEndpoints();
        test(endpoints.size() == 2);
        test(endpoints[0]->toString().find("-p 12050") != string::npos);
        test(endpoints[1]->toString().find("-p 12051") != string::npos);

        info = masterAdmin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Query"));
        test(slave1Admin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Query")) == info);
        test(info.type == IceGrid::Query::ice_staticId());
        endpoints = info.proxy->ice_getEndpoints();
        test(endpoints.size() == 2);
        test(endpoints[0]->toString().find("-p 12050") != string::npos);
        test(endpoints[1]->toString().find("-p 12051") != string::npos);

        admin->startServer("Slave2");
        slave2Admin = createAdminSession(slave2Locator, "Slave2");

        info = masterAdmin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Locator"));
        // We eventually need to wait here for the update of the replicated objects to propagate to the replica.
        int nRetry = 0;
        while(slave1Admin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Locator")) != info && nRetry < maxRetry)
        {
            IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(sleepTime));
            ++nRetry;
        }
        test(slave2Admin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Locator")) == info);
        test(info.type == Ice::Locator::ice_staticId());
        endpoints = info.proxy->ice_getEndpoints();
        test(endpoints.size() == 3);
        test(endpoints[0]->toString().find("-p 12050") != string::npos);
        test(endpoints[1]->toString().find("-p 12051") != string::npos);
        test(endpoints[2]->toString().find("-p 12052") != string::npos);

        info = masterAdmin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Query"));
        // We eventually need to wait here for the update of the replicated objects to propagate to the replica.
        nRetry = 0;
        while(slave1Admin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Query")) != info && nRetry < maxRetry)
        {
            IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(sleepTime));
            ++nRetry;
        }
        test(slave2Admin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Query")) == info);
        test(info.type == IceGrid::Query::ice_staticId());
        endpoints = info.proxy->ice_getEndpoints();
        test(endpoints.size() == 3);
        test(endpoints[0]->toString().find("-p 12050") != string::npos);
        test(endpoints[1]->toString().find("-p 12051") != string::npos);
        test(endpoints[2]->toString().find("-p 12052") != string::npos);

        slave2Admin->shutdown();
        waitForServerState(admin, "Slave2", false);

        info = masterAdmin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Locator"));
        // We eventually need to wait here for the update of the replicated objects to propagate to the replica.
        nRetry = 0;
        while(slave1Admin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Locator")) != info && nRetry < maxRetry)
        {
            IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(sleepTime));
            ++nRetry;
        }
        test(slave1Admin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Locator")) == info);
        test(info.type == Ice::Locator::ice_staticId());
        endpoints = info.proxy->ice_getEndpoints();
        test(endpoints.size() == 2);
        test(endpoints[0]->toString().find("-p 12050") != string::npos);
        test(endpoints[1]->toString().find("-p 12051") != string::npos);

        info = masterAdmin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Query"));
        nRetry = 0;
        while(slave1Admin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Query")) != info && nRetry < maxRetry)
        {
            IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(sleepTime));
            ++nRetry;
        }
        test(slave1Admin->getObjectInfo(Ice::stringToIdentity("RepTestIceGrid/Query")) == info);
        test(info.type == IceGrid::Query::ice_staticId());
        endpoints = info.proxy->ice_getEndpoints();
        test(endpoints.size() == 2);
        test(endpoints[0]->toString().find("-p 12050") != string::npos);
        test(endpoints[1]->toString().find("-p 12051") != string::npos);

        QueryPrx query;
        query = QueryPrx::uncheckedCast(comm->stringToProxy("RepTestIceGrid/Query:" + endpoints[0]->toString()));
        Ice::ObjectProxySeq objs = query->findAllObjectsByType("::IceGrid::Registry");
        test(objs.size() == 2);
        query = QueryPrx::uncheckedCast(comm->stringToProxy("RepTestIceGrid/Query:" + endpoints[1]->toString()));
        test(objs == query->findAllObjectsByType("::IceGrid::Registry"));
    }
    cout << "ok" << endl;

    cout << "testing well-known IceGrid objects... " << flush;
    {
        //
        // Test Registry well-known object (we have already tested
        // admin session creation for the creation of the admin
        // session above!)
        //
        RegistryPrx masterRegistry = RegistryPrx::checkedCast(
            comm->stringToProxy("RepTestIceGrid/Registry")->ice_locator(replicatedLocator));
        RegistryPrx slave1Registry = RegistryPrx::checkedCast(
            comm->stringToProxy("RepTestIceGrid/Registry-Slave1")->ice_locator(replicatedLocator));

        SessionPrx session = masterRegistry->createSession("dummy", "dummy");
        session->destroy();
        if(comm->getProperties()->getProperty("Ice.Default.Protocol") == "ssl")
        {
            session = masterRegistry->createSessionFromSecureConnection();
            session->destroy();
        }
        else
        {
            try
            {
                masterRegistry->createSessionFromSecureConnection();
            }
            catch(const PermissionDeniedException&)
            {
            }
        }

        try
        {
            slave1Registry->createSession("dummy", "");
        }
        catch(const PermissionDeniedException&)
        {
        }
        try
        {
            slave1Registry->createSessionFromSecureConnection();
        }
        catch(const PermissionDeniedException&)
        {
        }

        //
        // Test registry user-account mapper.
        //
        UserAccountMapperPrx masterMapper = UserAccountMapperPrx::checkedCast(
            comm->stringToProxy("RepTestIceGrid/RegistryUserAccountMapper")->ice_locator(replicatedLocator));
        UserAccountMapperPrx slave1Mapper = UserAccountMapperPrx::checkedCast(
            comm->stringToProxy("RepTestIceGrid/RegistryUserAccountMapper-Slave1")->ice_locator(replicatedLocator));

        test(masterMapper->getUserAccount("Dummy User Account1") == "dummy1");
        test(masterMapper->getUserAccount("Dummy User Account2") == "dummy2");
        test(slave1Mapper->getUserAccount("Dummy User Account1") == "dummy1");
        test(slave1Mapper->getUserAccount("Dummy User Account2") == "dummy2");
        try
        {
            masterMapper->getUserAccount("unknown");
            test(false);
        }
        catch(const UserAccountNotFoundException&)
        {
        }
        try
        {
            slave1Mapper->getUserAccount("unknown");
            test(false);
        }
        catch(const UserAccountNotFoundException&)
        {
        }

        //
        // Test SessionManager, SSLSessionManager,
        // AdminSessionManager, AdminSSLSessionManager
        //
        comm->stringToProxy("RepTestIceGrid/SessionManager")->ice_locator(replicatedLocator)->ice_ping();
        comm->stringToProxy("RepTestIceGrid/SSLSessionManager")->ice_locator(replicatedLocator)->ice_ping();
        try
        {
            comm->stringToProxy("RepTestIceGrid/SessionManager-Slave1")->ice_locator(replicatedLocator)->ice_ping();
            test(false);
        }
        catch(const Ice::NotRegisteredException&)
        {
        }
        try
        {
            comm->stringToProxy("RepTestIceGrid/SSLSessionManager-Slave1")->ice_locator(replicatedLocator)->ice_ping();
            test(false);
        }
        catch(const Ice::NotRegisteredException&)
        {
        }

        comm->stringToProxy("RepTestIceGrid/AdminSessionManager")->ice_locator(replicatedLocator)->ice_ping();
        comm->stringToProxy("RepTestIceGrid/AdminSSLSessionManager")->ice_locator(replicatedLocator)->ice_ping();
        comm->stringToProxy("RepTestIceGrid/AdminSessionManager-Slave1")->ice_locator(replicatedLocator)->ice_ping();
        comm->stringToProxy("RepTestIceGrid/AdminSSLSessionManager-Slave1")->ice_locator(replicatedLocator)->ice_ping();
    }
    cout << "ok" << endl;

    //
    // Registry update test:
    //
    // - start master
    // - start slave1: keep slave1 up for each update
    // - start slave2: shutdown slave2 for each update
    // - ensure updates are correctly replicated
    // - updates to test: application/adapter/object
    //
    cout << "testing registry updates... " << flush;
    {
        ApplicationDescriptor app;
        app.name = "TestApp";
        app.description = "added application";

        AdapterInfo adpt;
        adpt.id = "TestAdpt";
        adpt.proxy = comm->stringToProxy("dummy:tcp -p 12345 -h 127.0.0.1");

        ObjectInfo obj;
        obj.proxy = comm->stringToProxy("dummy:tcp -p 12345 -h 127.0.0.1");
        obj.type = "::Hello";

        //
        // We use the locator registry from Slave1 to ensure that the
        // forwarding to the master work (the slave locator registry
        // forwards everything to the master).
        //
        Ice::LocatorRegistryPrx locatorRegistry = slave1Locator->getRegistry();

        //
        // Test addition of application, adapter, object.
        //

        try
        {
            slave1Admin->addApplication(app);
            test(false);
        }
        catch(const DeploymentException&)
        {
            // Slave can't modify the database.
        }
        masterAdmin->addApplication(app);

        locatorRegistry->setAdapterDirectProxy(adpt.id, adpt.proxy);

        try
        {
            slave1Admin->addObjectWithType(obj.proxy, obj.type);
            test(false);
        }
        catch(const DeploymentException&)
        {
            // Slave can't modify the database
        }
        masterAdmin->addObjectWithType(obj.proxy, obj.type);

        admin->startServer("Slave2");
        slave2Admin = createAdminSession(slave2Locator, "Slave2");

        test(masterAdmin->getApplicationInfo("TestApp").descriptor.description == "added application");
        test(slave1Admin->getApplicationInfo("TestApp").descriptor.description == "added application");
        test(slave2Admin->getApplicationInfo("TestApp").descriptor.description == "added application");

        test(masterAdmin->getAdapterInfo("TestAdpt")[0] == adpt);
        test(slave1Admin->getAdapterInfo("TestAdpt")[0] == adpt);
        test(slave2Admin->getAdapterInfo("TestAdpt")[0] == adpt);

        test(masterAdmin->getObjectInfo(obj.proxy->ice_getIdentity()) == obj);
        test(slave1Admin->getObjectInfo(obj.proxy->ice_getIdentity()) == obj);
        test(slave2Admin->getObjectInfo(obj.proxy->ice_getIdentity()) == obj);

        slave2Admin->shutdown();
        waitForServerState(admin, "Slave2", false);

        //
        // Test sync of application.
        //

        app.description = "updated1 application";
        try
        {
            slave1Admin->syncApplication(app);
            test(false);
        }
        catch(const DeploymentException&)
        {
            // Slave can't modify the database.
        }
        masterAdmin->syncApplication(app);

        admin->startServer("Slave2");
        slave2Admin = createAdminSession(slave2Locator, "Slave2");
        test(masterAdmin->getApplicationInfo("TestApp").descriptor.description == "updated1 application");
        test(slave1Admin->getApplicationInfo("TestApp").descriptor.description == "updated1 application");
        test(slave2Admin->getApplicationInfo("TestApp").descriptor.description == "updated1 application");
        slave2Admin->shutdown();
        waitForServerState(admin, "Slave2", false);

        //
        // Test update of application, adapter, object.
        //

        ApplicationUpdateDescriptor appUpdate;
        appUpdate.name = "TestApp";
        appUpdate.description = new BoxedString("updated2 application");
        try
        {
            slave1Admin->updateApplication(appUpdate);
            test(false);
        }
        catch(const DeploymentException&)
        {
            // Slave can't modify the database.
        }
        masterAdmin->updateApplication(appUpdate);

        adpt.replicaGroupId = "TestReplicaGroup";
        locatorRegistry->setReplicatedAdapterDirectProxy(adpt.id, adpt.replicaGroupId, adpt.proxy);

        obj.proxy = comm->stringToProxy("dummy:tcp -p 12346 -h 127.0.0.1");
        try
        {
            slave1Admin->updateObject(obj.proxy);
            test(false);
        }
        catch(const DeploymentException&)
        {
            // Slave can't modify the database
        }
        masterAdmin->updateObject(obj.proxy);

        admin->startServer("Slave2");
        slave2Admin = createAdminSession(slave2Locator, "Slave2");

        test(masterAdmin->getApplicationInfo("TestApp").descriptor.description == "updated2 application");
        test(slave1Admin->getApplicationInfo("TestApp").descriptor.description == "updated2 application");
        test(slave2Admin->getApplicationInfo("TestApp").descriptor.description == "updated2 application");

        test(masterAdmin->getAdapterInfo("TestAdpt")[0] == adpt);
        test(slave1Admin->getAdapterInfo("TestAdpt")[0] == adpt);
        test(slave2Admin->getAdapterInfo("TestAdpt")[0] == adpt);

        test(masterAdmin->getObjectInfo(obj.proxy->ice_getIdentity()) == obj);
        test(slave1Admin->getObjectInfo(obj.proxy->ice_getIdentity()) == obj);
        test(slave2Admin->getObjectInfo(obj.proxy->ice_getIdentity()) == obj);

        slave2Admin->shutdown();
        waitForServerState(admin, "Slave2", false);

        //
        // Test removal of application, adapter and object.

        try
        {
            slave1Admin->removeApplication("TestApp");
            test(false);
        }
        catch(const DeploymentException&)
        {
            // Slave can't modify the database.
        }
        masterAdmin->removeApplication("TestApp");

        try
        {
            slave1Admin->removeAdapter("TestAdpt");
            test(false);
        }
        catch(const DeploymentException&)
        {
            // Slave can't modify the database.
        }
        masterAdmin->removeAdapter("TestAdpt");
        try
        {
            slave1Admin->removeObject(obj.proxy->ice_getIdentity());
        }
        catch(const DeploymentException&)
        {
            // Slave can't modify the database.
        }
        masterAdmin->removeObject(obj.proxy->ice_getIdentity());

        admin->startServer("Slave2");
        slave2Admin = createAdminSession(slave2Locator, "Slave2");
        try
        {
            masterAdmin->getApplicationInfo("TestApp");
            test(false);
        }
        catch(const ApplicationNotExistException&)
        {
        }
        try
        {
            slave1Admin->getApplicationInfo("TestApp");
            test(false);
        }
        catch(const ApplicationNotExistException&)
        {
        }
        try
        {
            slave2Admin->getApplicationInfo("TestApp");
            test(false);
        }
        catch(const ApplicationNotExistException&)
        {
        }
        try
        {
            masterAdmin->getAdapterInfo("TestAdpt");
            test(false);
        }
        catch(const AdapterNotExistException&)
        {
        }
        try
        {
            slave1Admin->getAdapterInfo("TestAdpt");
            test(false);
        }
        catch(const AdapterNotExistException&)
        {
        }
        try
        {
            slave2Admin->getAdapterInfo("TestAdpt");
            test(false);
        }
        catch(const AdapterNotExistException&)
        {
        }
        try
        {
            masterAdmin->getObjectInfo(obj.proxy->ice_getIdentity());
            test(false);
        }
        catch(const ObjectNotRegisteredException&)
        {
        }
        try
        {
            slave1Admin->getObjectInfo(obj.proxy->ice_getIdentity());
            test(false);
        }
        catch(const ObjectNotRegisteredException&)
        {
        }
        try
        {
            slave2Admin->getObjectInfo(obj.proxy->ice_getIdentity());
            test(false);
        }
        catch(const ObjectNotRegisteredException&)
        {
        }

        slave2Admin->shutdown();
        waitForServerState(admin, "Slave2", false);
    }
    cout << "ok" << endl;

    params.clear();
    params["id"] = "Node1";
    instantiateServer(admin, "IceGridNode", params);

    //
    // Add an application which is using Node1. Otherwise, when a
    // registry restarts it would throw aways the proxy of the nodes
    // because the node isn't used by any application.
    //
    ApplicationDescriptor app;
    app.name = "DummyApp";
    app.nodes["Node1"].description = "dummy node";
    try
    {
        masterAdmin->addApplication(app);
    }
    catch(const Ice::Exception& ex)
    {
        cerr << ex << endl;
        test(false);
    }

    //
    // Test node session establishment.
    //
    // - start master, start slave1, start node, start slave2
    // - shutdown slave1, start slave1 -> node should re-connect
    // - shutdown master
    // - shutdown slave2, start slave2 -> node should re-connect
    // - shutdown slave1
    // - start master -> node connects to master
    // - start slave1 -> node connects to slave1
    //
    cout << "testing node session establishment... " << flush;
    {
        admin->startServer("Node1");

        waitForNodeState(masterAdmin, "Node1", true);
        waitForNodeState(slave1Admin, "Node1", true);

        admin->startServer("Slave2");
        slave2Admin = createAdminSession(slave2Locator, "Slave2");

        waitForNodeState(slave2Admin, "Node1", true); // Node should connect.

        slave1Admin->shutdown();
        waitForServerState(admin, "Slave1", false);
        admin->startServer("Slave1");
        slave1Admin = createAdminSession(slave1Locator, "Slave1");

        try
        {
            test(slave1Admin->pingNode("Node1")); // Node should be re-connected.
        }
        catch(const NodeNotExistException&)
        {
            test(false);
        }

        masterAdmin->shutdown();
        waitForServerState(admin, "Master", false);

        slave2Admin->shutdown();
        waitForServerState(admin, "Slave2", false);
        admin->startServer("Slave2");
        slave2Admin = createAdminSession(slave2Locator, "Slave2");

        try
        {
            //
            // On slow environments, it can take a bit for the node to
            // re-establish the connection so we ping it twice. The
            // second should succeed.
            //
            slave2Admin->pingNode("Node1");
            test(slave2Admin->pingNode("Node1")); // Node should be re-connected even if the master is down.
        }
        catch(const NodeNotExistException&)
        {
            test(false);
        }

        slave1Admin->shutdown();
        waitForServerState(admin, "Slave1", false);

        admin->startServer("Master");
        masterAdmin = createAdminSession(masterLocator, "");

        try
        {
            test(masterAdmin->pingNode("Node1")); // Node should be re-connected.
        }
        catch(const NodeNotExistException&)
        {
            test(false);
        }

        admin->startServer("Slave1");
        slave1Admin = createAdminSession(slave1Locator, "Slave1");

        try
        {
            test(slave1Admin->pingNode("Node1")); // Node should be re-connected.
        }
        catch(const NodeNotExistException&)
        {
            test(false);
        }

        try
        {
            test(masterAdmin->pingNode("Node1"));
        }
        catch(const NodeNotExistException&)
        {
            test(false);
        }

        try
        {
            test(slave2Admin->pingNode("Node1"));
        }
        catch(const NodeNotExistException&)
        {
            test(false);
        }

        slave2Admin->shutdown();
        waitForServerState(admin, "Slave2", false);
        admin->startServer("Slave2");
        slave2Admin = createAdminSession(slave2Locator, "Slave2");
        try
        {
            test(slave2Admin->pingNode("Node1"));
        }
        catch(const NodeNotExistException&)
        {
            test(false);
        }
    }
    cout << "ok" << endl;

    //
    // Testing updates with out-of-date replicas.
    //
    cout << "testing out-of-date replicas... " << flush;
    {
        ApplicationDescriptor app;
        app.name = "TestApp";
        app.description = "added application";

        ServerDescriptorPtr server = new ServerDescriptor();
        server->id = "Server";
        server->exe = comm->getProperties()->getProperty("ServerDir") + "/server";
        server->pwd = ".";
        server->applicationDistrib = false;
        server->allocatable = false;
        addProperty(server, "Ice.Admin.Endpoints", "tcp -h 127.0.0.1");
        server->activation = "on-demand";
        AdapterDescriptor adapter;
        adapter.name = "TestAdapter";
        adapter.id = "TestAdapter.Server";
        adapter.registerProcess = false;
        adapter.serverLifetime = true;
        PropertyDescriptor property;
        property.name = "TestAdapter.Endpoints";
        property.value = "default";
        server->propertySet.properties.push_back(property);
        property.name = "Identity";
        property.value = "test";
        server->propertySet.properties.push_back(property);
        ObjectDescriptor object;
        object.id = Ice::stringToIdentity("test");
        object.type = "::Test::TestIntf";
        adapter.objects.push_back(object);
        server->adapters.push_back(adapter);
        app.nodes["Node1"].servers.push_back(server);

        masterAdmin->addApplication(app);

        try
        {
            comm->stringToProxy("test")->ice_locator(masterLocator)->ice_locatorCacheTimeout(0)->ice_ping();
            comm->stringToProxy("test")->ice_locator(slave1Locator)->ice_locatorCacheTimeout(0)->ice_ping();
            comm->stringToProxy("test")->ice_locator(slave2Locator)->ice_locatorCacheTimeout(0)->ice_ping();
        }
        catch(const Ice::LocalException& ex)
        {
            cerr << ex << endl;
            test(false);
        }

        masterAdmin->stopServer("Server");

        //
        // Shutdown Slave2 and update application.
        //
        slave2Admin->shutdown();
        waitForServerState(admin, "Slave2", false);

        ApplicationUpdateDescriptor update;
        update.name = "TestApp";
        NodeUpdateDescriptor node;
        node.name = "Node1";
        node.servers.push_back(server);
        update.nodes.push_back(node);
        property.name = "Dummy";
        property.value = "val";
        server->propertySet.properties.push_back(property);
        masterAdmin->updateApplication(update);

        try
        {
            comm->stringToProxy("test")->ice_locator(masterLocator)->ice_locatorCacheTimeout(0)->ice_ping();
            comm->stringToProxy("test")->ice_locator(slave1Locator)->ice_locatorCacheTimeout(0)->ice_ping();
        }
        catch(const Ice::LocalException& ex)
        {
            cerr << ex << endl;
            test(false);
        }

        masterAdmin->shutdown();
        waitForServerState(admin, "Master", false);

        admin->startServer("Slave2");
        slave2Admin = createAdminSession(slave2Locator, "Slave2");
        waitForNodeState(slave2Admin, "Node1", true); // Node should connect.

        try
        {
            slave2Admin->startServer("Server");
            test(false);
        }
        catch(const DeploymentException&)
        {
        }
        try
        {
            comm->stringToProxy("test")->ice_locator(slave2Locator)->ice_locatorCacheTimeout(0)->ice_ping();
            test(false);
        }
        catch(const Ice::NoEndpointException&)
        {
        }

        admin->startServer("Master");
        masterAdmin = createAdminSession(masterLocator, "");

        slave2Admin->shutdown();
        waitForServerState(admin, "Slave2", false);
        admin->startServer("Slave2");
        slave2Admin = createAdminSession(slave2Locator, "Slave2");

        try
        {
            comm->stringToProxy("test")->ice_locator(slave2Locator)->ice_locatorCacheTimeout(0)->ice_ping();
        }
        catch(const Ice::LocalException& ex)
        {
            cerr << ex << endl;
            test(false);
        }

        //
        // Shutdown Node1 and update the application, then, shutdown
        // the master.
        //
        slave1Admin->shutdownNode("Node1");
        waitForServerState(admin, "Node1", false);

        slave2Admin->shutdown();
        waitForServerState(admin, "Slave2", false);

        property.name = "Dummy2";
        property.value = "val";
        server->propertySet.properties.push_back(property);
        masterAdmin->updateApplication(update);

        masterAdmin->shutdown();
        waitForServerState(admin, "Master", false);

        //
        // Restart Node1 and Slave2, Slave2 still has the old version
        // of the server so it should be able to load it. Slave1 has
        // a more recent version, so it can't load it.
        //
        admin->startServer("Slave2");
        slave2Admin = createAdminSession(slave2Locator, "Slave2");

        admin->startServer("Node1");

        waitForNodeState(slave2Admin, "Node1", true);

        slave1Admin->shutdown();
        waitForServerState(admin, "Slave1", false);

        try
        {
            comm->stringToProxy("test")->ice_locator(slave2Locator)->ice_locatorCacheTimeout(0)->ice_ping();
        }
        catch(const Ice::LocalException& ex)
        {
            cerr << ex << endl;
            test(false);
        }

        admin->startServer("Slave1");
        slave1Admin = createAdminSession(slave1Locator, "Slave1");

        try
        {
            comm->stringToProxy("test")->ice_locator(slave1Locator)->ice_locatorCacheTimeout(0)->ice_ping();
        }
        catch(const Ice::NoEndpointException&)
        {
        }

        try
        {
            comm->stringToProxy("test")->ice_locator(slave2Locator)->ice_locatorCacheTimeout(0)->ice_ping();
        }
        catch(const Ice::LocalException& ex)
        {
            cerr << ex << endl;
            test(false);
        }
        slave2Admin->stopServer("Server");

        //
        // Start the master. This will re-load the server on the node
        // and update the out-of-date replicas.
        //
        admin->startServer("Master");
        masterAdmin = createAdminSession(masterLocator, "");

        slave1Admin->shutdown();
        waitForServerState(admin, "Slave1", false);
        admin->startServer("Slave1");
        slave1Admin = createAdminSession(slave1Locator, "Slave1");

        slave2Admin->shutdownNode("Node1");
        waitForServerState(admin, "Node1", false);
        admin->startServer("Node1");

        slave2Admin->shutdown();
        waitForServerState(admin, "Slave2", false);
        admin->startServer("Slave2");
        slave2Admin = createAdminSession(slave2Locator, "Slave2");

        waitForNodeState(masterAdmin, "Node1", true);
        waitForNodeState(slave1Admin, "Node1", true);
        waitForNodeState(slave2Admin, "Node1", true);

        try
        {
            comm->stringToProxy("test")->ice_locator(masterLocator)->ice_locatorCacheTimeout(0)->ice_ping();
            comm->stringToProxy("test")->ice_locator(slave1Locator)->ice_locatorCacheTimeout(0)->ice_ping();
            comm->stringToProxy("test")->ice_locator(slave2Locator)->ice_locatorCacheTimeout(0)->ice_ping();
        }
        catch(const Ice::LocalException& ex)
        {
            cerr << ex << endl;
            test(false);
        }

        slave2Admin->stopServer("Server");

        masterAdmin->removeApplication("TestApp");
    }
    cout << "ok" << endl;

    cout << "testing master upgrade... " << flush;
    {
        ApplicationDescriptor app;
        app.name = "TestApp";
        app.description = "added application";

        ServerDescriptorPtr server = new ServerDescriptor();
        server->id = "Server";
        server->exe = comm->getProperties()->getProperty("ServerDir") + "/server";
        server->pwd = ".";
        server->applicationDistrib = false;
        server->allocatable = false;
        addProperty(server, "Ice.Admin.Endpoints", "tcp -h 127.0.0.1");
        server->activation = "on-demand";
        AdapterDescriptor adapter;
        adapter.name = "TestAdapter";
        adapter.id = "TestAdapter.Server";
        adapter.serverLifetime = true;
        adapter.registerProcess = false;
        PropertyDescriptor property;
        property.name = "TestAdapter.Endpoints";
        property.value = "default";
        server->propertySet.properties.push_back(property);
        property.name = "Identity";
        property.value = "test";
        server->propertySet.properties.push_back(property);
        ObjectDescriptor object;
        object.id = Ice::stringToIdentity("test");
        object.type = "::Test::TestIntf";
        adapter.objects.push_back(object);
        server->adapters.push_back(adapter);
        app.nodes["Node1"].servers.push_back(server);

        masterAdmin->addApplication(app);

        comm->stringToProxy("test")->ice_locator(masterLocator)->ice_locatorCacheTimeout(0)->ice_ping();
        comm->stringToProxy("test")->ice_locator(slave1Locator)->ice_locatorCacheTimeout(0)->ice_ping();
        comm->stringToProxy("test")->ice_locator(slave2Locator)->ice_locatorCacheTimeout(0)->ice_ping();
        masterAdmin->stopServer("Server");

        //
        // Shutdown the Master, update Slave1 to be the Master.
        //
        masterAdmin->shutdown();
        waitForServerState(admin, "Master", false);
        slave1Admin->shutdown();
        waitForServerState(admin, "Slave1", false);

        params.clear();
        params["id"] = "Slave1";
        params["port"] = "12051";
        params["replicaName"] = "Master";
        instantiateServer(admin, "IceGridRegistry", params);

        admin->startServer("Slave1");
        slave1Locator =
            Ice::LocatorPrx::uncheckedCast(comm->stringToProxy("RepTestIceGrid/Locator-Master:default -p 12051"));
        slave1Admin = createAdminSession(slave1Locator, "");

        waitForReplicaState(slave1Admin, "Slave2", true);

        ApplicationUpdateDescriptor update;
        update.name = "TestApp";
        NodeUpdateDescriptor node;
        node.name = "Node1";
        node.servers.push_back(server);
        update.nodes.push_back(node);
        property.name = "Dummy";
        property.value = "val";
        server->propertySet.properties.push_back(property);
        slave1Admin->updateApplication(update);

        comm->stringToProxy("test")->ice_locator(slave1Locator)->ice_locatorCacheTimeout(0)->ice_ping();
        comm->stringToProxy("test")->ice_locator(slave2Locator)->ice_locatorCacheTimeout(0)->ice_ping();

        slave1Admin->shutdown();
        waitForServerState(admin, "Slave1", false);

        params.clear();
        params["id"] = "Slave1";
        params["replicaName"] = "Slave1";
        params["port"] = "12051";
        instantiateServer(admin, "IceGridRegistry", params);

        params.clear();
        params["id"] = "Master";
        params["replicaName"] = "";
        params["port"] = "12050";
        params["arg"] = "--initdb-from-replica=Slave2";
        instantiateServer(admin, "IceGridRegistry", params);

        admin->startServer("Master");
        masterAdmin = createAdminSession(masterLocator, "");

        admin->startServer("Slave1");
        slave1Locator =
            Ice::LocatorPrx::uncheckedCast(comm->stringToProxy("RepTestIceGrid/Locator-Slave1:default -p 12051"));
        slave1Admin = createAdminSession(slave1Locator, "Slave1");

        comm->stringToProxy("test")->ice_locator(masterLocator)->ice_locatorCacheTimeout(0)->ice_ping();
        comm->stringToProxy("test")->ice_locator(slave1Locator)->ice_locatorCacheTimeout(0)->ice_ping();
        comm->stringToProxy("test")->ice_locator(slave2Locator)->ice_locatorCacheTimeout(0)->ice_ping();

        masterAdmin->stopServer("Server");

        waitForReplicaState(masterAdmin, "Slave1", true);
        waitForReplicaState(masterAdmin, "Slave2", true);

        ApplicationInfo info = masterAdmin->getApplicationInfo("TestApp");
        test(info.revision == 2);

        masterAdmin->removeApplication("TestApp");
    }
    cout << "ok" << endl;

    cout << "testing interop with registry and node using the 1.0 encoding... " << flush;
    {
        params.clear();
        params["id"] = "Slave3";
        params["replicaName"] = "Slave3";
        params["port"] = "12053";
        params["encoding"] = "1.0";
        instantiateServer(admin, "IceGridRegistry", params);

        params.clear();
        params["id"] = "Node2";
        params["encoding"] = "1.0";
        instantiateServer(admin, "IceGridNode", params);

        admin->startServer("Slave3");
        waitForServerState(admin, "Slave3", true);
        waitForReplicaState(masterAdmin, "Slave3", true);

        admin->startServer("Node2");
        waitForNodeState(masterAdmin, "Node2", true);

        Ice::LocatorPrx slave3Locator =
            Ice::LocatorPrx::uncheckedCast(
                comm->stringToProxy("RepTestIceGrid/Locator-Slave3 -e 1.0:default -p 12053"));
        IceGrid::AdminPrx slave3Admin = createAdminSession(slave3Locator, "Slave3");
        waitForNodeState(slave3Admin, "Node2", true);

        ApplicationDescriptor app;
        app.name = "TestApp";
        app.description = "added application";

        ServerDescriptorPtr server = new ServerDescriptor();
        server->id = "Server";
        server->exe = comm->getProperties()->getProperty("ServerDir") + "/server";
        server->pwd = ".";
        server->applicationDistrib = false;
        server->allocatable = false;
        addProperty(server, "Ice.Admin.Endpoints", "tcp -h 127.0.0.1");
        server->activation = "on-demand";
        AdapterDescriptor adapter;
        adapter.name = "TestAdapter";
        adapter.id = "TestAdapter.Server";
        adapter.serverLifetime = true;
        adapter.registerProcess = false;
        PropertyDescriptor property;
        property.name = "TestAdapter.Endpoints";
        property.value = "default";
        server->propertySet.properties.push_back(property);
        property.name = "Identity";
        property.value = "test";
        server->propertySet.properties.push_back(property);
        ObjectDescriptor object;
        object.id = Ice::stringToIdentity("test");
        object.type = "::Test::TestIntf";
        adapter.objects.push_back(object);
        server->adapters.push_back(adapter);
        app.nodes["Node2"].servers.push_back(server);

        masterAdmin->addApplication(app);

        comm->stringToProxy("test -e 1.0")->ice_locator(
            masterLocator->ice_encodingVersion(Ice::Encoding_1_0))->ice_locatorCacheTimeout(0)->ice_ping();
        comm->stringToProxy("test -e 1.0")->ice_locator(
            slave1Locator->ice_encodingVersion(Ice::Encoding_1_0))->ice_locatorCacheTimeout(0)->ice_ping();
        comm->stringToProxy("test -e 1.0")->ice_locator(slave3Locator)->ice_locatorCacheTimeout(0)->ice_ping();
        masterAdmin->stopServer("Server");

    }
    cout << "ok" << endl;


    slave1Admin->shutdownNode("Node1");
    removeServer(admin, "Node1");

    removeServer(admin, "Slave2");

    slave1Admin->shutdown();
    removeServer(admin, "Slave1");
    masterAdmin->shutdown();
    removeServer(admin, "Master");
}