Example #1
0
Freeze::MapDb::MapDb(const ConnectionIPtr& connection, 
                     const string& dbName,
                     const string& key,
                     const string& value,
                     const KeyCompareBasePtr& keyCompare,
                     const vector<MapIndexBasePtr>& indices,
                     bool createDb) :
    Db(connection->dbEnv()->getEnv(), 0),
    _communicator(connection->communicator()),
    _dbName(dbName),
    _trace(connection->trace()),
    _keyCompare(keyCompare)
{
    if(_trace >= 1)
    {
        Trace out(_communicator->getLogger(), "Freeze.Map");
        out << "opening Db \"" << _dbName << "\"";
    }

    Catalog catalog(connection, _catalogName);
   
    TransactionPtr tx = connection->currentTransaction();
    bool ownTx = (tx == 0);

    for(;;)
    {
        try
        {
            if(ownTx)
            {
                tx = 0;
                tx = connection->beginTransaction();
            }

            Catalog::iterator ci = catalog.find(_dbName);
        
            if(ci != catalog.end())
            {
                if(ci->second.evictor)
                {
                    throw DatabaseException(__FILE__, __LINE__, _dbName + " is an evictor database");
                }
                
                _key = ci->second.key;
                _value = ci->second.value;
                checkTypes(key, value);
            }
            else
            {
                _key = key;
                _value = value;
            }
    
            set_app_private(this);
            if(_keyCompare->compareEnabled())
            {
                set_bt_compare(&customCompare);
            }
            
            PropertiesPtr properties = _communicator->getProperties();
            string propPrefix = "Freeze.Map." + _dbName + ".";
            
            int btreeMinKey = properties->getPropertyAsInt(propPrefix + "BtreeMinKey");
            if(btreeMinKey > 2)
            {
                if(_trace >= 1)
                {
                    Trace out(_communicator->getLogger(), "Freeze.Map");
                    out << "Setting \"" << _dbName << "\"'s btree minkey to " << btreeMinKey;
                }
                set_bt_minkey(btreeMinKey);
            }
            
            bool checksum = properties->getPropertyAsInt(propPrefix + "Checksum") > 0;
            if(checksum)
            {
                if(_trace >= 1)
                {
                    Trace out(_communicator->getLogger(), "Freeze.Map");
                    out << "Turning checksum on for \"" << _dbName << "\"";
                }
                
                set_flags(DB_CHKSUM);
            }
            
            int pageSize = properties->getPropertyAsInt(propPrefix + "PageSize");
            if(pageSize > 0)
            {
                if(_trace >= 1)
                {
                    Trace out(_communicator->getLogger(), "Freeze.Map");
                    out << "Setting \"" << _dbName << "\"'s pagesize to " << pageSize;
                }
                set_pagesize(pageSize);
            }
            

            DbTxn* txn = getTxn(tx);
            
            u_int32_t flags = DB_THREAD;
            if(createDb)
            {
                flags |= DB_CREATE;
            }

            open(txn, Ice::nativeToUTF8(_communicator, _dbName).c_str(), 0, DB_BTREE, flags, FREEZE_DB_MODE);
            
            
            StringSeq oldIndices;
            StringSeq newIndices;
            size_t oldSize = 0;
            CatalogIndexList catalogIndexList(connection, _catalogIndexListName);
            
            if(createDb)
            {
                CatalogIndexList::iterator cil = catalogIndexList.find(_dbName);
                if(cil != catalogIndexList.end())
                {
                    oldIndices = cil->second;
                    oldSize = oldIndices.size();
                }
            } 
            
            for(vector<MapIndexBasePtr>::const_iterator p = indices.begin();
                p != indices.end(); ++p)
            {
                const MapIndexBasePtr& indexBase = *p; 
                assert(indexBase->_impl == 0);
                assert(indexBase->_communicator == 0);
                indexBase->_communicator = connection->communicator();
                
                auto_ptr<MapIndexI> indexI;

                try
                {
                    indexI.reset(new MapIndexI(connection, *this, txn, createDb, indexBase));
                }
                catch(const DbDeadlockException&)
                {
                    throw;
                }
                catch(const DbException& dx)
                {
                    string message = "Error while opening index \"" + _dbName +
                        "." + indexBase->name() + "\": " + dx.what();

                    throw DatabaseException(__FILE__, __LINE__, message);
                }
                
#ifndef NDEBUG
                bool inserted = 
#endif
                    _indices.insert(IndexMap::value_type(indexBase->name(), indexI.get())).second;
                assert(inserted);
                
                indexBase->_impl = indexI.release();
                
                if(createDb)
                {
                    newIndices.push_back(indexBase->name());
                    oldIndices.erase(std::remove(oldIndices.begin(), oldIndices.end(), indexBase->name()), oldIndices.end());
                }
            }
            
            if(ci == catalog.end())
            {
                CatalogData catalogData;
                catalogData.evictor = false;
                catalogData.key = key;
                catalogData.value = value;
                catalog.put(Catalog::value_type(_dbName, catalogData));
            }
            
            if(createDb)
            {
                //
                // Remove old indices and write the new ones
                //
                bool indexRemoved = false;

                for(StringSeq::const_iterator q = oldIndices.begin(); q != oldIndices.end(); ++q)
                {
                    const string& index = *q;
                    
                    if(_trace >= 1)
                    {
                        Trace out(_communicator->getLogger(), "Freeze.Map");
                        out << "removing old index \"" << index << "\" on Db \"" << _dbName << "\"";
                    }
                    
                    try
                    {
                        connection->removeMapIndex(_dbName, *q);
                        indexRemoved = true;
                    }
                    catch(const IndexNotFoundException&)
                    {
                        // Ignored
                        
                        if(_trace >= 1)
                        {
                            Trace out(_communicator->getLogger(), "Freeze.Map");
                            out << "index \"" << index << "\" on Db \"" << _dbName << "\" does not exist";
                        }
                    }
                }
                
                if(indexRemoved || oldSize != newIndices.size())
                {
                    if(newIndices.size() == 0)
                    {
                        catalogIndexList.erase(_dbName);
                        if(_trace >= 1)
                        {
                            Trace out(_communicator->getLogger(), "Freeze.Map");
                            out << "Removed catalogIndexList entry for Db \"" << _dbName << "\"";
                        }
                        
                    }
                    else
                    {
                        catalogIndexList.put(CatalogIndexList::value_type(_dbName, newIndices));
                        if(_trace >= 1)
                        {
                            Trace out(_communicator->getLogger(), "Freeze.Map");
                            out << "Updated catalogIndexList entry for Db \"" << _dbName << "\"";
                        }
                    }
                }
            }

            if(ownTx)
            {
                tx->commit();
            }
            break; // for(;;)
        }
        catch(const DbDeadlockException& dx)
        {
            if(ownTx)
            {
                if(connection->deadlockWarning())
                {
                    Warning out(connection->communicator()->getLogger());
                    out << "Deadlock in Freeze::MapDb::MapDb on Map \"" 
                        << _dbName << "\"; retrying ...";
                }

                //
                // Ignored, try again
                //
            }
            else
            {
                throw DeadlockException(__FILE__, __LINE__, dx.what(), tx);
            }
        }
        catch(const DbException& dx)
        {
            if(ownTx)
            {
                try
                {
                    tx->rollback();
                }
                catch(...)
                {
                }
            }
                
            string message = "Error while opening Db \"" + _dbName +
                "\": " + dx.what();

            throw DatabaseException(__FILE__, __LINE__, message);
        }
        catch(...)
        {
            if(ownTx && tx != 0)
            {   
                try
                {
                    tx->rollback();
                }
                catch(...)
                {
                }
            }
            throw;
        }
    }
}
Freeze::ObjectStoreBase::ObjectStoreBase(const string& facet, const string& facetType,
                                         bool createDb,  EvictorIBase* evictor,
                                         const vector<IndexPtr>& indices,
                                         bool populateEmptyIndices) :
    _facet(facet),
    _evictor(evictor),
    _indices(indices),
    _communicator(evictor->communicator()),
    _encoding(evictor->encoding()),
    _keepStats(false)
{
    if(facet == "")
    {
        _dbName = EvictorIBase::defaultDb;
    }
    else
    {
        _dbName = facet;
    }

    if(!facetType.empty())
    {
        //
        // Create a sample servant with this type
        //
        ValueFactoryPtr factory = _communicator->getValueFactoryManager()->find(facetType);
        if(factory == 0)
        {
            throw DatabaseException(__FILE__, __LINE__, "No object factory registered for type-id '" + facetType + "'");
        }

        _sampleServant = factory->create(facetType);
    }

    ConnectionPtr catalogConnection = createConnection(_communicator, evictor->dbEnv()->getEnvName());
    Catalog catalog(catalogConnection, catalogName());

    Catalog::iterator p = catalog.find(evictor->filename());
    if(p != catalog.end())
    {
        if(p->second.evictor)
        {
            //
            // empty means the value is ::Freeze::ObjectRecord
            //
            _keepStats = p->second.value.empty();
        }
        else
        {
            DatabaseException ex(__FILE__, __LINE__);
            ex.message = evictor->filename() + " is not an evictor database";
            throw ex;
        }
    }

    DbEnv* dbEnv = evictor->dbEnv()->getEnv();

    try
    {
        _db.reset(new Db(dbEnv, 0));

        Ice::PropertiesPtr properties = evictor->communicator()->getProperties();
        string propPrefix = "Freeze.Evictor." + evictor->filename() + ".";

        int btreeMinKey = properties->getPropertyAsInt(propPrefix + _dbName + ".BtreeMinKey");
        if(btreeMinKey > 2)
        {
            if(evictor->trace() >= 1)
            {
                Trace out(evictor->communicator()->getLogger(), "Freeze.Evictor");
                out << "Setting \"" << evictor->filename() + "." + _dbName << "\"'s btree minkey to " << btreeMinKey;
            }

            _db->set_bt_minkey(btreeMinKey);
        }

        bool checksum = properties->getPropertyAsInt(propPrefix + "Checksum") > 0;
        if(checksum)
        {
            if(evictor->trace() >= 1)
            {
                Trace out(evictor->communicator()->getLogger(), "Freeze.Evictor");
                out << "Turning checksum on for \"" << evictor->filename() << "\"";
            }

            _db->set_flags(DB_CHKSUM);
        }

        int pageSize = properties->getPropertyAsInt(propPrefix + "PageSize");
        if(pageSize > 0)
        {
            if(evictor->trace() >= 1)
            {
                Trace out(evictor->communicator()->getLogger(), "Freeze.Evictor");
                out << "Setting \"" << evictor->filename() << "\"'s pagesize to " << pageSize;
            }

            _db->set_pagesize(pageSize);
        }


        TransactionPtr tx = catalogConnection->beginTransaction();
        DbTxn* txn = getTxn(tx);

        u_int32_t flags = DB_THREAD;
        if(createDb)
        {
            flags |= DB_CREATE;
        }

        //
        // Berkeley DB expects file paths to be UTF8 encoded. We keep
        // _dbName as a native string here, while it might have
        // been better to convert it to UTF-8, changing this isn't
        // possible without potentially breaking backward compatibility
        // with deployed databases.
        //
        _db->open(txn,
                  nativeToUTF8(evictor->filename(), getProcessStringConverter()).c_str(),
                  _dbName.c_str(), DB_BTREE, flags, FREEZE_DB_MODE);

        for(size_t i = 0; i < _indices.size(); ++i)
        {
            _indices[i]->_impl->associate(this, txn, createDb, populateEmptyIndices);
        }

        if(p == catalog.end())
        {
            CatalogData catalogData;
            catalogData.evictor = true;
            catalogData.key = "Ice::Identity";
            catalogData.value = "Object";
            catalog.put(Catalog::value_type(evictor->filename(), catalogData));
        }

        tx->commit();
    }
    catch(const DbException& dx)
    {
        TransactionPtr tx = catalogConnection->currentTransaction();
        if(tx != 0)
        {
            try
            {
                tx->rollback();
            }
            catch(...)
            {
            }
        }

        if(dx.get_errno() == ENOENT)
        {
            NotFoundException ex(__FILE__, __LINE__);
            ex.message = dx.what();
            throw ex;
        }
        else
        {
            DatabaseException ex(__FILE__, __LINE__);
            ex.message = dx.what();
            throw ex;
        }
    }
    catch(...)
    {
        TransactionPtr tx = catalogConnection->currentTransaction();
        if(tx != 0)
        {
            try
            {
                tx->rollback();
            }
            catch(...)
            {
            }
        }
        throw;
    }
}
Example #3
0
/*static*/ void
Freeze::MapHelper::recreate(const Freeze::ConnectionPtr& connection,
                            const string& dbName,
                            const string& key,
                            const string& value,
                            const Freeze::KeyCompareBasePtr& keyCompare,
                            const std::vector<MapIndexBasePtr>& indices)
{
    Freeze::ConnectionIPtr connectionI = Freeze::ConnectionIPtr::dynamicCast(connection.get());
    if(connectionI == 0)
    {
        throw DatabaseException(__FILE__, __LINE__, "Invalid connection");
    }

    if(dbName == catalogName() || dbName == catalogIndexListName())
    {
        throw DatabaseException(__FILE__, __LINE__,
                                "You cannot destroy recreate the \"" + dbName + "\" database");
    }

    if(connectionI->trace() >= 1)
    {
        Trace out(connectionI->communicator()->getLogger(), "Freeze.Map");
        out << "Recreating \"" << dbName << "\"";
    }

    TransactionPtr tx = connectionI->currentTransaction();
    bool ownTx = (tx == 0);

    Dbt keyDbt;
    keyDbt.set_flags(DB_DBT_REALLOC);
    Dbt valueDbt;
    valueDbt.set_flags(DB_DBT_REALLOC);

    try
    {
        for(;;)
        {
            try
            {
                if(ownTx)
                {
                    tx = 0;
                    tx = connectionI->beginTransaction();
                }

                DbTxn* txn = connectionI->dbTxn();

                if(connectionI->trace() >= 2)
                {
                    Trace out(connectionI->communicator()->getLogger(), "Freeze.Map");
                    out << "Removing all existing indices for \"" << dbName << "\"";
                }
                CatalogIndexList catalogIndexList(connection, catalogIndexListName());
                CatalogIndexList::iterator p = catalogIndexList.find(dbName);
                if(p != catalogIndexList.end())
                {
                    const StringSeq& idxs = p->second;

                    for(size_t i = 0; i < idxs.size(); ++i)
                    {
                        try
                        {
                            connection->removeMapIndex(dbName, idxs[i]);
                        }
                        catch(const IndexNotFoundException&)
                        {
                            //
                            // Ignored
                            //
                        }
                    }
                    catalogIndexList.erase(p);
                }

                //
                // Rename existing database
                //
                string oldDbName = dbName + ".old-" + generateUUID();

                if(connectionI->trace() >= 2)
                {
                    Trace out(connectionI->communicator()->getLogger(), "Freeze.Map");
                    out << "Renaming \"" << dbName << "\" to \"" << oldDbName << "\"";
                }

                connectionI->dbEnv()->getEnv()->dbrename(txn, dbName.c_str(), 0, oldDbName.c_str(), 0);

                //
                // Fortunately, DB closes oldDb automatically when it goes out of scope
                //
                Db oldDb(connectionI->dbEnv()->getEnv(), 0);

                //
                // Berkeley DB expects file paths to be UTF8 encoded.
                //
                oldDb.open(txn, nativeToUTF8(oldDbName, getProcessStringConverter()).c_str(),
                           0, DB_BTREE, DB_THREAD, FREEZE_DB_MODE);

                IceInternal::UniquePtr<MapDb> newDb(new MapDb(connectionI, dbName, key, value, keyCompare, indices, true));

                if(connectionI->trace() >= 2)
                {
                    Trace out(connectionI->communicator()->getLogger(), "Freeze.Map");
                    out << "Writing contents of \"" << oldDbName << "\" to fresh \"" << dbName << "\"";
                }

                //
                // Now simply write all of oldDb into newDb
                //
                Dbc* dbc = 0;
                oldDb.cursor(txn, &dbc, 0);

                try
                {
                    while(dbc->get(&keyDbt, &valueDbt, DB_NEXT) == 0)
                    {
                        newDb->put(txn, &keyDbt, &valueDbt, 0);
                    }
                }
                catch(...)
                {
                    dbc->close();
                    throw;
                }
                dbc->close();

                if(connectionI->trace() >= 2)
                {
                    Trace out(connectionI->communicator()->getLogger(), "Freeze.Map");
                    out << "Transfer complete; removing \"" << oldDbName << "\"";
                }
                connectionI->dbEnv()->getEnv()->dbremove(txn, oldDbName.c_str(), 0, 0);

                if(ownTx)
                {
                    tx->commit();
                }

                break; // for (;;)
            }
            catch(const DbDeadlockException& dx)
            {
                if(ownTx)
                {
                    if(connectionI->deadlockWarning())
                    {
                        Warning out(connectionI->communicator()->getLogger());
                        out << "Deadlock in Freeze::MapHelperI::recreate on Db \""
                            << dbName << "\"; retrying ...";
                    }

                    //
                    // Ignored, try again
                    //
                }
                else
                {
                    throw DeadlockException(__FILE__, __LINE__, dx.what(), tx);
                }
            }
            catch(const DbException& dx)
            {
                if(ownTx)
                {
                    try
                    {
                        tx->rollback();
                    }
                    catch(...)
                    {
                    }
                }

                throw DatabaseException(__FILE__, __LINE__, dx.what());
            }
            catch(...)
            {
                if(ownTx && tx != 0)
                {
                    try
                    {
                        tx->rollback();
                    }
                    catch(...)
                    {
                    }
                }
                throw;
            }
        }
        free(keyDbt.get_data());
        free(valueDbt.get_data());
    }
    catch(...)
    {
        free(keyDbt.get_data());
        free(valueDbt.get_data());

        throw;
    }
}