int MMKVImpl::SAdd(DBID db, const Data& key, const DataArray& elements) { if (m_readonly) { return ERR_PERMISSION_DENIED; } int err = 0; RWLockGuard<MemorySegmentManager, WRITE_LOCK> keylock_guard(m_segment); EnsureWritableValueSpace(); ObjectAllocator allocator = m_segment.MSpaceAllocator<Object>(); StringSet* set = GetObject<StringSet>(db, key, V_TYPE_SET, true, err)(std::less<Object>(), allocator); if (0 != err) { return err; } int inserted = 0; for (size_t i = 0; i < elements.size(); i++) { std::pair<StringSet::iterator, bool> ret = set->insert(Object(elements[i], true)); if (ret.second) { m_segment.AssignObjectValue(*(ret.first), elements[i], true); inserted++; } } return inserted; }
void PerconaFTEngine::Stats(Context& ctx, std::string& str) { str.append("perconaft_version:").append(stringfromll(DB_VERSION_MAJOR)).append(".").append(stringfromll(DB_VERSION_MINOR)).append(".").append( stringfromll(DB_VERSION_PATCH)).append("\r\n"); DataArray nss; ListNameSpaces(ctx, nss); PerconaFTLocalContext& local_ctx = g_local_ctx.GetValue(); for (size_t i = 0; i < nss.size(); i++) { DB* db = GetFTDB(ctx, nss[i], false); if (NULL == db) continue; str.append("\r\nDB[").append(nss[i].AsString()).append("] Stats:\r\n"); DB_BTREE_STAT64 st; memset(&st, 0, sizeof(st)); db->stat64(db, local_ctx.transc.Peek(), &st); str.append("bt_nkeys:").append(stringfromll(st.bt_nkeys)).append("\r\n"); str.append("bt_ndata:").append(stringfromll(st.bt_ndata)).append("\r\n"); str.append("bt_fsize:").append(stringfromll(st.bt_fsize)).append("\r\n"); str.append("bt_dsize:").append(stringfromll(st.bt_dsize)).append("\r\n"); str.append("bt_create_time_sec:").append(stringfromll(st.bt_create_time_sec)).append("\r\n"); str.append("bt_modify_time_sec:").append(stringfromll(st.bt_modify_time_sec)).append("\r\n"); str.append("bt_verify_time_sec:").append(stringfromll(st.bt_verify_time_sec)).append("\r\n"); } }
Data& getElement(uint32_t idx) { if (elements.size() <= idx) { elements.resize(idx + 1); } return elements[idx]; }
int MMKVImpl::SDiff(DBID db, const DataArray& keys, const StringArrayResult& diffs) { if (keys.size() < 2) { return ERR_INVALID_TYPE; } RWLockGuard<MemorySegmentManager, READ_LOCK> keylock_guard(m_segment); return GenericSInterDiffUnion(db, OP_DIFF, keys, NULL, &diffs); }
void SortTimeFunction::f() { const int n = m_copy.size(); const int elementSize = m_copy.getElementSize(); DataArray data = m_copy; switch(elementSize) { case 1 : m_sortMethod->getMethod()(data.getData(), n, elementSize, CountComparator<BYTE>( m_compareCount)); break; case 2 : m_sortMethod->getMethod()(data.getData(), n, elementSize, CountComparator<unsigned short>(m_compareCount)); break; default: m_sortMethod->getMethod()(data.getData(), n, elementSize, CountComparator<unsigned int >(m_compareCount)); break; } }
int Engine::FlushAll(Context& ctx) { DataArray nss; ListNameSpaces(ctx, nss); for (size_t i = 0; i < nss.size(); i++) { Flush(ctx, nss[i]); } return 0; }
int Engine::CompactAll(Context& ctx) { DataArray nss; ListNameSpaces(ctx, nss); for (size_t i = 0; i < nss.size(); i++) { KeyObject start, end; start.SetNameSpace(nss[i]); Compact(ctx, start, end); } return 0; }
int MMKVImpl::SRem(DBID db, const Data& key, const DataArray& members) { if (m_readonly) { return ERR_PERMISSION_DENIED; } int err = 0; RWLockGuard<MemorySegmentManager, WRITE_LOCK> keylock_guard(m_segment); EnsureWritableValueSpace(); StringSet* set = GetObject<StringSet>(db, key, V_TYPE_SET, false, err)(); if (IS_NOT_EXISTS(err)) { return 0; } if (NULL == set || 0 != err) { return err; } int removed = 0; for (size_t i = 0; i < members.size(); i++) { StringSet::iterator found = set->find(Object(members[i], true)); if (found != set->end()) { Object cc = *found; set->erase(found); DestroyObjectContent(cc); removed++; } } if (set->empty()) { GenericDel(GetMMKVTable(db, false),db, Object(key, false)); } return removed; }
int MMKVImpl::GenericSInterDiffUnion(DBID db, int op, const DataArray& keys, const Data* dest, const StringArrayResult* res) { StdObjectSet results[2]; int result_index = 0; StringSetArray sets; sets.resize(keys.size()); int err = 0; size_t start_index = 0; StringSet* destset = NULL; StdObjectSet* result = NULL; StdObjectSet* cmp = NULL; int current_result_index = 0; ObjectAllocator allocator = m_segment.MSpaceAllocator<Object>(); StringSet empty_set(std::less<Object>(), allocator); for (size_t i = 0; i < keys.size(); i++) { StringSet* set = GetObject<StringSet>(db, keys[i], V_TYPE_SET, false, err)(); if (IS_NOT_EXISTS(err)) { sets[i] = &empty_set; continue; } if (0 != err) { return err; } sets[i] = set; } if (NULL != dest) { ObjectAllocator allocator = m_segment.MSpaceAllocator<Object>(); destset = GetObject<StringSet>(db, *dest, V_TYPE_SET, true, err)(std::less<Object>(), allocator); if (0 != err) { return err; } } if (NULL == sets[0]) { if (op == OP_DIFF || op == OP_INTER) { result_index = 0; goto _end; } } for (size_t i = 0; i < keys.size(); i++) { if (sets[i] != NULL) { start_index = i; break; } } for (size_t i = start_index + 1; i < keys.size(); i++) { result = results + current_result_index; if (sets[i]->empty()) { if (op == OP_INTER) { results->clear(); result_index = 0; goto _end; } } result->clear(); switch (op) { case OP_DIFF: { if (cmp == NULL) { std::set_difference(sets[start_index]->begin(), sets[start_index]->end(), sets[i]->begin(), sets[i]->end(), std::inserter(*result, result->end()), std::less<Object>()); } else { std::set_difference(cmp->begin(), cmp->end(), sets[i]->begin(), sets[i]->end(), std::inserter(*result, result->end()), std::less<Object>()); } if (result->empty()) { result_index = current_result_index; goto _end; } break; } case OP_INTER: { if (cmp == NULL) { std::set_intersection(sets[start_index]->begin(), sets[start_index]->end(), sets[i]->begin(), sets[i]->end(), std::inserter(*result, result->end()), std::less<Object>()); } else { std::set_intersection(cmp->begin(), cmp->end(), sets[i]->begin(), sets[i]->end(), std::inserter(*result, result->end()), std::less<Object>()); } if (result->empty()) { result_index = current_result_index; goto _end; } break; } case OP_UNION: { if (cmp == NULL) { std::set_union(sets[start_index]->begin(), sets[start_index]->end(), sets[i]->begin(), sets[i]->end(), std::inserter(*result, result->end()), std::less<Object>()); } else { std::set_union(cmp->begin(), cmp->end(), sets[i]->begin(), sets[i]->end(), std::inserter(*result, result->end()), std::less<Object>()); } break; } } current_result_index = 1 - current_result_index; cmp = result; } result_index = result == results ? 0 : 1; _end: if (NULL != res) { StdObjectSet::iterator it = results[result_index].begin(); while (it != results[result_index].end()) { it->ToString(res->Get()); it++; } } if (NULL != destset) { //remove elements not in dest set StringSet::iterator it = destset->begin(); while (it != destset->end()) { Object element = *it; StdObjectSet::iterator cit = results[result_index].find(element); if (cit != results[result_index].end()) //remove elements from results which already in dest set { results[result_index].erase(cit); it++; } else { it = destset->erase(it); DestroyObjectContent(element); } } //insert rest elements StdObjectSet::iterator cit = results[result_index].begin(); while (cit != results[result_index].end()) { Object clone = CloneStrObject(*cit); destset->insert(clone); cit++; } return destset->size(); } return 0; }
int PerconaFTEngine::Init(const std::string& dir, const std::string& options) { Properties props; parse_conf_content(options, props); //parse config conf_get_int64(props, "cache_size", g_perconaft_config.cache_size); conf_get_uint32(props, "checkpoint_pool_threads", g_perconaft_config.checkpoint_pool_threads); conf_get_uint32(props, "checkpoint_period", g_perconaft_config.checkpoint_period); conf_get_uint32(props, "cleaner_period", g_perconaft_config.cleaner_period); conf_get_uint32(props, "cleaner_iterations", g_perconaft_config.cleaner_iterations); conf_get_bool(props, "evictor_enable_partial_eviction", g_perconaft_config.evictor_enable_partial_eviction); std::string compression; conf_get_string(props, "compression", compression); if (compression == "none") { g_perconaft_config.compression = TOKU_NO_COMPRESSION; } else if (compression == "snappy" || compression.empty()) { g_perconaft_config.compression = TOKU_SNAPPY_METHOD; } else if (compression == "zlib") { g_perconaft_config.compression = TOKU_ZLIB_METHOD; } else { ERROR_LOG("Invalid compression config:%s for PercanoFT.", compression.c_str()); return -1; } uint32 env_open_flags = DB_CREATE | DB_PRIVATE | DB_THREAD | DB_INIT_MPOOL | DB_INIT_TXN | DB_INIT_LOCK | DB_INIT_LOG | DB_RECOVER; int env_open_mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; db_env_create(&m_env, 0); m_env->set_default_bt_compare(m_env, ardb_perconaft_compare); m_env->set_errcall(m_env, _err_callback); /* * set env config */ uint32 cache_gsize = g_perconaft_config.cache_size >> 30; uint32 cache_bytes = g_perconaft_config.cache_size % (1024 * 1024 * 1024); m_env->set_cachesize(m_env, cache_gsize, cache_bytes, 1); m_env->set_cachetable_pool_threads(m_env, g_perconaft_config.checkpoint_pool_threads); m_env->checkpointing_set_period(m_env, g_perconaft_config.checkpoint_period); m_env->cleaner_set_period(m_env, g_perconaft_config.cleaner_period); m_env->cleaner_set_iterations(m_env, g_perconaft_config.cleaner_iterations); m_env->evictor_set_enable_partial_eviction(m_env, g_perconaft_config.evictor_enable_partial_eviction); int r = m_env->open(m_env, dir.c_str(), env_open_flags, env_open_mode); CHECK_EXPR(r); DataArray nss; if (0 == r) { g_toku_env = m_env; DB* db = m_env->get_db_for_directory(m_env); if (NULL != db) { PerconaFTLocalContext& local_ctx = g_local_ctx.GetValue(); DB_TXN* txn = local_ctx.transc.Get(); DBC *c = NULL; CHECK_EXPR(r = db->cursor(db, txn, &c, 0)); if (0 == r) { r = c->c_getf_first(c, 0, nil_callback, NULL); } while (0 == r) { DBT raw_key; DBT raw_val; memset(&raw_key, 0, sizeof(raw_key)); memset(&raw_val, 0, sizeof(raw_key)); if (0 == c->c_get(c, &raw_key, &raw_val, DB_CURRENT)) { //std::string ns_str Data ns; ns.SetString((const char*) raw_key.data, false); INFO_LOG("TokuFT directory db %s:%s", (const char* ) raw_key.data, (const char* ) raw_val.data); nss.push_back(ns); } r = c->c_getf_next(c, 0, nil_callback, NULL); } if (NULL != c) { c->c_close(c); } local_ctx.transc.Release(true); r = 0; } } for (size_t i = 0; i < nss.size(); i++) { Context tmp; GetFTDB(tmp, nss[i], true); } return ENGINE_ERR(r); }
int Ardb::SortCommand(Context& ctx, const Slice& key, SortOptions& options, DataArray& values) { values.clear(); KeyType keytype = KEY_END; GetType(ctx, key, keytype); switch (keytype) { case LIST_META: { ListRange(ctx, key, 0, -1); break; } case SET_META: { SetMembers(ctx, key); break; } case ZSET_META: { ZSetRange(ctx, key, 0, -1, false, false, OP_GET); if (NULL == options.by) { options.nosort = true; } break; } default: { return ERR_INVALID_TYPE; } } DataArray sortvals; if (ctx.reply.MemberSize() > 0) { for (uint32 i = 0; i < ctx.reply.MemberSize(); i++) { Data v; v.SetString(ctx.reply.MemberAt(i).str, true); sortvals.push_back(v); } } if (sortvals.empty()) { return 0; } if (options.with_limit) { if (options.limit_offset < 0) { options.limit_offset = 0; } if ((uint32) options.limit_offset > sortvals.size()) { values.clear(); return 0; } if (options.limit_count < 0) { options.limit_count = sortvals.size(); } } std::vector<SortValue> sortvec; if (!options.nosort) { if (NULL != options.by) { sortvec.reserve(sortvals.size()); } for (uint32 i = 0; i < sortvals.size(); i++) { if (NULL != options.by) { sortvec.push_back(SortValue(&sortvals[i])); if (GetValueByPattern(ctx, options.by, sortvals[i], sortvec[i].cmp) < 0) { DEBUG_LOG("Failed to get value by pattern:%s", options.by); sortvec[i].cmp.Clear(); continue; } } if (options.with_alpha) { if (NULL != options.by) { sortvec[i].cmp.ToString(); } else { sortvals[i].ToString(); } } } if (NULL != options.by) { if (!options.is_desc) { std::sort(sortvec.begin(), sortvec.end(), less_value<SortValue>); } else { std::sort(sortvec.begin(), sortvec.end(), greater_value<SortValue>); } } else { if (!options.is_desc) { std::sort(sortvals.begin(), sortvals.end(), less_value<Data>); } else { std::sort(sortvals.begin(), sortvals.end(), greater_value<Data>); } } } if (!options.with_limit) { options.limit_offset = 0; options.limit_count = sortvals.size(); } uint32 count = 0; for (uint32 i = options.limit_offset; i < sortvals.size() && count < (uint32) options.limit_count; i++, count++) { Data* patternObj = NULL; if (NULL != options.by) { patternObj = sortvec[i].value; } else { patternObj = &(sortvals[i]); } if (options.get_patterns.empty()) { values.push_back(*patternObj); } else { for (uint32 j = 0; j < options.get_patterns.size(); j++) { Data vo; if (GetValueByPattern(ctx, options.get_patterns[j], *patternObj, vo) < 0) { DEBUG_LOG("Failed to get value by pattern for:%s", options.get_patterns[j]); vo.Clear(); } values.push_back(vo); } } } uint32 step = options.get_patterns.empty() ? 1 : options.get_patterns.size(); switch (options.aggregate) { case AGGREGATE_SUM: case AGGREGATE_AVG: { DataArray result; result.resize(step); for (uint32 i = 0; i < result.size(); i++) { for (uint32 j = i; j < values.size(); j += step) { result[i].IncrBy(values[j]); } } if (options.aggregate == AGGREGATE_AVG) { size_t count = values.size() / step; for (uint32 i = 0; i < result.size(); i++) { result[i].SetDouble(result[i].NumberValue() / count); } } values.assign(result.begin(), result.end()); break; } case AGGREGATE_MAX: case AGGREGATE_MIN: { DataArray result; result.resize(step); for (uint32 i = 0; i < result.size(); i++) { for (uint32 j = i; j < values.size(); j += step) { if (result[i].IsNil()) { result[i] = values[j]; } else { if (options.aggregate == AGGREGATE_MIN) { if (values[j] < result[i]) { result[i] = values[j]; } } else { if (values[j] > result[i]) { result[i] = values[j]; } } } } } values.assign(result.begin(), result.end()); break; } case AGGREGATE_COUNT: { size_t size = values.size() / step; values.clear(); Data v; v.SetInt64(size); values.push_back(v); break; } default: { break; } } if (options.store_dst != NULL && !values.empty()) { DeleteKey(ctx, options.store_dst); ValueObject list_meta; list_meta.key.key = options.store_dst; list_meta.key.type = KEY_META; list_meta.key.db = ctx.currentDB; list_meta.type = LIST_META; list_meta.meta.SetEncoding(COLLECTION_ECODING_ZIPLIST); BatchWriteGuard guard(GetKeyValueEngine()); DataArray::iterator it = values.begin(); while (it != values.end()) { if (!it->IsNil()) { std::string tmp; it->GetDecodeString(tmp); ListInsert(ctx, list_meta, NULL, tmp, false, false); } it++; } SetKeyValue(ctx, list_meta); } return 0; }
// Helper Funcitons void Socket::ClientHandshake() { std::cout << "Client Handshake" << std::endl; unsigned char previous = 0; bool bEncrypted = false; std::vector<unsigned char> handshakeData; bool bDone = false; while( !bDone ) { Record *record; if( !bEncrypted ) { record = new Record(); }else{ record = new Record( mClient ); } record->Read( *mSocket ); // Alert if( record->GetType() == Record::Alert ){ std::vector<unsigned char> data = record->GetData(); std::cerr << "Alert: " << std::hex << std::setw( 2 ) << std::setfill('0') << (int)data[0] << ", " << std::hex << std::setw( 2 ) << std::setfill('0') << (int)data[1] << std::endl; exit( EXIT_FAILURE ); } // Cipher if( record->GetType() == Record::Cipher ){ if( previous == Handshake::Type::ClientKey ){ bEncrypted = true; } previous = Record::Cipher; } // Handshake if( record->GetType() == Record::Handshake ){ std::vector<unsigned char> data = record->GetData(); if( previous == 0 ){ if( data[0] == Handshake::Type::ClientHello ){ Handshake::ClientHello hClientHello; hClientHello.Deserialize( std::vector<unsigned char>( data.begin() + 4, data.end() ) ); ClientRandom = hClientHello.Random(); handshakeData.insert( handshakeData.end(), data.begin(), data.end() ); Handshake::ServerHello hServerHello( 0x00, 0x35, 0x00 ); Record rServerHello( Record::Handshake, hServerHello.Serialize() ); rServerHello.Write( *mSocket ); ServerRandom = hServerHello.Random(); std::vector<unsigned char> dServerHello = hServerHello.Serialize(); handshakeData.insert( handshakeData.end(), dServerHello.begin(), dServerHello.end() ); Handshake::Certificate hCertificate( mCertificatePath ); Record rCertificate( Record::Handshake, hCertificate.Serialize() ); rCertificate.Write( *mSocket ); std::vector<unsigned char> dCertificate = hCertificate.Serialize(); handshakeData.insert( handshakeData.end(), dCertificate.begin(), dCertificate.end() ); std::vector<unsigned char> dServerDone; // Insert Size unsigned int size = htonl( dServerDone.size() ); unsigned char* cSize = (unsigned char*)&size; dServerDone.insert( dServerDone.begin(), cSize[3] ); dServerDone.insert( dServerDone.begin(), cSize[2] ); dServerDone.insert( dServerDone.begin(), cSize[1] ); // Insert Type dServerDone.insert( dServerDone.begin(), Handshake::Type::ServerDone ); Record rServerDone( Record::Handshake, dServerDone ); rServerDone.Write( *mSocket ); handshakeData.insert( handshakeData.end(), dServerDone.begin(), dServerDone.end() ); } previous = data[0]; } if( previous == Handshake::Type::ClientHello ){ if( data[0] == Handshake::Type::ClientKey ){ Handshake::ClientKeyExchange hClientKeyExchange( mKeyPath ); hClientKeyExchange.Deserialize( std::vector<unsigned char>( data.begin() + 4, data.end() ) ); handshakeData.insert( handshakeData.end(), data.begin(), data.end() ); DataArray secret = hClientKeyExchange.Secret(); MasterKey = PRF( 48, secret, "master secret", Concatonate( ClientRandom, ServerRandom ) ); KeyBlock = PRF( 136, MasterKey, "key expansion", Concatonate( ServerRandom, ClientRandom ) ); DataArray ClientMAC( KeyBlock.begin(), KeyBlock.begin() + 20 ); DataArray ServerMAC( KeyBlock.begin() + 20, KeyBlock.begin() + 40 ); DataArray ClientKey( KeyBlock.begin() + 40, KeyBlock.begin() + 72 ); DataArray ClientIV( KeyBlock.begin() + 104, KeyBlock.begin() + 120 ); mClient->Initialise( ClientMAC, ClientKey , ClientIV ); DataArray ServerKey( KeyBlock.begin() + 72, KeyBlock.begin() + 104 ); DataArray ServerIV( KeyBlock.begin() + 120, KeyBlock.begin() + 136 ); mServer->Initialise( ServerMAC, ServerKey , ServerIV ); } previous = data[0]; } if( previous == Record::Cipher ){ if( data[0] == Handshake::Type::Finished ){ Handshake::Finished hFinished; hFinished.Deserialize( std::vector<unsigned char>( data.begin() + 4, data.end() ) ); unsigned char md5hash[MD5_DIGEST_LENGTH]; unsigned char shahash[SHA_DIGEST_LENGTH]; MD5( &handshakeData[0], handshakeData.size(), md5hash ); SHA1( &handshakeData[0], handshakeData.size(), shahash ); std::vector<unsigned char> signiture; signiture.insert( signiture.end(), md5hash, md5hash + 16 ); signiture.insert( signiture.end(), shahash, shahash + 20 ); DataArray clientprf = PRF( 12, MasterKey, "client finished", signiture ); DataArray clientfin = std::vector<unsigned char>( data.begin() + 4, data.end() ); bool bDiff = false; for( unsigned int i = 0; i < clientfin.size(); i++ ){ if( clientfin[i] != clientprf[i] ) { bDiff = true; break; } } if( bDiff ) { std::cout << "Finished Different" << std::endl; std::cout << "Finish Should Be: " << std::dec << (int)clientprf.size() << std::endl; for( unsigned int i = 0 ; i < clientprf.size(); i++ ){ std::cout << std::hex << std::setw( 2 ) << std::setfill('0') << (int) clientprf[i] << " ";; if( (i + 1) % 16 == 0 ) std::cout << std::endl; } std::cout << std::endl; std::cout << "Finish Is: " << std::dec << (int)clientfin.size() << std::endl; for( unsigned int i = 0 ; i < clientfin.size(); i++ ){ std::cout << std::hex << std::setw( 2 ) << std::setfill('0') << (int) clientfin[i] << " ";; if( (i + 1) % 16 == 0 ) std::cout << std::endl; } std::cout << std::endl; } handshakeData.insert( handshakeData.end(), data.begin(), data.end() ); std::vector<unsigned char> dChangeCipher; dChangeCipher.push_back( 1 ); Record rChangeCipher( Record::Cipher, dChangeCipher ); rChangeCipher.Write( *mSocket ); MD5( &handshakeData[0], handshakeData.size(), md5hash ); SHA1( &handshakeData[0], handshakeData.size(), shahash ); std::vector<unsigned char> finSig; finSig.insert( finSig.end(), md5hash, md5hash + 16 ); finSig.insert( finSig.end(), shahash, shahash + 20 ); DataArray serverprf = PRF( 12, MasterKey, "server finished", finSig ); Record rServerFinished( Record::Handshake, Handshake::Finished( serverprf ).Serialize(), mServer ); rServerFinished.Write( *mSocket ); bDone = true; } previous = data[0]; } } } std::cout << "Finished " << this << " " << mSocket << std::endl; }