bool validateHelper ( RSA* key, Slice message, Slice signature) { int const keySize = RSA_size(key); if (!checkModulusLength (keySize)) return false; Buffer buf; auto ret = RSA_public_decrypt( keySize, signature.data(), buf.alloc (keySize), key, RSA_NO_PADDING); if (ret == -1) return false; sha256_hasher h; h (message.data(), message.size()); auto digest = static_cast<sha256_hasher::result_type>(h); return RSA_verify_PKCS1_PSS(key, digest.data(), EVP_sha256(), buf.data(), -1) == 1; }
int LMDBEngine::Put(const Slice& key, const Slice& value, const Options& options) { LMDBContext& holder = m_ctx_local.GetValue(); MDB_txn *txn = holder.readonly_txn; if (NULL == txn || holder.batch_write == 0) { MDB_val k, v; k.mv_data = const_cast<char*>(key.data()); k.mv_size = key.size(); v.mv_data = const_cast<char*>(value.data()); v.mv_size = value.size(); CHECK_OP(mdb_txn_begin(m_env, NULL, 0, &txn)); CHECK_OP(mdb_put(txn, m_dbi, &k, &v, 0)); CHECK_OP(mdb_txn_commit(txn)); return 0; } PutOperation* op = new PutOperation; op->key.assign((const char*) key.data(), key.size()); op->value.assign((const char*) value.data(), value.size()); m_write_queue.Push(op); if (holder.batch_write == 0) { CheckPointOperation* ck = new CheckPointOperation(holder.cond); m_write_queue.Push(ck); NotifyBackgroundThread(); ck->Wait(); DELETE(ck); return 0; } return 0; }
bool verify (PublicKey const& publicKey, Slice const& m, Slice const& sig, bool mustBeFullyCanonical) { if (auto const type = publicKeyType(publicKey)) { if (*type == KeyType::secp256k1) { return verifyDigest (publicKey, sha512Half(m), sig, mustBeFullyCanonical); } else if (*type == KeyType::ed25519) { if (! ed25519Canonical(sig)) return false; // We internally prefix Ed25519 keys with a 0xED // byte to distinguish them from secp256k1 keys // so when verifying the signature, we need to // first strip that prefix. return ed25519_sign_open( m.data(), m.size(), publicKey.data() + 1, sig.data()) == 0; } } return false; }
void Put(const Slice& key, const Slice& value) { std::string str(key.data(), key.size()); std::string v(value.data(), value.size()); dels.erase(str); inserts[str] = v; }
int LMDBEngine::Del(const Slice& key) { LMDBContext& holder = m_ctx_local.GetValue(); MDB_txn *txn = holder.readonly_txn; if (NULL == txn || holder.batch_write == 0) { MDB_val k; k.mv_data = const_cast<char*>(key.data()); k.mv_size = key.size(); mdb_txn_begin(m_env, NULL, 0, &txn); mdb_del(txn, m_dbi, &k, NULL); mdb_txn_commit(txn); return 0; } DelOperation* op = new DelOperation; op->key.assign((const char*) key.data(), key.size()); m_write_queue.Push(op); if (holder.batch_write == 0) { CheckPointOperation* ck = new CheckPointOperation(holder.cond); m_write_queue.Push(ck); NotifyBackgroundThread(); ck->Wait(); DELETE(ck); return 0; } return 0; }
void LevelDBEngine::CompactRange(const Slice& begin, const Slice& end) { leveldb::Slice s(begin.data(), begin.size()); leveldb::Slice e(end.data(), end.size()); leveldb::Slice* start = s.size() > 0 ? &s : NULL; leveldb::Slice* endpos = e.size() > 0 ? &e : NULL; m_db->CompactRange(start, endpos); }
virtual int Compare(const Slice& a, const Slice& b) const { uint64_t* idA = (uint64_t*)a.data(); uint64_t* idB = (uint64_t*)b.data(); int64_t delta = (*idA - *idB); if (delta == 0) return 0; return delta < 0 ? -1 : 1; }
bool parsePayloadHelper( Slice s, Buffer& modulus, Buffer& signature) { auto start = s.data (); auto finish = s.data () + s.size(); std::size_t len; std::tie (start, len) = oer::decode_length ( start, finish); if (std::distance (start, finish) < len) return false; std::memcpy (modulus.alloc (len), start, len); std::advance (start, len); std::tie (start, len) = oer::decode_length ( start, finish); if (std::distance (start, finish) < len) return false; std::memcpy (signature.alloc (len), start, len); std::advance (start, len); // Enforce constraints from the RFC: BigNum sig (BN_bin2bn ( signature.data(), signature.size(), nullptr)); BigNum mod (BN_bin2bn ( modulus.data(), modulus.size(), nullptr)); if (!sig || !mod) return false; // Per 4.4.1 of the RFC we are required to reject // moduli smaller than 128 bytes or greater than // 512 bytes. int modBytes = BN_num_bytes (mod.get()); if (!checkModulusLength (modBytes)) return false; // Per 4.4.2 of the RFC we must check whether the // signature and modulus consist of the same number // of octets and that the signature is numerically // less than the modulus: if (BN_num_bytes (sig.get()) != modBytes) return false; return BN_cmp (sig.get(), mod.get()) < 0; }
inline bool ParseInternalKey(const Slice& internal_key, ParsedInternalKey* result) { const size_t n = internal_key.size(); if (n < 8) return false; uint64_t num = DecodeFixed64(internal_key.data() + n - 8); unsigned char c = num & 0xff; result->sequence = num >> 8; result->type = static_cast<ValueType>(c); result->user_key = Slice(internal_key.data(), n - 8); return (c <= static_cast<unsigned char>(kTypeValue)); }
KVDictionary::Encoding::Encoding(const Slice &serialized) : _isRecordStore(serialized.size() > 0 && serialized.data()[0] == 0), _isIndex(serialized.size() > 0 && serialized.data()[0] == 1), _ordering(_isIndex ? orderingDeserialize(serialized.data() + 1) : Ordering::make(BSONObj())) { // Can't be both a record store and an index. dassert(!(_isRecordStore && _isIndex)); // If not a record store or index, we must be the "empty encoding". dassert((_isRecordStore || _isIndex) || serialized.size() == 0); }
int KVDictionary::Encoding::cmp(const Slice &a, const Slice &b) { const int cmp_len = std::min(a.size(), b.size()); const int c = memcmp(a.data(), b.data(), cmp_len); if (c != 0) { return c; } else if (a.size() < b.size()) { return -1; } else if (a.size() > b.size()) { return 1; } else { return 0; } }
int WiredTigerEngine::Put(const Slice& key, const Slice& value, const Options& options) { ContextHolder& holder = GetContextHolder(); WT_SESSION* session = holder.session; if (NULL == session) { return -1; } // if (holder.trasc_ref > 0) // { // PutOperation* op = new PutOperation; // op->key.assign((const char*) key.data(), key.size()); // op->value.assign((const char*) value.data(), value.size()); // m_write_queue.Push(op); // holder.readonly_transc = false; // return 0; // } int ret = 0; WT_CURSOR *cursor = holder.batch == NULL ? create_wiredtiger_cursor(session) : holder.batch; if (NULL == cursor) { return -1; } WT_ITEM key_item, value_item; key_item.data = key.data(); key_item.size = key.size(); value_item.data = value.data(); value_item.size = value.size(); cursor->set_key(cursor, &key_item); cursor->set_value(cursor, &value_item); ret = cursor->insert(cursor); if (0 != ret) { ERROR_LOG("Error write data for reason: %s", wiredtiger_strerror(ret)); ret = -1; } if (holder.batch == NULL) { ret = cursor->close(cursor); CHECK_WT_RETURN(ret); } else { holder.IncBatchWriteCount(); if (holder.batch_write_count >= m_cfg.batch_commit_watermark) { holder.RestartBatchWrite(); } } return ret; }
int OnRawKeyValue(const Slice& key, const Slice& value) { ArgumentArray args; args.push_back("__SET__"); args.push_back(std::string(key.data(), key.size())); args.push_back(std::string(value.data(), value.size())); RedisCommandFrame cmd(args); conn.conn->Write(cmd); count++; if (count % 10000 == 0) { conn.conn->GetService().Continue(); } return 0; }
bool RAMFile::write(uint64_t offset, Slice data) { ScopedMutex lock(&mtx_); assert(refcnt_ > 0); uint64_t end = offset + data.size(); if (end > total_) { // alloc buffer size_t more = (end - total_ + RAMFILE_BLK_SIZE - 1)/RAMFILE_BLK_SIZE; blks_.reserve(blks_.size() + more); for(size_t i = 0; i < more; i++ ) { char* blk = new char[RAMFILE_BLK_SIZE]; if (blk) { blks_.push_back(blk); total_ += RAMFILE_BLK_SIZE; } else { LOG_ERROR("write RAMFile error: out of memory"); return false; } } } lock.unlock(); size_t idx = offset / RAMFILE_BLK_SIZE; size_t off = 0; size_t left = data.size(); size_t len = RAMFILE_BLK_SIZE - offset % RAMFILE_BLK_SIZE; len = left < len ? left : len; memcpy(blks_[idx] + offset % RAMFILE_BLK_SIZE, data.data(), len); left -= len; off += len; idx ++; while (left > 0) { len = left < RAMFILE_BLK_SIZE ? left : RAMFILE_BLK_SIZE; memcpy(blks_[idx], data.data() + off, len); left -= len; off += len; idx ++; } assert(left == 0); lock.lock(); if (length_ < end) { length_ = end; } return true; }
void LMDBIterator::Seek(const Slice& target) { m_key.mv_data = const_cast<char*>(target.data()); m_key.mv_size = target.size(); int rc = mdb_cursor_get(m_cursor, &m_key, &m_value, MDB_SET_RANGE); m_valid = rc == 0; }
static void signal_safe_write_data(Slice data) { #if TD_PORT_POSIX while (!data.empty()) { auto res = write(2, data.begin(), data.size()); if (res < 0 && errno == EINTR) { continue; } if (res <= 0) { break; } if (res > 0) { data.remove_prefix(res); } } #elif TD_PORT_WINDOWS #if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP | WINAPI_PARTITION_SYSTEM) HANDLE stderr_handle = GetStdHandle(STD_ERROR_HANDLE); DWORD bytes_written; WriteFile(stderr_handle, data.data(), static_cast<DWORD>(data.size()), &bytes_written, nullptr); #else // there is no stderr #endif #endif }
int LMDBEngine::Get(const Slice& key, std::string* value, bool fill_cache) { MDB_val k, v; k.mv_data = const_cast<char*>(key.data()); k.mv_size = key.size(); int rc; LMDBContext& holder = m_ctx_local.GetValue(); MDB_txn *txn = holder.readonly_txn; if (NULL == holder.readonly_txn) { rc = mdb_txn_begin(m_env, NULL, MDB_RDONLY, &txn); if (rc != 0) { ERROR_LOG("Failed to create txn for get for reason:%s", mdb_strerror(rc)); return -1; } } rc = mdb_get(txn, m_dbi, &k, &v); if (NULL == holder.readonly_txn) { mdb_txn_commit(txn); } if (0 == rc && NULL != value && NULL != v.mv_data) { value->assign((const char*) v.mv_data, v.mv_size); } return rc; }
// Parse a length-prefixed number // Format: 0x02 <length-byte> <number> static boost::optional<Slice> sigPart (Slice& buf) { if (buf.size() < 3 || buf[0] != 0x02) return boost::none; auto const len = buf[1]; buf += 2; if (len > buf.size() || len < 1 || len > 33) return boost::none; // Can't be negative if ((buf[0] & 0x80) != 0) return boost::none; if (buf[0] == 0) { // Can't be zero if (len == 1) return boost::none; // Can't be padded if ((buf[1] & 0x80) == 0) return boost::none; } boost::optional<Slice> number = Slice(buf.data(), len); buf += len; return number; }
bool Ardb::WakeBlockList(Context& ctx, const Slice& key, const std::string& value) { DBItemKey kk; kk.db = ctx.currentDB; kk.key.assign(key.data(), key.size()); WriteLockGuard<SpinRWLock> guard(m_block_ctx_lock); BlockContextTable::iterator fit = m_block_context_table.find(kk); if (fit != m_block_context_table.end()) { while (!fit->second.empty()) { Context* cctx = fit->second.front(); if (NULL != cctx&& cctx->block != NULL && NULL != cctx->client && cctx->block->block_state == BLOCK_STATE_BLOCKED) { cctx->block->waked_key = kk; cctx->block->waked_value = value; cctx->block->block_state = BLOCK_STATE_WAKING; cctx->client->GetService().AsyncIO(cctx->client->GetID(), WakeBlockedConnCallback, cctx); fit->second.pop_front(); return true; } fit->second.pop_front(); } } return false; }
virtual bool KeyMayMatch(const Slice& key, const Slice& bloom_filter) const { const size_t len = bloom_filter.size(); if (len < 2) return false; const char* array = bloom_filter.data(); const size_t bits = (len - 1) * 8; // Use the encoded k so that we can read filters generated by // bloom filters created using different parameters. const size_t k = array[len-1]; if (k > 30) { // Reserved for potentially new encodings for short bloom filters. // Consider it a match. return true; } uint32_t h = BloomHash(key); const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits for (size_t j = 0; j < k; j++) { const uint32_t bitpos = h % bits; if ((array[bitpos/8] & (1 << (bitpos % 8))) == 0) return false; h += delta; } return true; }
inline ValueType ExtractValueType(const Slice& internal_key) { assert(internal_key.size() >= 8); const size_t n = internal_key.size(); uint64_t num = DecodeFixed64(internal_key.data() + n - 8); unsigned char c = num & 0xff; return static_cast<ValueType>(c); }
bool LeafNode::read_bucket(BlockReader& reader, size_t compressed_length, size_t uncompressed_length, RecordBucket *bucket, Slice buffer) { if (tree_->compressor_) { // 1. uncompress assert(compressed_length <= reader.remain()); assert(uncompressed_length <= buffer.size()); // 1. uncompress if (!tree_->compressor_->uncompress(reader.addr(), compressed_length, (char *)buffer.data())) { return false; } reader.skip(compressed_length); // 2. deserialize Block block(buffer, 0, uncompressed_length); BlockReader rr(&block); return read_bucket(rr, bucket); } else { return read_bucket(reader, bucket); } }
Status LocalDBWriter::file_write(const uint64_t& key, const Slice& value) { char* a = "xxxxxxxxxxxxxxxx"; // 为了内存对齐,一个内存页应该放多少个内存索引key,仅仅想计算一次 static uint64_t ssd_align_key = PAGESIZE / (sizeof(key_t) + sizeof(ssd_value_t)); static uint64_t mem_align_key = PAGESIZE / (sizeof(key_t) + sizeof(mem_value_t)) * ssd_align_key; if (_key_num != 0 && _key_num % mem_align_key == 0) { // //printf("1_key%lu mem_align_key:%lu\n", key, mem_align_key); check_write(_mem_index_fd, a, PAGESIZE % (sizeof(key_t) + sizeof(mem_value_t)), _mem_file_off); } if (_key_num % ssd_align_key == 0) { // 知道为0,省去 check_write(_mem_index_fd, &key, sizeof(key), _mem_file_off); uint32_t ssd_page_num = _key_num / ssd_align_key; check_write(_mem_index_fd, &ssd_page_num, sizeof(ssd_page_num), _mem_file_off); } check_write(_ssd_index_fd, &key, sizeof(key), _ssd_file_off); check_write(_ssd_index_fd, &_data_file_off, sizeof(_data_file_off), _ssd_file_off); // //printf("data off:%lu ssd off:%lu \n", _data_file_off, _ssd_file_off); uint32_t value_len = value.size(); check_write(_data_file_fd, &value_len, sizeof(value_len), _data_file_off); check_write(_data_file_fd, value.data(), value_len, _data_file_off); // //printf("data off:%lu \n", _data_file_off); return OK; }
bool InnerNode::write_msgbuf(BlockWriter& writer, MsgBuf *mb, Slice buffer) { if (tree_->compressor_) { // 1. write to buffer Block block(buffer, 0, 0); BlockWriter wr(&block); if (!mb->write_to(wr)) return false; // 2. compress assert(tree_->compressor_->max_compressed_length(block.size()) <= writer.remain()); size_t compressed_length; if (!tree_->compressor_->compress(buffer.data(), block.size(), writer.addr(), &compressed_length)) { LOG_ERROR("compress msgbuf error, nid " << nid_); return false; } // 3. skip writer.skip(compressed_length); return true; } else { return mb->write_to(writer); } }
int WiredTigerEngine::Get(const Slice& key, std::string* value, const Options& options) { WT_SESSION* session = GetContextHolder().session; if (NULL == session) { return -1; } WT_CURSOR *cursor = create_wiredtiger_cursor(session); if (NULL == cursor) { return -1; } WT_ITEM key_item, value_item; key_item.data = key.data(); key_item.size = key.size(); cursor->set_key(cursor, &key_item); int ret = 0; if ((ret = cursor->search(cursor)) == 0) { ret = cursor->get_value(cursor, &value_item); if (0 == ret) { if (NULL != value) { value->assign((const char*) value_item.data, value_item.size); } } else { CHECK_WT_RETURN(ret); } } cursor->close(cursor); return ret; }
PublicKey::PublicKey (Slice const& slice) { if(! publicKeyType(slice)) LogicError("PublicKey::PublicKey invalid type"); size_ = slice.size(); std::memcpy(buf_, slice.data(), size_); }
bool LeafNode::write_bucket(BlockWriter& writer, RecordBucket *bucket, Slice buffer) { if (tree_->compressor_) { // 1. write to buffer Block block(buffer, 0, 0); BlockWriter wr(&block); if (!write_bucket(wr, bucket)) return false; // 2. compress assert(tree_->compressor_->max_compressed_length(block.size()) <= writer.remain()); size_t compressed_length; if (!tree_->compressor_->compress(buffer.data(), block.size(), writer.addr(), &compressed_length)) { LOG_ERROR("compress msgbuf error, nid " << nid_); return false; } // 3. skip writer.skip(compressed_length); return true; } else { return write_bucket(writer, bucket); } }
Status::Status(Code code, const Slice& msg, const Slice& msg2) { assert(code != kOk); const uint32_t len1 = msg.size(); const uint32_t len2 = msg2.size(); const uint32_t size = len1 + (len2 ? (2 + len2) : 0); char* result = new char[size + 5]; memcpy(result, &size, sizeof(size)); result[4] = static_cast<char>(code); memcpy(result + 5, msg.data(), len1); if (len2) { result[5 + len1] = ':'; result[6 + len1] = ' '; memcpy(result + 7 + len1, msg2.data(), len2); } state_ = result; }
void Seek(const Slice &target) { char buf[prefix.size() + target.size()]; (void) memcpy(buf, prefix.data(), prefix.size()); (void) memcpy(buf + sizeof(prefix), target.data(), target.size()); impl.Seek(Slice(buf, sizeof(buf))); }
bool signHelper ( RSA* key, Slice message, Buffer& modulus, Buffer& signature) { int const keySize = RSA_size(key); if (!checkModulusLength (keySize)) return false; sha256_hasher h; h (message.data(), message.size()); auto digest = static_cast<sha256_hasher::result_type>(h); Buffer buf; // Pad the result (-1 -> use hash length as salt length) if (!RSA_padding_add_PKCS1_PSS(key, buf.alloc(keySize), digest.data(), EVP_sha256(), -1)) return false; // Sign - we've manually padded the input already. auto ret = RSA_private_encrypt(keySize, buf.data(), signature.alloc (buf.size()), key, RSA_NO_PADDING); if (ret == -1) return false; BN_bn2bin (key->n, modulus.alloc(BN_num_bytes (key->n))); return true; }