int Binlog::load(const leveldb::Slice &s){
if(s.size() < HEADER_LEN){
return -1;
}
buf.assign(s.data(), s.size());
return 0;
}
Binlog::Binlog(uint64_t seq, char type, char cmd, const leveldb::Slice &key, int64_t ttl){
buf.append((char *)(&seq), sizeof(uint64_t));
buf.push_back(type);
buf.push_back(cmd);
buf.append(key.data(), key.size());
buf.append(reinterpret_cast<char*>(&ttl), sizeof(ttl));
}
int PaxosComparator :: PCompare(const leveldb::Slice & a, const leveldb::Slice & b)
{
if (a.size() != sizeof(uint64_t))
{
NLErr("assert a.size %zu b.size %zu", a.size(), b.size());
assert(a.size() == sizeof(uint64_t));
}
if (b.size() != sizeof(uint64_t))
{
NLErr("assert a.size %zu b.size %zu", a.size(), b.size());
assert(b.size() == sizeof(uint64_t));
}
uint64_t lla = 0;
uint64_t llb = 0;
memcpy(&lla, a.data(), sizeof(uint64_t));
memcpy(&llb, b.data(), sizeof(uint64_t));
if (lla == llb)
{
return 0;
}
return lla < llb ? -1 : 1;
}
virtual void Put(const leveldb::Slice& key, const leveldb::Slice& value) {
if (key.ToString() == needle) {
foundEntry = true;
*deleted = false;
*foundValue = value.ToString();
}
}
static inline uint64_t decode_seq_key(const leveldb::Slice &key){
uint64_t seq = 0;
if(key.size() == (sizeof(uint64_t) + 1) && key.data()[0] == DataType::SYNCLOG){
seq = *((uint64_t *)(key.data() + 1));
seq = big_endian(seq);
}
return seq;
}
Binlog::Binlog(uint64_t seq, char type, char cmd, const leveldb::Slice &key, const leveldb::Slice &val){
buf.append((char *)(&seq), sizeof(uint64_t));
buf.push_back(type);
buf.push_back(cmd);
buf.append(key.data(), key.size());
val_buf.append(val.data(), val.size());
}
void load_checksum(leveldb::Slice key)
{
const uint8_t* begin = slice_begin(key.data());
const uint8_t* end = begin + key.size();
BITCOIN_ASSERT(key.size() == 1 + short_hash_size + 8);
auto deserial = make_deserializer(begin + 1 + short_hash_size, end);
checksum_ = deserial.read_8_bytes();
BITCOIN_ASSERT(deserial.iterator() == end);
}
int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const
{
int ret;
PyObject* bytes_a;
PyObject* bytes_b;
PyObject* compare_result;
PyGILState_STATE gstate;
gstate = PyGILState_Ensure();
/* Create two Python byte strings */
bytes_a = PyBytes_FromStringAndSize(a.data(), a.size());
bytes_b = PyBytes_FromStringAndSize(b.data(), b.size());
if ((bytes_a == NULL) || (bytes_b == NULL)) {
PyErr_Print();
std::cerr << "FATAL ERROR: Plyvel comparator could not allocate byte strings" << std::endl;
std::cerr << "Aborting to avoid database corruption..." << std::endl;
abort();
}
/* Invoke comparator callable */
compare_result = PyObject_CallFunctionObjArgs(comparator, bytes_a, bytes_b, 0);
if (compare_result == NULL) {
PyErr_Print();
std::cerr << "FATAL ERROR: Exception raised from custom Plyvel comparator" << std::endl;
std::cerr << "Aborting to avoid database corruption..." << std::endl;
abort();
}
/* The comparator callable can return any Python object. Compare it
* to our "0" value to get a -1, 0, or 1 for LevelDB. */
if (PyObject_RichCompareBool(compare_result, zero, Py_GT) == 1) {
ret = 1;
} else if (PyObject_RichCompareBool(compare_result, zero, Py_LT) == 1) {
ret = -1;
} else {
ret = 0;
}
if (PyErr_Occurred()) {
PyErr_Print();
std::cerr << "FATAL ERROR: Exception raised while comparing custom Plyvel comparator result with 0" << std::endl;
std::cerr << "Aborting to avoid database corruption..." << std::endl;
abort();
}
Py_DECREF(compare_result);
Py_DECREF(bytes_a);
Py_DECREF(bytes_b);
PyGILState_Release(gstate);
return ret;
}
void load_data(leveldb::Slice data)
{
const uint8_t* begin = slice_begin(data.data());
const uint8_t* end = begin + data.size();
BITCOIN_ASSERT(data.size() == 36 + 4);
auto deserial = make_deserializer(begin, end);
inpoint_.hash = deserial.read_hash();
inpoint_.index = deserial.read_4_bytes();
height_ = deserial.read_4_bytes();
BITCOIN_ASSERT(deserial.iterator() == end);
}
static std::vector<std::string> uniform_init(const leveldb::Slice& begin,
const leveldb::Slice& end,
int K, int prefix_len = 0) {
auto ibegin = std::stoull(begin.ToString().substr(prefix_len));
auto iend = std::stoull(end.ToString().substr(prefix_len));
std::vector<std::string> ret(K);
for (auto i = ibegin; i < iend; i += (iend - ibegin) / K) {
ret.push_back(begin.ToString().substr(0, prefix_len) + std::to_string(i));
}
return ret;
}
int EventComparator::Compare(const leveldb::Slice &a, const leveldb::Slice &b) const {
pair < string, size_t > key1 = GtidHandler::ParseGTID(a.ToString());
pair < string, size_t > key2 = GtidHandler::ParseGTID(b.ToString());
if (key1.first == key2.first) {
if (key1.second == key2.second)
return 0;
return key1.second < key2.second ? -1 : 1;
}
return key1.first < key2.first ? -1 : 1;
}
std::string Database::get(const leveldb::Slice& key)
{
std::string s;
leveldb::Status status = db_->Get(readOptions_, key, &s);
CHECK(status.ok()) << "key is " << key.ToString();
return s;
}
// skip expired time
void BitcmpLdbComparatorImpl::FindShortestSeparator(
std::string* start,
const leveldb::Slice& limit) const
{
// Find length of common prefix
size_t min_length = std::min(start->size(), limit.size());
assert(min_length > LDB_COMPARE_SKIP_SIZE);
size_t diff_index = LDB_COMPARE_SKIP_SIZE;
while ((diff_index < min_length) &&
((*start)[diff_index] == limit[diff_index]))
{
diff_index++;
}
if (diff_index >= min_length) {
// Do not shorten if one string is a prefix of the other
}
else
{
uint8_t diff_byte = static_cast<uint8_t>((*start)[diff_index]);
if (diff_byte < static_cast<uint8_t>(0xff) &&
diff_byte + 1 < static_cast<uint8_t>(limit[diff_index]))
{
(*start)[diff_index]++;
start->resize(diff_index + 1);
assert(Compare(*start, limit) < 0);
}
}
}
int get_centroid(leveldb::DB* work_db, const leveldb::Slice& key, int K) {
std::string str;
auto keystr = get_centroid_prefix(-1, K) + key.ToString();
auto status = work_db->Get({}, keystr, &str);
if (status.IsNotFound()) return -1;
CHECK_OK(status);
return std::stoi(str);
}
virtual void Delete(const leveldb::Slice& key)
{
if (key.ToString() == needle)
{
foundEntry = true;
*deleted = true;
};
};
void
persistence_leveldb::retrieve (es::storage& es)
{
std::unique_ptr<leveldb::Iterator> iter (db_->NewIterator(leveldb::ReadOptions()));
uint32_t start_key[2];
start_key[0] = type_entity;
start_key[1] = 0;
uint32_t end_key[2];
end_key[0] = type_entity;
end_key[1] = 0xffffffff;
leveldb::Slice start (reinterpret_cast<const char*>(start_key), sizeof(start_key));
leveldb::Slice end (reinterpret_cast<const char*>(end_key), sizeof(end_key));
for (iter->Seek(start);
iter->Valid() && options_.comparator->Compare(iter->key(), end) <= 0;
iter->Next())
{
const leveldb::Slice value (iter->value());
if (value.empty())
{
continue;
}
const leveldb::Slice key (iter->key());
const uint32_t entity (*reinterpret_cast<const uint32_t*>(key.data() + 4));
es.deserialize(es.make(entity), { value.data(), value.data() + value.size() });
}
}
void MojDbLevelTableTxn::Put(const leveldb::Slice& key,
const leveldb::Slice& val)
{
// populate local view
std::string keyStr = key.ToString();
m_pendingValues[keyStr] = val.ToString();
m_pendingDeletes.erase(keyStr); // if hidden before
// notify all iterators
for(std::set<MojDbLevelTxnIterator*>::const_iterator i = m_iterators.begin();
i != m_iterators.end();
++i)
{
(*i)->notifyPut(key);
}
}
void set_centroid(leveldb::DB* work_db, const leveldb::Slice& key,
int oldc, int newc, int K) {
if (oldc == newc) return;
auto rawkey = key.ToString();
auto keystr = get_centroid_prefix(-1, K) + rawkey;
CHECK_OK(work_db->Put({}, key, std::to_string(newc)));
CHECK_OK(work_db->Delete({}, get_centroid_prefix(oldc, K) + rawkey));
CHECK_OK(work_db->Put({}, get_centroid_prefix(newc, K) + rawkey, key));
}
static void lookup(leveldb::DB* db, const leveldb::Slice& key, GDELTMini& val) {
std::string strval;
leveldb::Status status = db->Get({}, key, &strval);
if (status.IsNotFound()) {
std::cerr << "Key " << key.ToString() << " not found." << std::endl;
}
CHECK_OK(status);
std::stringstream sstr;
sstr.str(std::move(strval));
sstr >> val;
}
bool LdbLogsReader::parse_one_key(int32_t type, leveldb::Slice& key)
{
bool ret = leveldb::GetLengthPrefixedSlice(last_log_record_, &key);
log_debug("@@ parse key ret : %d", ret);
if (ret && filter_ != NULL && filter_->ok(type, key, last_sequence_))
{
// skip this key
key.clear();
}
return ret;
}
void c_seq::seq_set(leveldb::WriteBatch& bh, leveldb::Slice& value)
{
uint32 keyId = m_seq_head.g_index() + 1;
m_seq_head.s_index(keyId);
m_seq_head.s_count(m_seq_head.g_count() + 1);
_zmsg_head head;
head.type = T_SEQ_VALUE;
head.s_effective(time(0));
head.s_crc(c_crc::crc16(0, (uint8*)value.data(), value.size()));
memcpy(TSEQ_BUF(), &head, sizeof(_zmsg_head));
memcpy(TSEQ_BUF() + sizeof(_zmsg_head), (void*)value.data(), value.size());
int len = sizeof(_zmsg_head) + value.size();
if (len > 0)
{
leveldb::Slice data(TSEQ_BUF(), len);
bh.Put(__tos(m_key << "@" << keyId), data);
}
}
leveldb::Status MojDbLevelTableTxn::Get(const leveldb::Slice& key,
std::string& val)
{
std::string keyStr = key.ToString();
// for keys deleted in this transaction
if (m_pendingDeletes.count(keyStr) > 0)
{
return leveldb::Status::NotFound("Deleted inside transaction");
}
// for keys added in this transaction
PendingValues::const_iterator it = m_pendingValues.find(key.ToString());
if (it != m_pendingValues.end())
{
val = it->second;
return leveldb::Status::OK();
}
// go directly to db
return m_db->Get(leveldb::ReadOptions(), key, &val);
}
void ldb_key_printer(const leveldb::Slice& key, std::string& output)
{
// we only care bucket number, area and first byte of key now
if (key.size() < LDB_KEY_META_SIZE + LDB_KEY_AREA_SIZE + 1)
{
log_error("invalid ldb key. igore print");
output.append("DiRtY");
}
else
{
char buf[32];
int32_t skip = 0;
// bucket number
skip += snprintf(buf + skip, sizeof(buf) - skip, "%d",
LdbKey::decode_bucket_number(key.data() + LDB_EXPIRED_TIME_SIZE));
// area
skip += snprintf(buf + skip, sizeof(buf) - skip, "-%d", LdbKey::decode_area(key.data() + LDB_KEY_META_SIZE));
// first byte of key
skip += snprintf(buf + skip, sizeof(buf) - skip, "-0x%X", *(key.data() + LDB_KEY_META_SIZE + LDB_KEY_AREA_SIZE));
output.append(buf);
}
}
// input format: version(char) + seq(uint64_t) + type(char) + cmd(char) + klen(uint32_t) + vlen(uint32_t) + key + val
// 注：buf 的变量存储的数据格式保持了和官网一致, 新扩展的val放到val_buf
int Binlog::load_log_data(const leveldb::Slice &s){
const char* data = s.data();
// 1 => version(char), 2 * sizeof(uint32_t) => key_size(uint32_t) + val_size(uint32_t);
// HEADER_LEN = seq(uint64_t) + type(char) + cmd(char)
const uint32_t VERSION_LEN = 1;
const uint32_t new_header_len = VERSION_LEN + HEADER_LEN;
uint32_t minLen = new_header_len + 2 * sizeof(uint32_t);
if (s.size() < minLen) {
return -1;
}
uint32_t klen = *(uint32_t *)(data + new_header_len);
uint32_t vlen = *(uint32_t *)(data + new_header_len + sizeof(uint32_t));
if (s.size() < minLen + klen + vlen) {
return -1;
}
buf.assign(data + VERSION_LEN, HEADER_LEN);
buf.append(data + new_header_len + 2 * sizeof(uint32_t), klen);
val_buf.assign(data + new_header_len + 2 * sizeof(uint32_t) + klen, vlen);
return 0;
}
void MojDbLevelTableTxn::Delete(const leveldb::Slice& key)
{
// notify all iterators
for(std::set<MojDbLevelTxnIterator*>::const_iterator i = m_iterators.begin();
i != m_iterators.end();
++i)
{
(*i)->notifyDelete(key);
}
// populate local view
std::string keyStr = key.ToString();
m_pendingDeletes.insert(keyStr);
m_pendingValues.erase(keyStr); // if set before
// XXX: work around for delete by query
// make this delete visible to cursors
//m_db->Delete(m_writeOptions, key);
}
bool LdbLogsReader::parse_one_kv(int32_t type, leveldb::Slice& key, leveldb::Slice& value)
{
bool ret = leveldb::GetLengthPrefixedSlice(last_log_record_, &key);
log_debug("@@ parse key ret : %d", ret);
if (ret)
{
if (filter_ == NULL || filter_->ok(type, key, last_sequence_))
{
ret = leveldb::GetLengthPrefixedSlice(last_log_record_, &value);
}
else
{
// skip this key
key.clear();
// skip value
ret = leveldb::SkipLengthPrefixedSlice(last_log_record_);
}
}
return ret;
}
static std::vector<IterRange> range_uniform(const leveldb::Slice& begin,
const leveldb::Slice& end,
int K, leveldb::DB* db,
int size) {
int jump = size / K;
std::vector<IterRange> ret(K);
if (jump == 0) {
jump = 1;
for (int i = size; i < K; ++i) {
ret[i].reset = []() {};
ret[i].should_continue = []() { return false; };
}
K = size;
}
auto it = iter(db);
it->Seek(begin);
CHECK(it->Valid());
int i = 0;
for (; i < K; ++i) {
CHECK(it->Valid());
CHECK(it->key().compare(end) < 0);
ret[i].it = iter(db);
std::string start = it->key().ToString();
auto ptr = ret[i].it.get();
ret[i].reset = [=]() {
ptr->Seek(start);
CHECK(ptr->Valid());
};
for (int j = 0; j < jump; ++j) {
it->Next();
CHECK(it->Valid());
}
std::string lend = it->key().ToString();
if (i == K - 1) lend = end.ToString();
ret[i].should_continue = [=]() {
return ptr->Valid() && ptr->key().compare(lend) < 0;
};
}
return ret;
}
int LdbLogsReader::get_record(int32_t& type, leveldb::Slice& key, leveldb::Slice& value, bool& alloc)
{
alloc = false;
int ret = TAIR_RETURN_SUCCESS;
while (true)
{
if (last_log_record_->size() <= 0)
{
ret = get_log_record();
log_debug("@@ get new record: %lu", last_log_record_->size());
}
// read one log record
if (TAIR_RETURN_SUCCESS == ret && last_log_record_->size() > 0)
{
update_last_sequence();
ret = parse_one_kv_record(type, key, value);
// this kv record is skipped, just try next
if (TAIR_RETURN_SUCCESS == ret && key.empty())
{
continue;
}
else
{
// read fail or got a kv
break;
}
}
else
{
// read fail or read over
break;
}
}
return ret;
}
// skip expired time
int BitcmpLdbComparatorImpl::Compare(const leveldb::Slice& a, const leveldb::Slice& b) const
{
assert(a.size() > LDB_COMPARE_SKIP_SIZE && b.size() > LDB_COMPARE_SKIP_SIZE);
const int min_len = (a.size() < b.size()) ? a.size() - LDB_COMPARE_SKIP_SIZE :
b.size() - LDB_COMPARE_SKIP_SIZE;
int r = memcmp(a.data() + LDB_COMPARE_SKIP_SIZE, b.data() + LDB_COMPARE_SKIP_SIZE, min_len);
if (r == 0)
{
if (a.size() < b.size())
{
r = -1;
}
else if (a.size() > b.size())
{
r = +1;
}
}
return r;
}
bool BitcmpLdbComparatorImpl::ShouldStopBefore(const leveldb::Slice& start_key, const leveldb::Slice& key) const
{
return LdbKey::decode_bucket_number_with_key(key.data()) !=
LdbKey::decode_bucket_number_with_key(start_key.data());
}