// Numerical user_key format: "%s%ld%s",prefix,inc_number,suffix
// if no numbers found, treat it as string
int NumericalComparatorImpl::Compare(const leveldb::Slice& a, const leveldb::Slice& b) const
{
assert(a.size() > LDB_COMPARE_META_ALL_SIZE && b.size() > LDB_COMPARE_META_ALL_SIZE );
int64_t num_a = 0, num_b = 0;
int ret = 0;
const char *prefix_a, *prefix_b, *delimiter_a, *delimiter_b, *suffix_a, *suffix_b;
prefix_a = MetaSkip(a.data() + LDB_COMPARE_META_ALL_SIZE, a.size() - LDB_COMPARE_META_ALL_SIZE);
delimiter_a = FindNumber(prefix_a, a.size() - (prefix_a - a.data()));
prefix_b = MetaSkip(b.data() + LDB_COMPARE_META_ALL_SIZE, b.size() - LDB_COMPARE_META_ALL_SIZE);
delimiter_b = FindNumber(prefix_b, b.size() - (prefix_b - b.data()));
//compare bucket_num+area+meta+prefix
const size_t pre_len_a = delimiter_a - a.data() - LDB_COMPARE_SKIP_SIZE;
const size_t pre_len_b = delimiter_b - b.data() - LDB_COMPARE_SKIP_SIZE;
ret = StringCompare(a.data() + LDB_COMPARE_SKIP_SIZE, pre_len_a, b.data() + LDB_COMPARE_SKIP_SIZE, pre_len_b);
if (ret == 0)
{
//prefixs equal, compare number
num_a = Strntoul(delimiter_a, a.size() - (delimiter_a - a.data()), &suffix_a);
num_b = Strntoul(delimiter_b, b.size() - (delimiter_b - b.data()), &suffix_b);
if (num_a != num_b)
{
ret = (num_a - num_b) > 0 ? 1 : -1;
}
else
{
//numbers equal or no numbers found, compare suffix
const size_t suf_len_a = a.size() - (suffix_a - a.data());
const size_t suf_len_b = b.size() - (suffix_b - b.data());
ret = StringCompare(suffix_a, suf_len_a, suffix_b, suf_len_b);
}
}
return ret;
}
int PaxosComparator :: PCompare(const leveldb::Slice & a, const leveldb::Slice & b)
{
if (a.size() != sizeof(uint64_t))
{
NLErr("assert a.size %zu b.size %zu", a.size(), b.size());
assert(a.size() == sizeof(uint64_t));
}
if (b.size() != sizeof(uint64_t))
{
NLErr("assert a.size %zu b.size %zu", a.size(), b.size());
assert(b.size() == sizeof(uint64_t));
}
uint64_t lla = 0;
uint64_t llb = 0;
memcpy(&lla, a.data(), sizeof(uint64_t));
memcpy(&llb, b.data(), sizeof(uint64_t));
if (lla == llb)
{
return 0;
}
return lla < llb ? -1 : 1;
}
void
persistence_leveldb::retrieve (es::storage& es)
{
std::unique_ptr<leveldb::Iterator> iter (db_->NewIterator(leveldb::ReadOptions()));
uint32_t start_key[2];
start_key[0] = type_entity;
start_key[1] = 0;
uint32_t end_key[2];
end_key[0] = type_entity;
end_key[1] = 0xffffffff;
leveldb::Slice start (reinterpret_cast<const char*>(start_key), sizeof(start_key));
leveldb::Slice end (reinterpret_cast<const char*>(end_key), sizeof(end_key));
for (iter->Seek(start);
iter->Valid() && options_.comparator->Compare(iter->key(), end) <= 0;
iter->Next())
{
const leveldb::Slice value (iter->value());
if (value.empty())
{
continue;
}
const leveldb::Slice key (iter->key());
const uint32_t entity (*reinterpret_cast<const uint32_t*>(key.data() + 4));
es.deserialize(es.make(entity), { value.data(), value.data() + value.size() });
}
}
Binlog::Binlog(uint64_t seq, char type, char cmd, const leveldb::Slice &key, const leveldb::Slice &val){
buf.append((char *)(&seq), sizeof(uint64_t));
buf.push_back(type);
buf.push_back(cmd);
buf.append(key.data(), key.size());
val_buf.append(val.data(), val.size());
}
static inline uint64_t decode_seq_key(const leveldb::Slice &key){
uint64_t seq = 0;
if(key.size() == (sizeof(uint64_t) + 1) && key.data()[0] == DataType::SYNCLOG){
seq = *((uint64_t *)(key.data() + 1));
seq = big_endian(seq);
}
return seq;
}
int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const
{
int ret;
PyObject* bytes_a;
PyObject* bytes_b;
PyObject* compare_result;
PyGILState_STATE gstate;
gstate = PyGILState_Ensure();
/* Create two Python byte strings */
bytes_a = PyBytes_FromStringAndSize(a.data(), a.size());
bytes_b = PyBytes_FromStringAndSize(b.data(), b.size());
if ((bytes_a == NULL) || (bytes_b == NULL)) {
PyErr_Print();
std::cerr << "FATAL ERROR: Plyvel comparator could not allocate byte strings" << std::endl;
std::cerr << "Aborting to avoid database corruption..." << std::endl;
abort();
}
/* Invoke comparator callable */
compare_result = PyObject_CallFunctionObjArgs(comparator, bytes_a, bytes_b, 0);
if (compare_result == NULL) {
PyErr_Print();
std::cerr << "FATAL ERROR: Exception raised from custom Plyvel comparator" << std::endl;
std::cerr << "Aborting to avoid database corruption..." << std::endl;
abort();
}
/* The comparator callable can return any Python object. Compare it
* to our "0" value to get a -1, 0, or 1 for LevelDB. */
if (PyObject_RichCompareBool(compare_result, zero, Py_GT) == 1) {
ret = 1;
} else if (PyObject_RichCompareBool(compare_result, zero, Py_LT) == 1) {
ret = -1;
} else {
ret = 0;
}
if (PyErr_Occurred()) {
PyErr_Print();
std::cerr << "FATAL ERROR: Exception raised while comparing custom Plyvel comparator result with 0" << std::endl;
std::cerr << "Aborting to avoid database corruption..." << std::endl;
abort();
}
Py_DECREF(compare_result);
Py_DECREF(bytes_a);
Py_DECREF(bytes_b);
PyGILState_Release(gstate);
return ret;
}
int Binlog::load(const leveldb::Slice &s){
if(s.size() < HEADER_LEN){
return -1;
}
buf.assign(s.data(), s.size());
return 0;
}
Binlog::Binlog(uint64_t seq, char type, char cmd, const leveldb::Slice &key, int64_t ttl){
buf.append((char *)(&seq), sizeof(uint64_t));
buf.push_back(type);
buf.push_back(cmd);
buf.append(key.data(), key.size());
buf.append(reinterpret_cast<char*>(&ttl), sizeof(ttl));
}
void load_checksum(leveldb::Slice key)
{
const uint8_t* begin = slice_begin(key.data());
const uint8_t* end = begin + key.size();
BITCOIN_ASSERT(key.size() == 1 + short_hash_size + 8);
auto deserial = make_deserializer(begin + 1 + short_hash_size, end);
checksum_ = deserial.read_8_bytes();
BITCOIN_ASSERT(deserial.iterator() == end);
}
// skip expired time
int BitcmpLdbComparatorImpl::Compare(const leveldb::Slice& a, const leveldb::Slice& b) const
{
assert(a.size() > LDB_COMPARE_SKIP_SIZE && b.size() > LDB_COMPARE_SKIP_SIZE);
const int min_len = (a.size() < b.size()) ? a.size() - LDB_COMPARE_SKIP_SIZE :
b.size() - LDB_COMPARE_SKIP_SIZE;
int r = memcmp(a.data() + LDB_COMPARE_SKIP_SIZE, b.data() + LDB_COMPARE_SKIP_SIZE, min_len);
if (r == 0)
{
if (a.size() < b.size())
{
r = -1;
}
else if (a.size() > b.size())
{
r = +1;
}
}
return r;
}
void load_data(leveldb::Slice data)
{
const uint8_t* begin = slice_begin(data.data());
const uint8_t* end = begin + data.size();
BITCOIN_ASSERT(data.size() == 36 + 4);
auto deserial = make_deserializer(begin, end);
inpoint_.hash = deserial.read_hash();
inpoint_.index = deserial.read_4_bytes();
height_ = deserial.read_4_bytes();
BITCOIN_ASSERT(deserial.iterator() == end);
}
void c_seq::seq_set(leveldb::WriteBatch& bh, leveldb::Slice& value)
{
uint32 keyId = m_seq_head.g_index() + 1;
m_seq_head.s_index(keyId);
m_seq_head.s_count(m_seq_head.g_count() + 1);
_zmsg_head head;
head.type = T_SEQ_VALUE;
head.s_effective(time(0));
head.s_crc(c_crc::crc16(0, (uint8*)value.data(), value.size()));
memcpy(TSEQ_BUF(), &head, sizeof(_zmsg_head));
memcpy(TSEQ_BUF() + sizeof(_zmsg_head), (void*)value.data(), value.size());
int len = sizeof(_zmsg_head) + value.size();
if (len > 0)
{
leveldb::Slice data(TSEQ_BUF(), len);
bh.Put(__tos(m_key << "@" << keyId), data);
}
}
void ldb_key_printer(const leveldb::Slice& key, std::string& output)
{
// we only care bucket number, area and first byte of key now
if (key.size() < LDB_KEY_META_SIZE + LDB_KEY_AREA_SIZE + 1)
{
log_error("invalid ldb key. igore print");
output.append("DiRtY");
}
else
{
char buf[32];
int32_t skip = 0;
// bucket number
skip += snprintf(buf + skip, sizeof(buf) - skip, "%d",
LdbKey::decode_bucket_number(key.data() + LDB_EXPIRED_TIME_SIZE));
// area
skip += snprintf(buf + skip, sizeof(buf) - skip, "-%d", LdbKey::decode_area(key.data() + LDB_KEY_META_SIZE));
// first byte of key
skip += snprintf(buf + skip, sizeof(buf) - skip, "-0x%X", *(key.data() + LDB_KEY_META_SIZE + LDB_KEY_AREA_SIZE));
output.append(buf);
}
}
// input format: version(char) + seq(uint64_t) + type(char) + cmd(char) + klen(uint32_t) + vlen(uint32_t) + key + val
// 注：buf 的变量存储的数据格式保持了和官网一致, 新扩展的val放到val_buf
int Binlog::load_log_data(const leveldb::Slice &s){
const char* data = s.data();
// 1 => version(char), 2 * sizeof(uint32_t) => key_size(uint32_t) + val_size(uint32_t);
// HEADER_LEN = seq(uint64_t) + type(char) + cmd(char)
const uint32_t VERSION_LEN = 1;
const uint32_t new_header_len = VERSION_LEN + HEADER_LEN;
uint32_t minLen = new_header_len + 2 * sizeof(uint32_t);
if (s.size() < minLen) {
return -1;
}
uint32_t klen = *(uint32_t *)(data + new_header_len);
uint32_t vlen = *(uint32_t *)(data + new_header_len + sizeof(uint32_t));
if (s.size() < minLen + klen + vlen) {
return -1;
}
buf.assign(data + VERSION_LEN, HEADER_LEN);
buf.append(data + new_header_len + 2 * sizeof(uint32_t), klen);
val_buf.assign(data + new_header_len + 2 * sizeof(uint32_t) + klen, vlen);
return 0;
}
void MyReplication::Put(uint64_t seq, const leveldb::Slice& key, const leveldb::Slice& val){
Synclog log(seq, Synclog::SET, key);
log_trace("%llu, set %s", seq, hexmem(key.data(), key.size()).c_str());
logs->put(log);
}
bool BitcmpLdbComparatorImpl::ShouldStopBefore(const leveldb::Slice& start_key, const leveldb::Slice& key) const
{
return LdbKey::decode_bucket_number_with_key(key.data()) !=
LdbKey::decode_bucket_number_with_key(start_key.data());
}
static LevelDBSlice makeLevelDBSlice(const leveldb::Slice& s)
{
return LevelDBSlice(s.data(), s.data() + s.size());
}
Synclog::Synclog(uint64_t seq, char type, const leveldb::Slice &key){
buf.append((char *)(&seq), sizeof(uint64_t));
buf.push_back(type);
buf.append(key.data(), key.size());
}
int LevelDBComparator::Compare(const leveldb::Slice& a, const leveldb::Slice& b) const
{
return ardb_compare_keys(a.data(), a.size(), b.data(), b.size());
}
static void ParseSlice(const leveldb::Slice &slice, size_t prefix_size, TThat &that) {
std::string blob(slice.data() + prefix_size, slice.size() - prefix_size);
std::istringstream strm(std::move(blob));
LevelDB::Parse(strm, that);
}
void MyReplication::Delete(uint64_t seq, const leveldb::Slice& key){
Synclog log(seq, Synclog::DEL, key);
log_trace("%llu, del %s", seq, hexmem(key.data(), key.size()).c_str());
logs->put(log);
}
/*
* Hash key format:
* +---------------------------------------+
* | Type | Field-Length | Field | Key |
* +---------------------------------------+
* | 1byte | varuint | | |
* +---------------------------------------+
*/
void encode_hash_key(leveldb::Slice user_key, leveldb::Slice field, std::string* hash_key) {
hash_key->assign(1, DataType::DATA_HASH);
put_varuint(hash_key, field.size());
hash_key->append(field.data(), field.size());
hash_key->append(user_key.data(), user_key.size());
}