예제 #1
0
파일: binlog.cpp 프로젝트: Mumujane/ssdb
int Binlog::load(const leveldb::Slice &s){
	if(s.size() < HEADER_LEN){
		return -1;
	}
	buf.assign(s.data(), s.size());
	return 0;
}
예제 #2
0
파일: binlog.cpp 프로젝트: dolfly/ssdb
Binlog::Binlog(uint64_t seq, char type, char cmd, const leveldb::Slice &key, int64_t ttl){
	buf.append((char *)(&seq), sizeof(uint64_t));
	buf.push_back(type);
	buf.push_back(cmd);
	buf.append(key.data(), key.size());
	buf.append(reinterpret_cast<char*>(&ttl), sizeof(ttl));
}
예제 #3
0
파일: db.cpp 프로젝트: LngMH/phxpaxos
int PaxosComparator :: PCompare(const leveldb::Slice & a, const leveldb::Slice & b) 
{
    if (a.size() != sizeof(uint64_t))
    {
        NLErr("assert a.size %zu b.size %zu", a.size(), b.size());
        assert(a.size() == sizeof(uint64_t));
    }

    if (b.size() != sizeof(uint64_t))
    {
        NLErr("assert a.size %zu b.size %zu", a.size(), b.size());
        assert(b.size() == sizeof(uint64_t));
    }
    
    uint64_t lla = 0;
    uint64_t llb = 0;

    memcpy(&lla, a.data(), sizeof(uint64_t));
    memcpy(&llb, b.data(), sizeof(uint64_t));

    if (lla == llb)
    {
        return 0;
    }

    return lla < llb ? -1 : 1;
}
예제 #4
0
 virtual void Put(const leveldb::Slice& key, const leveldb::Slice& value) {
     if (key.ToString() == needle) {
         foundEntry = true;
         *deleted = false;
         *foundValue = value.ToString();
     }
 }
예제 #5
0
파일: binlog.cpp 프로젝트: Mumujane/ssdb
static inline uint64_t decode_seq_key(const leveldb::Slice &key){
	uint64_t seq = 0;
	if(key.size() == (sizeof(uint64_t) + 1) && key.data()[0] == DataType::SYNCLOG){
		seq = *((uint64_t *)(key.data() + 1));
		seq = big_endian(seq);
	}
	return seq;
}
예제 #6
0
파일: binlog.cpp 프로젝트: xcodecraft/qssdb
Binlog::Binlog(uint64_t seq, char type, char cmd, const leveldb::Slice &key, const leveldb::Slice &val){
	buf.append((char *)(&seq), sizeof(uint64_t));
	buf.push_back(type);
	buf.push_back(cmd);
	buf.append(key.data(), key.size());
    
    val_buf.append(val.data(), val.size());
}
 void load_checksum(leveldb::Slice key)
 {
     const uint8_t* begin = slice_begin(key.data());
     const uint8_t* end = begin + key.size();
     BITCOIN_ASSERT(key.size() == 1 + short_hash_size + 8);
     auto deserial = make_deserializer(begin + 1 + short_hash_size, end);
     checksum_ = deserial.read_8_bytes();
     BITCOIN_ASSERT(deserial.iterator() == end);
 }
예제 #8
0
    int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const
    {
        int ret;
        PyObject* bytes_a;
        PyObject* bytes_b;
        PyObject* compare_result;
        PyGILState_STATE gstate;

        gstate = PyGILState_Ensure();

        /* Create two Python byte strings */
        bytes_a = PyBytes_FromStringAndSize(a.data(), a.size());
        bytes_b = PyBytes_FromStringAndSize(b.data(), b.size());

        if ((bytes_a == NULL) || (bytes_b == NULL)) {
            PyErr_Print();
            std::cerr << "FATAL ERROR: Plyvel comparator could not allocate byte strings" << std::endl;
            std::cerr << "Aborting to avoid database corruption..." << std::endl;
            abort();
        }

        /* Invoke comparator callable */
        compare_result = PyObject_CallFunctionObjArgs(comparator, bytes_a, bytes_b, 0);

        if (compare_result == NULL) {
            PyErr_Print();
            std::cerr << "FATAL ERROR: Exception raised from custom Plyvel comparator" << std::endl;
            std::cerr << "Aborting to avoid database corruption..." << std::endl;
            abort();
        }

        /* The comparator callable can return any Python object. Compare it
         * to our "0" value to get a -1, 0, or 1 for LevelDB. */
        if (PyObject_RichCompareBool(compare_result, zero, Py_GT) == 1) {
            ret = 1;
        } else if (PyObject_RichCompareBool(compare_result, zero, Py_LT) == 1) {
            ret = -1;
        } else {
            ret = 0;
        }

        if (PyErr_Occurred()) {
            PyErr_Print();
            std::cerr << "FATAL ERROR: Exception raised while comparing custom Plyvel comparator result with 0" << std::endl;
            std::cerr << "Aborting to avoid database corruption..." << std::endl;
            abort();
        }

        Py_DECREF(compare_result);
        Py_DECREF(bytes_a);
        Py_DECREF(bytes_b);

        PyGILState_Release(gstate);

        return ret;
    }
 void load_data(leveldb::Slice data)
 {
     const uint8_t* begin = slice_begin(data.data());
     const uint8_t* end = begin + data.size();
     BITCOIN_ASSERT(data.size() == 36 + 4);
     auto deserial = make_deserializer(begin, end);
     inpoint_.hash = deserial.read_hash();
     inpoint_.index = deserial.read_4_bytes();
     height_ = deserial.read_4_bytes();
     BITCOIN_ASSERT(deserial.iterator() == end);
 }
예제 #10
0
static std::vector<std::string> uniform_init(const leveldb::Slice& begin,
                                             const leveldb::Slice& end,
                                             int K, int prefix_len = 0) {
  auto ibegin = std::stoull(begin.ToString().substr(prefix_len));
  auto iend = std::stoull(end.ToString().substr(prefix_len));
  std::vector<std::string> ret(K);
  for (auto i = ibegin; i < iend; i += (iend - ibegin) / K) {
    ret.push_back(begin.ToString().substr(0, prefix_len) + std::to_string(i));
  }
  return ret;
}
예제 #11
0
int EventComparator::Compare(const leveldb::Slice &a, const leveldb::Slice &b) const {
    pair < string, size_t > key1 = GtidHandler::ParseGTID(a.ToString());
    pair < string, size_t > key2 = GtidHandler::ParseGTID(b.ToString());

    if (key1.first == key2.first) {
        if (key1.second == key2.second)
            return 0;
        return key1.second < key2.second ? -1 : 1;
    }

    return key1.first < key2.first ? -1 : 1;

}
예제 #12
0
파일: db.cpp 프로젝트: NICTA/obsidian
 std::string Database::get(const leveldb::Slice& key)
 {
   std::string s;
   leveldb::Status status = db_->Get(readOptions_, key, &s);
   CHECK(status.ok()) << "key is " << key.ToString();
   return s;
 }
예제 #13
0
      // skip expired time
      void BitcmpLdbComparatorImpl::FindShortestSeparator(
        std::string* start,
        const leveldb::Slice& limit) const
      {
        // Find length of common prefix
        size_t min_length = std::min(start->size(), limit.size());
        assert(min_length > LDB_COMPARE_SKIP_SIZE);
        size_t diff_index = LDB_COMPARE_SKIP_SIZE;
        while ((diff_index < min_length) &&
               ((*start)[diff_index] == limit[diff_index]))
        {
          diff_index++;
        }

        if (diff_index >= min_length) {
          // Do not shorten if one string is a prefix of the other
        }
        else
        {
          uint8_t diff_byte = static_cast<uint8_t>((*start)[diff_index]);
          if (diff_byte < static_cast<uint8_t>(0xff) &&
              diff_byte + 1 < static_cast<uint8_t>(limit[diff_index]))
          {
            (*start)[diff_index]++;
            start->resize(diff_index + 1);
            assert(Compare(*start, limit) < 0);
          }
        }
      }
예제 #14
0
int get_centroid(leveldb::DB* work_db, const leveldb::Slice& key, int K) {
  std::string str;
  auto keystr = get_centroid_prefix(-1, K) + key.ToString();
  auto status = work_db->Get({}, keystr, &str);
  if (status.IsNotFound()) return -1;
  CHECK_OK(status);
  return std::stoi(str);
}
예제 #15
0
 virtual void Delete(const leveldb::Slice& key)
 {
     if (key.ToString() == needle)
     {
         foundEntry = true;
         *deleted = true;
     };
 };
void
persistence_leveldb::retrieve (es::storage& es)
{
    std::unique_ptr<leveldb::Iterator> iter (db_->NewIterator(leveldb::ReadOptions()));

    uint32_t start_key[2];
    start_key[0] = type_entity;
    start_key[1] = 0;

    uint32_t end_key[2];
    end_key[0] = type_entity;
    end_key[1] = 0xffffffff;

    leveldb::Slice start (reinterpret_cast<const char*>(start_key), sizeof(start_key));
    leveldb::Slice end   (reinterpret_cast<const char*>(end_key), sizeof(end_key));

    for (iter->Seek(start);
         iter->Valid() && options_.comparator->Compare(iter->key(), end) <= 0;
         iter->Next())
    {
        const leveldb::Slice value (iter->value());
        if (value.empty())
        {
            continue;
        }
        const leveldb::Slice key (iter->key());
        const uint32_t entity (*reinterpret_cast<const uint32_t*>(key.data() + 4));

        es.deserialize(es.make(entity), { value.data(), value.data() + value.size() });
    }
}
예제 #17
0
void MojDbLevelTableTxn::Put(const leveldb::Slice& key,
                             const leveldb::Slice& val)
{
    // populate local view
    std::string keyStr = key.ToString();

    m_pendingValues[keyStr] = val.ToString();
    m_pendingDeletes.erase(keyStr); // if hidden before

    // notify all iterators
    for(std::set<MojDbLevelTxnIterator*>::const_iterator i = m_iterators.begin();
        i != m_iterators.end();
        ++i)
    {
        (*i)->notifyPut(key);
    }

}
예제 #18
0
void set_centroid(leveldb::DB* work_db, const leveldb::Slice& key,
                  int oldc, int newc, int K) {
  if (oldc == newc) return;
  auto rawkey = key.ToString();
  auto keystr = get_centroid_prefix(-1, K) + rawkey;
  CHECK_OK(work_db->Put({}, key, std::to_string(newc)));
  CHECK_OK(work_db->Delete({}, get_centroid_prefix(oldc, K) + rawkey));
  CHECK_OK(work_db->Put({}, get_centroid_prefix(newc, K) + rawkey, key));
}
예제 #19
0
static void lookup(leveldb::DB* db, const leveldb::Slice& key, GDELTMini& val) {
  std::string strval;
  leveldb::Status status = db->Get({}, key, &strval);
  if (status.IsNotFound()) {
    std::cerr << "Key " << key.ToString() << " not found." << std::endl;
  }
  CHECK_OK(status);
  std::stringstream sstr;
  sstr.str(std::move(strval));
  sstr >> val;
}
예제 #20
0
 bool LdbLogsReader::parse_one_key(int32_t type, leveldb::Slice& key)
 {
   bool ret = leveldb::GetLengthPrefixedSlice(last_log_record_, &key);
   log_debug("@@ parse key ret : %d", ret);
   if (ret && filter_ != NULL && filter_->ok(type, key, last_sequence_))
   {
     // skip this key
     key.clear();  
   }
   return ret;
 }
예제 #21
0
파일: c_seq.cpp 프로젝트: Mignet/zstorage
void c_seq::seq_set(leveldb::WriteBatch& bh, leveldb::Slice& value)
{
	uint32 keyId = m_seq_head.g_index() + 1;

	m_seq_head.s_index(keyId);
	m_seq_head.s_count(m_seq_head.g_count() + 1);

	_zmsg_head head;
	head.type = T_SEQ_VALUE;
	head.s_effective(time(0));
	head.s_crc(c_crc::crc16(0, (uint8*)value.data(), value.size()));

	memcpy(TSEQ_BUF(), &head, sizeof(_zmsg_head));
	memcpy(TSEQ_BUF() + sizeof(_zmsg_head), (void*)value.data(), value.size());
	int len = sizeof(_zmsg_head) + value.size();
	if (len > 0)
	{
		leveldb::Slice data(TSEQ_BUF(), len);
		bh.Put(__tos(m_key << "@" << keyId), data);
	}
}
예제 #22
0
leveldb::Status MojDbLevelTableTxn::Get(const leveldb::Slice& key,
                                        std::string& val)
{
    std::string keyStr = key.ToString();

    // for keys deleted in this transaction
    if (m_pendingDeletes.count(keyStr) > 0)
    {
        return leveldb::Status::NotFound("Deleted inside transaction");
    }

    // for keys added in this transaction
    PendingValues::const_iterator it = m_pendingValues.find(key.ToString());
    if (it != m_pendingValues.end())
    {
        val = it->second;
        return leveldb::Status::OK();
    }

    // go directly to db
    return m_db->Get(leveldb::ReadOptions(), key, &val);
}
예제 #23
0
 void ldb_key_printer(const leveldb::Slice& key, std::string& output)
 {
   // we only care bucket number, area and first byte of key now
   if (key.size() < LDB_KEY_META_SIZE + LDB_KEY_AREA_SIZE + 1)
   {
     log_error("invalid ldb key. igore print");
     output.append("DiRtY");
   }
   else
   {
     char buf[32];
     int32_t skip = 0;
     // bucket number
     skip += snprintf(buf + skip, sizeof(buf) - skip, "%d",
                     LdbKey::decode_bucket_number(key.data() + LDB_EXPIRED_TIME_SIZE));
     // area
     skip += snprintf(buf + skip, sizeof(buf) - skip, "-%d", LdbKey::decode_area(key.data() + LDB_KEY_META_SIZE));
     // first byte of key
     skip += snprintf(buf + skip, sizeof(buf) - skip, "-0x%X", *(key.data() + LDB_KEY_META_SIZE + LDB_KEY_AREA_SIZE));
     output.append(buf);
   }
 }
예제 #24
0
파일: binlog.cpp 프로젝트: xcodecraft/qssdb
// input format: version(char) + seq(uint64_t) + type(char) + cmd(char) + klen(uint32_t) + vlen(uint32_t) + key + val
// 注:buf 的变量存储的数据格式保持了和官网一致, 新扩展的val放到val_buf
int Binlog::load_log_data(const leveldb::Slice &s){
    const char* data = s.data();
    // 1 => version(char), 2 * sizeof(uint32_t) => key_size(uint32_t) + val_size(uint32_t);
    // HEADER_LEN = seq(uint64_t) + type(char) + cmd(char)
    const uint32_t VERSION_LEN = 1;
    const uint32_t new_header_len = VERSION_LEN + HEADER_LEN;
    uint32_t minLen = new_header_len + 2 * sizeof(uint32_t); 
    if (s.size() < minLen) {
        return -1;
    }
    uint32_t klen = *(uint32_t *)(data + new_header_len);
    uint32_t vlen = *(uint32_t *)(data + new_header_len + sizeof(uint32_t));
    if (s.size() < minLen + klen + vlen) {
        return -1;
    }
    
    buf.assign(data + VERSION_LEN, HEADER_LEN);
    buf.append(data + new_header_len + 2 * sizeof(uint32_t), klen);

    val_buf.assign(data + new_header_len + 2 * sizeof(uint32_t) + klen, vlen);

    return 0;
}
예제 #25
0
void MojDbLevelTableTxn::Delete(const leveldb::Slice& key)
{
    // notify all iterators
    for(std::set<MojDbLevelTxnIterator*>::const_iterator i = m_iterators.begin();
        i != m_iterators.end();
        ++i)
    {
        (*i)->notifyDelete(key);
    }

    // populate local view
    std::string keyStr = key.ToString();

    m_pendingDeletes.insert(keyStr);
    m_pendingValues.erase(keyStr); // if set before

    // XXX: work around for delete by query
    // make this delete visible to cursors
    //m_db->Delete(m_writeOptions, key);
}
예제 #26
0
 bool LdbLogsReader::parse_one_kv(int32_t type, leveldb::Slice& key, leveldb::Slice& value)
 {
   bool ret = leveldb::GetLengthPrefixedSlice(last_log_record_, &key);
   log_debug("@@ parse key ret : %d", ret);
   if (ret)
   {
     if (filter_ == NULL || filter_->ok(type, key, last_sequence_))
     {
       ret = leveldb::GetLengthPrefixedSlice(last_log_record_, &value);
     }
     else
     {
       // skip this key
       key.clear();  
       // skip value
       ret = leveldb::SkipLengthPrefixedSlice(last_log_record_);
     }
   }
   return ret;
 }
예제 #27
0
static std::vector<IterRange> range_uniform(const leveldb::Slice& begin,
                                            const leveldb::Slice& end,
                                            int K, leveldb::DB* db,
                                            int size) {
  int jump = size / K;
  std::vector<IterRange> ret(K);
  if (jump == 0) {
    jump = 1;
    for (int i = size; i < K; ++i) {
      ret[i].reset = []() {};
      ret[i].should_continue = []() { return false; };
    }
    K = size;
  }
  auto it = iter(db);
  it->Seek(begin);
  CHECK(it->Valid());
  int i = 0;
  for (; i < K; ++i) {
    CHECK(it->Valid());
    CHECK(it->key().compare(end) < 0);
    ret[i].it = iter(db);
    std::string start = it->key().ToString();
    auto ptr = ret[i].it.get();
    ret[i].reset = [=]() {
      ptr->Seek(start);
      CHECK(ptr->Valid());
    };
    for (int j = 0; j < jump; ++j) {
      it->Next();
      CHECK(it->Valid());
    }
    std::string lend = it->key().ToString();
    if (i == K - 1) lend = end.ToString();
    ret[i].should_continue = [=]() {
      return ptr->Valid() && ptr->key().compare(lend) < 0;
    };
  }
  return ret;
}
예제 #28
0
      int LdbLogsReader::get_record(int32_t& type, leveldb::Slice& key, leveldb::Slice& value, bool& alloc)
      {
        alloc = false;

        int ret = TAIR_RETURN_SUCCESS;

        while (true)
        {
          if (last_log_record_->size() <= 0)
          {
            ret = get_log_record();
            log_debug("@@ get new record: %lu", last_log_record_->size());
          }

          // read one log record
          if (TAIR_RETURN_SUCCESS == ret && last_log_record_->size() > 0)
          {
            update_last_sequence();
            ret = parse_one_kv_record(type, key, value);
            // this kv record is skipped, just try next
            if (TAIR_RETURN_SUCCESS == ret && key.empty())
            {
              continue;
            }
            else
            {
              // read fail or got a kv
              break;
            }
          }
          else
          {
            // read fail or read over
            break;
          }
        }

        return ret;
      }
예제 #29
0
 // skip expired time
 int BitcmpLdbComparatorImpl::Compare(const leveldb::Slice& a, const leveldb::Slice& b) const
 {
   assert(a.size() > LDB_COMPARE_SKIP_SIZE && b.size() > LDB_COMPARE_SKIP_SIZE);
   const int min_len = (a.size() < b.size()) ? a.size() - LDB_COMPARE_SKIP_SIZE :
     b.size() - LDB_COMPARE_SKIP_SIZE;
   int r = memcmp(a.data() + LDB_COMPARE_SKIP_SIZE, b.data() + LDB_COMPARE_SKIP_SIZE, min_len);
   if (r == 0)
   {
     if (a.size() < b.size())
     {
       r = -1;
     }
     else if (a.size() > b.size())
     {
       r = +1;
     }
   }
   return r;
 }
예제 #30
0
 bool BitcmpLdbComparatorImpl::ShouldStopBefore(const leveldb::Slice& start_key, const leveldb::Slice& key) const
 {
   return LdbKey::decode_bucket_number_with_key(key.data()) !=
     LdbKey::decode_bucket_number_with_key(start_key.data());
 }