static uint8_t manage_block_overflow(cir_storage_flash_t *storage, uint32_t size, pointer_type pointer_type)
{
uint32_t pointer;
if (pointer_type == POINTER_READ) {
pointer = READ_PTR(storage);
} else {
pointer = WRITE_PTR(storage);
}
/* If there is not enough place on the current block, we read at the beginning
of the next block */
if (storage->block_size - (pointer%storage->block_size) < size) {
pointer = storage->block_size*(pointer/storage->block_size+1);
/* If we are at the end of the circular storage, we move the read pointer
on the first data block */
if (pointer + size > (storage->address_start + storage->parent.size)) {
pointer = storage->address_start+storage->block_size;
}
if (pointer_type == POINTER_READ) {
READ_PTR(storage) = pointer;
} else {
WRITE_PTR(storage) = pointer;
}
return 1;
}
return 0;
}
bool LshKBM::save(FILE *f, unsigned char ver) const
{
unsigned int signature = 0xFAADBEEB;
WRITE_VAR(signature);
ver = mVer; // added mJfinal and mDtKBinMeans
WRITE_VAR(ver);
WRITE_VAR(type);
WRITE_VAR(dims);
WRITE_VAR(num_descriptors);
WRITE_VAR(num_images);
unsigned int word_count = wordCount(); // how many centroids?
WRITE_VAR(word_count);
WRITE_VAR(mJfinal);
WRITE_VAR(mDtKBinMeans);
WRITE_VAR(m_guid);
uint32_t m=mLshKbmP.getM(),l=mLshKbmP.getL();
WRITE_VAR(m);
WRITE_VAR(l);
cout<<"sig="<<signature<<" ver="<<ver<<" type="<<type<<" dims="<<dims<<" #descriptors="<<num_descriptors<<" #images="<<num_images<<" #words="<<word_count<<endl;
cout<<" mJfinal="<<mJfinal<<" mDtKBinMeans="<<mDtKBinMeans<<endl;
// write centroids to file - decide which one to save!
if(mCentroids.size()>0){
for(uint32_t i=0; i<mCentroids.size(); ++i)
WRITE_PTR(&(mCentroids[i]->bd[0]), dims*sizeof(mCentroids[i]->bd[0]));
}else{
Log::error("Could not determine which centroids to save to file!");
return false;
}
//TODO: maybe save LSH buckets here? try without first
// LSH table building should be quite fast.
return true;
}
/**
* Writes a 128 bit decimal to the byte buffer.
*/
VALUE rb_bson_byte_buffer_put_decimal128(VALUE self, VALUE low, VALUE high)
{
byte_buffer_t *b;
const int64_t low64 = BSON_UINT64_TO_LE(NUM2ULL(low));
const int64_t high64 = BSON_UINT64_TO_LE(NUM2ULL(high));
TypedData_Get_Struct(self, byte_buffer_t, &rb_byte_buffer_data_type, b);
ENSURE_BSON_WRITE(b, 8);
memcpy(WRITE_PTR(b), &low64, 8);
b->write_position += 8;
ENSURE_BSON_WRITE(b, 8);
memcpy(WRITE_PTR(b), &high64, 8);
b->write_position += 8;
return self;
}
/**
* Writes a 64 bit double to the buffer.
*/
VALUE rb_bson_byte_buffer_put_double(VALUE self, VALUE f)
{
byte_buffer_t *b;
const double d = BSON_DOUBLE_TO_LE(NUM2DBL(f));
TypedData_Get_Struct(self, byte_buffer_t, &rb_byte_buffer_data_type, b);
ENSURE_BSON_WRITE(b, 8);
memcpy(WRITE_PTR(b), (char*)&d, 8);
b->write_position += 8;
return self;
}
/**
* Writes a 64 bit integer to the byte buffer.
*/
VALUE rb_bson_byte_buffer_put_int64(VALUE self, VALUE i)
{
byte_buffer_t *b;
const int64_t i64 = BSON_UINT64_TO_LE(NUM2LL(i));
TypedData_Get_Struct(self, byte_buffer_t, &rb_byte_buffer_data_type, b);
ENSURE_BSON_WRITE(b, 8);
memcpy(WRITE_PTR(b), (char*)&i64, 8);
b->write_position += 8;
return self;
}
/**
* Writes a 32 bit integer to the byte buffer.
*/
VALUE rb_bson_byte_buffer_put_int32(VALUE self, VALUE i)
{
byte_buffer_t *b;
const int32_t i32 = BSON_UINT32_TO_LE(NUM2INT(i));
TypedData_Get_Struct(self, byte_buffer_t, &rb_byte_buffer_data_type, b);
ENSURE_BSON_WRITE(b, 4);
memcpy(WRITE_PTR(b), (char*)&i32, 4);
b->write_position += 4;
return self;
}
/**
* Writes bytes to the byte buffer.
*/
VALUE rb_bson_byte_buffer_put_bytes(VALUE self, VALUE bytes)
{
byte_buffer_t *b;
const char *str = RSTRING_PTR(bytes);
const size_t length = RSTRING_LEN(bytes);
TypedData_Get_Struct(self, byte_buffer_t, &rb_byte_buffer_data_type, b);
ENSURE_BSON_WRITE(b, length);
memcpy(WRITE_PTR(b), str, length);
b->write_position += length;
return self;
}
/**
* Writes a byte to the byte buffer.
*/
VALUE rb_bson_byte_buffer_put_byte(VALUE self, VALUE byte)
{
byte_buffer_t *b;
const char *str = RSTRING_PTR(byte);
TypedData_Get_Struct(self, byte_buffer_t, &rb_byte_buffer_data_type, b);
ENSURE_BSON_WRITE(b, 1);
memcpy(WRITE_PTR(b), str, 1);
b->write_position += 1;
return self;
}
/**
* Writes a string to the byte buffer.
*/
VALUE rb_bson_byte_buffer_put_string(VALUE self, VALUE string)
{
byte_buffer_t *b;
int32_t length_le;
char *str = RSTRING_PTR(string);
const int32_t length = RSTRING_LEN(string) + 1;
length_le = BSON_UINT32_TO_LE(length);
if (!rb_bson_utf8_validate(str, length - 1, true)) {
rb_raise(rb_eArgError, "String %s is not valid UTF-8.", str);
}
TypedData_Get_Struct(self, byte_buffer_t, &rb_byte_buffer_data_type, b);
ENSURE_BSON_WRITE(b, length + 4);
memcpy(WRITE_PTR(b), (char*)&length_le, 4);
b->write_position += 4;
memcpy(WRITE_PTR(b), str, length);
b->write_position += length;
return self;
}
cir_storage_err_t cir_storage_push(cir_storage_t *self,
uint8_t *buf,
uint32_t len)
{
cir_storage_flash_t *storage = (cir_storage_flash_t *)self;
if ( (len > storage->block_size) || (len <= 0) || (buf == NULL) ) {
return CBUFFER_STORAGE_ERROR;
}
if (manage_block_overflow(storage, len, POINTER_WRITE)) {
if (storage->erase(WRITE_PTR(storage)/storage->block_size, 1) !=0) {
return CBUFFER_STORAGE_ERROR;
}
/* If read pointer is in the erased block, we move it on the next one */
if ((WRITE_PTR(storage) / storage->block_size) ==
(READ_PTR(storage) / storage->block_size)) {
READ_PTR(storage) = storage->address_start+storage->block_size+
(WRITE_PTR(storage) - storage->address_start)%
(storage->parent.size - storage->block_size);
}
}
if (storage->write(WRITE_PTR(storage), len, buf) != 0) {
return CBUFFER_STORAGE_ERROR;
}
/* We increase data pointer write and copy it in pointers block */
WRITE_PTR(storage) += len;
if (set_pointers(storage, &storage->pointers) != 0) {
return CBUFFER_STORAGE_ERROR;
}
return CBUFFER_STORAGE_SUCCESS;
}
/**
* Writes a cstring to the byte buffer.
*/
VALUE rb_bson_byte_buffer_put_cstring(VALUE self, VALUE string)
{
byte_buffer_t *b;
char *c_str = RSTRING_PTR(string);
size_t length = RSTRING_LEN(string) + 1;
if (!rb_bson_utf8_validate(c_str, length - 1, false)) {
rb_raise(rb_eArgError, "String %s is not a valid UTF-8 CString.", c_str);
}
TypedData_Get_Struct(self, byte_buffer_t, &rb_byte_buffer_data_type, b);
ENSURE_BSON_WRITE(b, length);
memcpy(WRITE_PTR(b), c_str, length);
b->write_position += length;
return self;
}
cir_storage_err_t cir_storage_clear(cir_storage_t * self, uint32_t size)
{
cir_storage_flash_t *storage = (cir_storage_flash_t *)self;
if (size == 0) {
/* Put the read pointer at same location as the write pointer */
READ_PTR(storage) = WRITE_PTR(storage);
} else {
/* Move the read pointer of "size" */
READ_PTR(storage) += size;
manage_block_overflow(storage, size, POINTER_READ);
}
if (set_pointers(storage, &storage->pointers) == 0)
return CBUFFER_STORAGE_SUCCESS;
else
return CBUFFER_STORAGE_ERROR;
}
cir_storage_err_t cir_storage_pop(cir_storage_t *self,
uint8_t *buf,
uint32_t buf_size)
{
cir_storage_flash_t *storage = (cir_storage_flash_t *)self;
if ( (buf_size > storage->block_size) || (buf_size <= 0) || (buf == NULL) ) {
return CBUFFER_STORAGE_ERROR;
}
/* nothing to read */
if (READ_PTR(storage) == WRITE_PTR(storage)) {
return CBUFFER_STORAGE_EMPTY_ERROR;
}
if (storage->read(READ_PTR(storage), buf_size, buf) != 0) {
return CBUFFER_STORAGE_ERROR;
}
/* We increase data pointer read and copy it in pointers block */
return cir_storage_clear(self, buf_size);
}
int32_t cir_storage_flash_init(cir_storage_flash_t *storage)
{
pointers_block_t local_pointers;
if (storage->parent.size%storage->block_size != 0) {
return -1;
}
if (storage->read(storage->address_start,
sizeof(pointers_block_t),
(uint8_t *)&local_pointers) != 0) {
return -1;
}
/* If an init as already been done */
if (local_pointers.data_pointer_read != 0xFFFFFFFF) {
/* Initialize pointers with previous value */
if (get_pointers(storage, &storage->pointers) != 0){
return -1;
}
return 0;
}
/* Storage first init */
READ_PTR(storage) = storage->address_start + storage->block_size;
WRITE_PTR(storage) = READ_PTR(storage);
storage->pointers_index = storage->address_start;
/* Erase the two firsts blocks (pointers block and first data block) */
if (storage->erase(storage->block_start, 2) != 0) {
return -1;
}
/* Store our pointers at the beginning of the first block */
return storage->write(storage->address_start,
sizeof(pointers_block_t),
(uint8_t *)&storage->pointers);
}
cir_storage_err_t cir_storage_peek(cir_storage_t * self,
uint32_t offset,
uint8_t *buf,
uint32_t size)
{
int32_t ret;
cir_storage_flash_t *storage = (cir_storage_flash_t *)self;
if ( (size > storage->block_size) || (size <= 0) || (buf == NULL) ) {
return CBUFFER_STORAGE_ERROR;
}
/* nothing to read */
if (READ_PTR(storage) == WRITE_PTR(storage)) {
return CBUFFER_STORAGE_EMPTY_ERROR;
}
ret = storage->read(READ_PTR(storage) + offset, size, buf);
if (ret == 0) {
return CBUFFER_STORAGE_SUCCESS;
} else {
return CBUFFER_STORAGE_ERROR;
}
}
