unsigned char* CassandraFS::read_block(CassUuid* physical_file_id, int block, int* bytes_read) {
CassStatement* statement = cass_statement_new("SELECT data, size FROM file_blocks WHERE physical_file_id = ? AND block_number = ?", 2);
cass_statement_bind_uuid(statement, 0, *physical_file_id);
cass_statement_bind_int32(statement, 1, block);
unsigned char* return_data = NULL;
CassFuture* result_future = cass_session_execute(ctxt->session, statement);
cass_statement_free(statement);
if (cass_future_error_code(result_future) == CASS_OK) {
const CassResult* result = cass_future_get_result(result_future);
CassIterator* rows = cass_iterator_from_result(result);
if (cass_iterator_next(rows)) {
const CassRow* row = cass_iterator_get_row(rows);
const CassValue* data_value = cass_row_get_column_by_name(row, "data");
const CassValue* size_value = cass_row_get_column_by_name(row, "size");
const cass_byte_t* cass_data;
size_t size;
int size2;
cass_value_get_bytes(data_value, &cass_data, &size);
cass_value_get_int32(size_value, &size2);
// Let's use the value from "size" field unless bigger than the actual data.
if (size2>size) {
warning("Problem: size field is showing value %d which is bigger than the actual block data length %zu", size2, size);
size2 = size;
}
return_data = (unsigned char*)malloc(size2);
memcpy(return_data, cass_data, size2);
(*bytes_read) = size2;
}
cass_result_free(result);
cass_iterator_free(rows);
} else {
/* Handle error */
cassandra_log_error(result_future);
}
cass_future_free(result_future);
return return_data;
}
static void ClearDatabase(Upload& upload) {
DatabaseSession session;
std::string str = "delete from uploadchunks where model_id = ? and upload_id = ?";
CassStatement* statement = cass_statement_new(str.c_str(), 2);
cass_statement_bind_uuid(statement, 0, upload.ModelId());
cass_statement_bind_uuid(statement, 1, upload.UploadId());
CassFuture* future = cass_session_execute(session.Session(), statement);
cass_future_wait(future);
cass_statement_free(statement);
CassError rc = cass_future_error_code(future);
if (rc != CASS_OK) {
std::string err = std::string("Failed to execute query: ")
+ CassandraUtils::GetCassandraError(future);
cass_future_free(future);
throw err;
}
cass_future_free(future);
}
bool execute_insert(CassSession* session, const std::string& table_name) {
std::string query = str(boost::format("INSERT INTO %s (id, event_time, text_sample) VALUES (?, ?, ?)") % table_name);
test_utils::CassStatementPtr statement(cass_statement_new(query.c_str(), 3));
// Determine if bound parameters can be used based on C* version
if (version.major == 1) {
test_utils::CassPreparedPtr prepared = test_utils::prepare(session, query.c_str());
statement = test_utils::CassStatementPtr(cass_prepared_bind(prepared.get()));
}
boost::chrono::system_clock::time_point now(boost::chrono::system_clock::now());
boost::chrono::milliseconds event_time(boost::chrono::duration_cast<boost::chrono::milliseconds>(now.time_since_epoch()));
std::string text_sample(test_utils::string_from_time_point(now));
cass_statement_bind_uuid(statement.get(), 0, test_utils::generate_time_uuid(uuid_gen));
cass_statement_bind_int64(statement.get(), 1, event_time.count());
cass_statement_bind_string(statement.get(), 2, text_sample.c_str());
test_utils::CassFuturePtr future(cass_session_execute(session, statement.get()));
cass_future_wait(future.get());
CassError code = cass_future_error_code(future.get());
if (code != CASS_OK && code != CASS_ERROR_LIB_REQUEST_TIMED_OUT) { // Timeout is okay
CassString message;
cass_future_error_message(future.get(), &message.data, &message.length);
fprintf(stderr, "Error occurred during insert '%.*s'\n", static_cast<int>(message.length), message.data);
return false;
}
return true;
}
void on_create_table(CassFuture* future, void* data) {
const char* insert_query = "INSERT INTO callbacks (key, value) "
"VALUES (?, ?)";
CassUuid key;
CassStatement* statement = NULL;
CassFuture* insert_future = NULL;
CassError code = cass_future_error_code(future);
if (code != CASS_OK) {
print_error(future);
}
statement = cass_statement_new(insert_query, 2);
cass_uuid_gen_time(uuid_gen, &key);
cass_statement_bind_uuid(statement, 0, key);
cass_statement_bind_int64(statement, 1, cass_uuid_timestamp(key));
insert_future = cass_session_execute((CassSession*)data, statement);
cass_future_set_callback(insert_future, on_insert, data);
cass_statement_free(statement);
cass_future_free(insert_future);
}
static std::vector<CassUuid> insert_async(CassSession* session,
CassUuidGen* uuid_gen,
const std::string& table_name,
size_t num_concurrent_requests,
std::vector<test_utils::CassFuturePtr>* futures) {
std::string create_table_query = str(boost::format("CREATE TABLE %s (id timeuuid PRIMARY KEY, num int, str text);") % table_name);
test_utils::execute_query(session, create_table_query);
std::string insert_query = str(boost::format("INSERT INTO %s (id, num, str) VALUES(?, ?, ?)") % table_name);
std::vector<CassUuid> ids;
for (size_t i = 0; i < num_concurrent_requests; ++i) {
CassUuid id = test_utils::generate_time_uuid(uuid_gen);
test_utils::CassStatementPtr statement(cass_statement_new(insert_query.c_str(), 3));
// Determine if bound parameters can be used based on C* version
if (version.major == 1) {
insert_query = str(boost::format("INSERT INTO %s (id, num, str) VALUES(%s, %s, 'row%s')") % table_name % test_utils::Value<CassUuid>::to_string(id) % i % i);
statement = test_utils::CassStatementPtr(cass_statement_new(insert_query.c_str(), 0));
} else {
BOOST_REQUIRE(cass_statement_bind_uuid(statement.get(), 0, id) == CASS_OK);
BOOST_REQUIRE(cass_statement_bind_int32(statement.get(), 1, i) == CASS_OK);
std::string str_value = str(boost::format("row%d") % i);
BOOST_REQUIRE(cass_statement_bind_string(statement.get(), 2, str_value.c_str()) == CASS_OK);
}
cass_statement_set_consistency(statement.get(), CASS_CONSISTENCY_QUORUM);
futures->push_back(test_utils::CassFuturePtr(cass_session_execute(session, statement.get())));
ids.push_back(id);
}
return ids;
}
CassFuture* CassandraFS::delete_file_block(struct stat* stat, struct cfs_attrs* cfs_attrs, int block_number) {
CassStatement* statement = cass_statement_new("DELETE FROM file_blocks WHERE block_number = ? AND physical_file_id = ?", 2);
cass_statement_bind_int32(statement, 0, block_number);
cass_statement_bind_uuid(statement, 1, cfs_attrs->physical_file_id);
CassFuture* result_future = cass_session_execute(ctxt->session, statement);
cass_statement_free(statement);
return result_future;
}
CassFuture* CassandraFS::create_physical_file(CassUuid* uuid) {
CassStatement* statement = cass_statement_new("INSERT INTO physical_files(id, size, block_size) VALUES(?, 0, " DEFAULT_BLOCK_SIZE ")", 1);
cass_statement_bind_uuid(statement, 0, *uuid);
CassFuture* result_future = cass_session_execute(ctxt->session, statement);
cass_statement_free(statement);
return result_future;
}
CassFuture* CassandraFS::update_file_length(CassUuid* physical_file_id, long size) {
CassStatement* statement = cass_statement_new("UPDATE physical_files SET size = ? WHERE id = ?", 2);
cass_statement_bind_int64(statement, 0, size);
cass_statement_bind_uuid(statement, 1, *physical_file_id);
CassFuture* result_future = cass_session_execute(ctxt->session, statement);
cass_statement_free(statement);
return result_future;
}
CassFuture* CassandraFS::truncate_block(struct stat* stat, struct cfs_attrs* cfs_attrs, int block_number, int size) {
CassStatement* statement = cass_statement_new("UPDATE file_blocks SET size = ? WHERE physical_file_id = ? AND block_number = ?", 3);
cass_statement_bind_int32(statement, 0, size);
cass_statement_bind_uuid(statement, 1, cfs_attrs->physical_file_id);
cass_statement_bind_int32(statement, 2, block_number);
CassFuture* future = cass_session_execute(ctxt->session, statement);
cass_statement_free(statement);
return future;
}
void CassandraFS::write_block(CassUuid* physical_file_id, int block, const unsigned char* data, int length, CassandraFutureSpool* spool) {
CassStatement* statement = cass_statement_new("INSERT INTO file_blocks(physical_file_id, block_number, data, size) VALUES(?,?,?,?)", 4);
cass_statement_bind_uuid(statement, 0, *physical_file_id);
cass_statement_bind_int32(statement, 1, block);
cass_statement_bind_bytes(statement, 2, data, length);
cass_statement_bind_int32(statement, 3, length);
CassFuture* result_future = cass_session_execute(ctxt->session, statement);
cass_statement_free(statement);
spool->append(result_future);
}
bool bind_and_execute_insert(CassStatement* statement) {
boost::chrono::system_clock::time_point now(boost::chrono::system_clock::now());
boost::chrono::milliseconds event_time(boost::chrono::duration_cast<boost::chrono::milliseconds>(now.time_since_epoch()));
std::string text_sample(test_utils::string_from_time_point(now));
cass_statement_bind_uuid(statement, 0, test_utils::generate_time_uuid(uuid_gen));
cass_statement_bind_int64(statement, 1, event_time.count());
cass_statement_bind_string_n(statement, 2, text_sample.data(), text_sample.size());
return execute_insert(statement);
}
static void alter_schmea_version(void *arg) {
ClusterInit* cluster_init = static_cast<ClusterInit*>(arg);
test_utils::CassLog::reset("No schema agreement on live nodes after ");
test_utils::CassStatementPtr schema_stmt(cass_prepared_bind(cluster_init->schema_alter_prepared.get()));
boost::chrono::steady_clock::time_point end =
boost::chrono::steady_clock::now() + boost::chrono::milliseconds(MAX_SCHEMA_AGREEMENT_WAIT_MS + 1000);
// mess with system.peers for node 1 more than the wait time; targetting specific nodes
do {
cass_statement_bind_uuid(schema_stmt.get(), 0, test_utils::generate_random_uuid(cluster_init->inst.uuid_gen));
test_utils::CassFuturePtr future(cass_session_execute(cluster_init->schema_alter_session, schema_stmt.get()));
cass_future_wait(future.get());
BOOST_REQUIRE_EQUAL(cass_future_error_code(future.get()), CASS_OK);
} while (boost::chrono::steady_clock::now() < end && test_utils::CassLog::message_count() == 0);
}
void CassandraFS::remove_physical_file(struct stat* stat, struct cfs_attrs* cfs_attrs, CassandraFutureSpool* spool) {
CassStatement* statement = cass_statement_new("DELETE FROM physical_files WHERE id = ?", 1);
cass_statement_bind_uuid(statement, 0, cfs_attrs->physical_file_id);
CassFuture* result_future = cass_session_execute(ctxt->session, statement);
cass_statement_free(statement);
spool->append(result_future);
int blocks = blocks_needed(stat->st_size, cfs_attrs->block_size);
for (int i = 0; i<blocks; i++) {
spool->append(delete_file_block(stat, cfs_attrs, i));
}
}
CassFuture* CassandraFS::create_file_entry(const char* path, CassUuid* uuid, mode_t mode) {
CassStatement* statement = cass_statement_new("INSERT INTO entries(path, mode, created_at, modified_at, physical_file_id) VALUES(?,?,?,?, ?)", 5);
cass_statement_bind_string(statement, 0, path);
cass_statement_bind_int32(statement, 1, mode);
cass_statement_bind_int64(statement, 2, time(NULL)*1000);
cass_statement_bind_int64(statement, 3, time(NULL)*1000);
cass_statement_bind_uuid(statement, 4, *uuid);
CassFuture* result_future = cass_session_execute(ctxt->session, statement);
cass_statement_free(statement);
return result_future;
}
CassError CassandraFS::read_physical_file_info(struct stat* stat, struct cfs_attrs* cfs_attrs) {
CassStatement* statement = cass_statement_new("SELECT size, block_size FROM physical_files WHERE id = ?", 1);
cass_statement_bind_uuid(statement, 0, cfs_attrs->physical_file_id);
CassFuture* result_future = cass_session_execute(ctxt->session, statement);
cass_statement_free(statement);
CassError error = cass_future_error_code(result_future);
if (error == CASS_OK) {
/* Retrieve result set and iterate over the rows */
const CassResult* result = cass_future_get_result(result_future);
CassIterator* rows = cass_iterator_from_result(result);
if (cass_iterator_next(rows)) {
const CassRow* row = cass_iterator_get_row(rows);
const CassValue* size_value = cass_row_get_column_by_name(row, "size");
if (!cass_value_is_null(size_value)) {
cass_int64_t size;
cass_value_get_int64(size_value, &size);
stat->st_size = size;
}
const CassValue* block_size_value = cass_row_get_column_by_name(row, "block_size");
if (!cass_value_is_null(block_size_value)) {
int size;
cass_value_get_int32(block_size_value, &size);
cfs_attrs->block_size = size;
}
}
cass_result_free(result);
cass_iterator_free(rows);
} else {
/* Handle error */
cassandra_log_error(result_future);
}
cass_future_free(result_future);
return error;
}
void insert_into_perf(CassSession* session, const char* query, const CassPrepared* prepared) {
int i;
CassFuture* futures[NUM_CONCURRENT_REQUESTS];
CassCollection* collection = cass_collection_new(CASS_COLLECTION_TYPE_SET, 2);
cass_collection_append_string(collection, "jazz");
cass_collection_append_string(collection, "2013");
for (i = 0; i < NUM_CONCURRENT_REQUESTS; ++i) {
CassUuid id;
CassStatement* statement;
if (prepared != NULL) {
statement = cass_prepared_bind(prepared);
} else {
statement = cass_statement_new(query, 5);
}
cass_uuid_gen_time(uuid_gen, &id);
cass_statement_bind_uuid(statement, 0, id);
cass_statement_bind_string(statement, 1, big_string);
cass_statement_bind_string(statement, 2, big_string);
cass_statement_bind_string(statement, 3, big_string);
cass_statement_bind_collection(statement, 4, collection);
futures[i] = cass_session_execute(session, statement);
cass_statement_free(statement);
}
for (i = 0; i < NUM_CONCURRENT_REQUESTS; ++i) {
CassFuture* future = futures[i];
CassError rc = cass_future_error_code(future);
if (rc != CASS_OK) {
print_error(future);
}
cass_future_free(future);
}
cass_collection_free(collection);
}
int CassandraFS::truncate(const char* path, off_t size) {
CassStatement* statement = NULL;
struct stat stat;
struct cfs_attrs cfs_attrs;
CassandraFutureSpool* spool = new CassandraFutureSpool(4);
int err = getattr(path, &stat, &cfs_attrs);
if (err != 0) {
return -EIO;
}
statement = cass_statement_new("UPDATE physical_files SET size = ? WHERE id = ?", 2);
cass_statement_bind_int64(statement, 0, size);
cass_statement_bind_uuid(statement, 1, cfs_attrs.physical_file_id);
spool->append(cass_session_execute(ctxt->session, statement));
cass_statement_free(statement);
off_t current_size = stat.st_size;
int current_blocks = blocks_needed(current_size, cfs_attrs.block_size);
int needed_blocks = blocks_needed(size, cfs_attrs.block_size);
for (int a = needed_blocks; a<current_blocks; a++) {
spool->append(delete_file_block(&stat, &cfs_attrs, a));
}
if (size % cfs_attrs.block_size > 0) {
spool->append(truncate_block(&stat, &cfs_attrs, needed_blocks-1, size % cfs_attrs.block_size));
}
spool->wait_all();
int errors = spool->get_errors();
delete spool;
if (errors>0) {
return -EIO;
}
return 0;
}
int CassandraFS::hardlink(const char* from, const char* to) {
int operation_done = 0;
CassStatement* statement = cass_statement_new("SELECT mode, created_at, modified_at, physical_file_id FROM entries WHERE path = ?", 1);
cass_statement_bind_string(statement, 0, from);
CassFuture* result_future = cass_session_execute(ctxt->session, statement);
cass_statement_free(statement);
int error = 0;
if (cass_future_error_code(result_future) == CASS_OK) {
/* Retrieve result set and iterate over the rows */
const CassResult* result = cass_future_get_result(result_future);
CassIterator* rows = cass_iterator_from_result(result);
if (cass_iterator_next(rows)) {
const CassRow* row = cass_iterator_get_row(rows);
const CassValue* mode_value = cass_row_get_column_by_name(row, "mode");
const CassValue* modified_at_value = cass_row_get_column_by_name(row, "modified_at");
const CassValue* created_at_value = cass_row_get_column_by_name(row, "created_at");
const CassValue* physical_file_id_value = cass_row_get_column_by_name(row, "physical_file_id");
int mode;
cass_value_get_int32(mode_value, &mode);
cass_int64_t modified_at;
cass_value_get_int64(modified_at_value, &modified_at);
cass_int64_t created_at;
cass_value_get_int64(created_at_value, &created_at);
CassUuid physical_file_id;
cass_value_get_uuid(physical_file_id_value, &physical_file_id);
CassStatement* insert_statement = cass_statement_new("INSERT INTO entries(path, mode, created_at, modified_at, physical_file_id) VALUES(?,?,?,?,?)", 5);
cass_statement_bind_string(insert_statement, 0, to);
cass_statement_bind_int32(insert_statement, 1, mode);
cass_statement_bind_int64(insert_statement, 2, created_at);
cass_statement_bind_int64(insert_statement, 3, modified_at);
cass_statement_bind_uuid(insert_statement, 4, physical_file_id);
CassFuture* insert_future2 = create_sub_entry(to);
CassFuture* insert_future = cass_session_execute(ctxt->session, insert_statement);
cass_statement_free(insert_statement);
if (cass_future_error_code(insert_future) == CASS_OK) {
operation_done = 1;
} else {
operation_done = 0;
cassandra_log_error(insert_future);
}
if (cass_future_error_code(insert_future2) == CASS_OK) {
operation_done = 1;
} else {
operation_done = 0;
cassandra_log_error(insert_future2);
}
cass_future_free(insert_future);
cass_future_free(insert_future2);
}
cass_result_free(result);
cass_iterator_free(rows);
} else {
/* Handle error */
error = -EIO;
cassandra_log_error(result_future);
}
cass_future_free(result_future);
if (!operation_done) {
return -ENOENT;
}
return error;
}
static std::string LoadFile(Upload& upload)
{
std::stringstream ss;
DatabaseSession session;
try {
CassError rc = CASS_OK;
//std::string str = ("SELECT upload_id, model_id, date, processed, result {includeFile} FROM prophet.models {onlyPending}");
std::string str = "SELECT chunk from prophet.uploadchunks where model_id = ? and upload_id = ?";
CassStatement* statement = cass_statement_new(str.c_str(), 2);
cass_statement_bind_uuid(statement, 0, upload.ModelId());
cass_statement_bind_uuid(statement, 1, upload.UploadId());
CassFuture * future = cass_session_execute(session.Session(), statement);
cass_future_wait(future);
rc = cass_future_error_code(future);
if (rc != CASS_OK) {
auto err = std::string("Failed to execute query: ")
+ CassandraUtils::GetCassandraError(future);
cass_future_free(future);
cass_statement_free(statement);
throw err;
}
else {
const CassResult* result = cass_future_get_result(future);
CassIterator* iterator = cass_iterator_from_result(result);
while (cass_iterator_next(iterator)) {
const CassRow* row = cass_iterator_get_row(iterator);
std::string buffer;
CassString::Load(buffer, cass_row_get_column(row, 0));
ss << buffer;
}
cass_result_free(result);
cass_iterator_free(iterator);
}
cass_future_free(future);
cass_statement_free(statement);
}
catch (...) {
std::exception_ptr e = std::current_exception();
try {
if (e) {
std::rethrow_exception(e);
}
}
catch (const std::exception& ex) {
std::cout << "Caught exception \"" << ex.what() << "\"\n";
}
}
return ss.str();
}
static CassError bind(CassStatement* statement, size_t index, CassUuid value) {
return cass_statement_bind_uuid(statement, index, value);
}
CassError insert_into_udt(CassSession* session) {
CassError rc = CASS_OK;
CassStatement* statement = NULL;
CassFuture* future = NULL;
CassUuid id;
char id_str[CASS_UUID_STRING_LENGTH];
const CassKeyspaceMeta* keyspace_meta = NULL;
const CassDataType* udt_address = NULL;
const CassDataType* udt_phone = NULL;
const char* query = "INSERT INTO examples.udt (id, address) VALUES (?, ?)";
statement = cass_statement_new(query, 2);
cass_uuid_gen_time(uuid_gen, &id);
cass_uuid_string(id, id_str);
keyspace_meta = cass_schema_meta_keyspace_by_name(schema_meta, "examples");
if (keyspace_meta != NULL) {
udt_address = cass_keyspace_meta_user_type_by_name(keyspace_meta, "address");
udt_phone = cass_keyspace_meta_user_type_by_name(keyspace_meta, "phone_numbers");
}
if (udt_address != NULL && udt_phone != NULL) {
int i;
CassUserType* address = cass_user_type_new_from_data_type(udt_address);
CassCollection* phone = cass_collection_new(CASS_COLLECTION_TYPE_SET, 2);
for (i = 0; i < 2; ++i) {
CassUserType* phone_numbers = cass_user_type_new_from_data_type(udt_phone);
cass_user_type_set_int32_by_name(phone_numbers, "phone1", i + 1);
cass_user_type_set_int32_by_name(phone_numbers, "phone2", i + 2);
cass_collection_append_user_type(phone, phone_numbers);
cass_user_type_free(phone_numbers);
}
cass_user_type_set_string_by_name(address, "street", id_str);
cass_user_type_set_string_by_name(address, "city", id_str);
cass_user_type_set_int32_by_name(address, "zip", (cass_int32_t)id.time_and_version);
cass_user_type_set_collection_by_name(address, "phone", phone);
cass_statement_bind_uuid(statement, 0, id);
cass_statement_bind_user_type(statement, 1, address);
future = cass_session_execute(session, statement);
cass_future_wait(future);
rc = cass_future_error_code(future);
if (rc != CASS_OK) {
print_error(future);
}
cass_future_free(future);
cass_user_type_free(address);
cass_collection_free(phone);
}
cass_statement_free(statement);
return rc;
}
