int append_cg_rows(
ObSSTableWriter& writer, const ObCellInfo** cell_infos,
const int64_t column_group_id, const int64_t row_num,
const int64_t start_col, const int64_t end_col)
{
int err = 0;
uint64_t table_id = cell_infos[0][0].table_id_;
for (int64_t i = 0; i < row_num; ++i)
{
ObSSTableRow row;
err = row.set_rowkey(cell_infos[i][0].row_key_);
EXPECT_EQ(0, err);
row.set_table_id(table_id);
row.set_column_group_id(column_group_id);
for (int64_t j = start_col; j < end_col; ++j)
{
err = row.add_obj(cell_infos[i][j].value_);
EXPECT_EQ(0, err);
}
int64_t space_usage = 0;
err = writer.append_row(row, space_usage);
EXPECT_EQ(0, err);
}
return err;
}
int write_cg_sstable(const ObCellInfo** cell_infos,
const int64_t row_num, const int64_t col_num, const char* sstable_file_path )
{
int err = OB_SUCCESS;
EXPECT_TRUE(NULL != cell_infos);
EXPECT_TRUE(row_num > 0);
EXPECT_TRUE(col_num > 0);
if (sstable_file_path) unlink(sstable_file_path);
ObSSTableSchema sstable_schema;
generate_cg_schema(sstable_schema, cell_infos, 12, 0, 1);
generate_cg_schema(sstable_schema, cell_infos, 13, 1, 3);
generate_cg_schema(sstable_schema, cell_infos, 15, 3, col_num);
ObString path;
ObString compress_name;
const char* path_str = NULL;
if (NULL != sstable_file_path)
{
path_str = sstable_file_path;
}
else
{
path_str = DEF_SSTABLE_PATH;
}
path.assign((char*) path_str, static_cast<int32_t>(strlen(path_str)));
compress_name.assign((char*)"lzo", static_cast<int32_t>(strlen("lzo")));
ObSSTableWriter writer;
uint64_t table_id = cell_infos[0][0].table_id_;
err = writer.create_sstable(sstable_schema, path, compress_name, table_id);
EXPECT_EQ(0, err);
err = append_cg_rows(writer, cell_infos, 12, row_num, 0, 1);
EXPECT_EQ(0, err);
err = append_cg_rows(writer, cell_infos, 13, row_num, 1, 3);
EXPECT_EQ(0, err);
err = append_cg_rows(writer, cell_infos, 15, row_num, 3, col_num);
EXPECT_EQ(0, err);
int64_t offset = 0;
err = writer.close_sstable(offset);
EXPECT_EQ(0, err);
return err;
}
static int init_sstable(ObSSTableReader& sstable, const ObCellInfo** cell_infos,
const int64_t row_num, const int64_t col_num, const int64_t sst_id = 0L)
{
int err = OB_SUCCESS;
ObSSTableSchema sstable_schema;
ObSSTableSchemaColumnDef column_def;
EXPECT_TRUE(NULL != cell_infos);
EXPECT_TRUE(row_num > 0);
EXPECT_TRUE(col_num > 0);
uint64_t table_id = cell_infos[0][0].table_id_;
ObString path;
int64_t sstable_file_id = 0;
ObString compress_name;
char* path_str = sstable_file_path;
int64_t path_len = OB_MAX_FILE_NAME_LENGTH;
for (int64_t i = 0; i < col_num; ++i)
{
column_def.reserved_ = 0;
if (i >=2)
{
column_def.column_group_id_= 2;
}
else
{
column_def.column_group_id_= 0;
}
column_def.column_name_id_ = cell_infos[0][i].column_id_;
column_def.column_value_type_ = cell_infos[0][i].value_.get_type();
column_def.table_id_ = table_id;
sstable_schema.add_column_def(column_def);
}
if (0 == sst_id)
{
sstable_file_id = 100;
}
else
{
sstable_file_id = sst_id;
}
ObSSTableId sstable_id(sst_id);
get_sstable_path(sstable_id, path_str, path_len);
char cmd[256];
sprintf(cmd, "mkdir -p %s", path_str);
system(cmd);
path.assign((char*)path_str, strlen(path_str));
compress_name.assign("lzo_1.0", strlen("lzo_1.0"));
remove(path.ptr());
ObSSTableWriter writer;
err = writer.create_sstable(sstable_schema, path, compress_name, 0);
EXPECT_EQ(OB_SUCCESS, err);
for (int64_t i = 0; i < row_num; ++i)
{
ObSSTableRow row;
row.set_table_id(table_id);
row.set_column_group_id(0);
err = row.set_row_key(cell_infos[i][0].row_key_);
EXPECT_EQ(OB_SUCCESS, err);
for (int64_t j = 0; j < 2; ++j)
{
err = row.add_obj(cell_infos[i][j].value_);
EXPECT_EQ(OB_SUCCESS, err);
}
int64_t space_usage = 0;
err = writer.append_row(row, space_usage);
EXPECT_EQ(OB_SUCCESS, err);
}
for (int64_t i = 0; i < row_num; ++i)
{
ObSSTableRow row;
row.set_table_id(table_id);
row.set_column_group_id(2);
err = row.set_row_key(cell_infos[i][0].row_key_);
EXPECT_EQ(OB_SUCCESS, err);
for (int64_t j = 2; j < col_num; ++j)
{
err = row.add_obj(cell_infos[i][j].value_);
EXPECT_EQ(OB_SUCCESS, err);
}
int64_t space_usage = 0;
err = writer.append_row(row, space_usage);
EXPECT_EQ(OB_SUCCESS, err);
}
int64_t offset = 0;
err = writer.close_sstable(offset);
EXPECT_EQ(OB_SUCCESS, err);
err = fic.init(1024);
EXPECT_EQ(OB_SUCCESS, err);
err = reader_.open(sstable_id);
EXPECT_EQ(OB_SUCCESS, err);
EXPECT_TRUE(reader_.is_opened());
return err;
}
int write_empty_sstable(const ObCellInfo** cell_infos,
const int64_t row_num, const int64_t col_num, const char* sstable_file_path )
{
int err = OB_SUCCESS;
if (sstable_file_path) unlink(sstable_file_path);
ObSSTableSchema sstable_schema;
ObSSTableSchemaColumnDef column_def;
EXPECT_TRUE(NULL != cell_infos);
EXPECT_TRUE(row_num > 0);
EXPECT_TRUE(col_num > 0);
uint64_t table_id = cell_infos[0][0].table_id_;
// add rowkey column
column_def.reserved_ = 0;
column_def.rowkey_seq_ = 1;
column_def.column_group_id_= 0;
column_def.column_name_id_ = 1;
column_def.column_value_type_ = ObVarcharType;
column_def.table_id_ = static_cast<uint32_t>(table_id);
sstable_schema.add_column_def(column_def);
for (int64_t i = 0; i < col_num; ++i)
{
column_def.reserved_ = 0;
column_def.rowkey_seq_ = 0;
column_def.table_id_ = static_cast<uint32_t>(table_id);
column_def.column_group_id_ = 0;
column_def.column_name_id_ = static_cast<uint16_t>(cell_infos[0][i].column_id_);
column_def.column_value_type_ = cell_infos[0][i].value_.get_type();
sstable_schema.add_column_def(column_def);
}
ObString path;
ObString compress_name;
const char* path_str = NULL;
if (NULL != sstable_file_path)
{
path_str = sstable_file_path;
}
else
{
path_str = DEF_SSTABLE_PATH;
}
path.assign((char*) path_str, static_cast<int32_t>(strlen(path_str)));
compress_name.assign((char*)"lzo_1.0", static_cast<int32_t>(strlen("lzo_1.0")));
ObSSTableWriter writer;
err = writer.create_sstable(sstable_schema, path, compress_name, table_id);
EXPECT_EQ(0, err);
int64_t offset = 0;
err = writer.close_sstable(offset);
EXPECT_EQ(0, err);
return err;
}
void test_write_one_file()
{
ObSSTableSchemaColumnDef column_def;
ObSSTableSchema schema;
ObSSTableRow row;
char *compressor_name = (char*)COMPRESSOR_NAME;
ObSSTableWriter writer;
ObString file_name(static_cast<int32_t>(strlen(sstable_path) + 1),
static_cast<int32_t>(strlen(sstable_path) + 1), sstable_path);
ObString compressor(static_cast<int32_t>(strlen(compressor_name) + 1),
static_cast<int32_t>(strlen(compressor_name) + 1), compressor_name);
int64_t disk_usage = 0;
int64_t trailer_offset = 0;
ObObj obj;
ObObj key_obj;
uint64_t table_id = 0;
uint64_t column_group_id = 0;
char value_data[1024 + 1];
char rowkey_str[32];
ObString row_key;
ObRowkey key;
char *ptr;
int ret;
// init schema
for (int64_t i = 0; i < 5; ++i)
{
column_def.table_id_ = static_cast<uint32_t>(1025 + i);
// add rowkey column
column_def.column_group_id_ = 0;
column_def.column_name_id_ = 1;
column_def.column_value_type_ = ObVarcharType;
column_def.rowkey_seq_ = 1;
schema.add_column_def(column_def);
for ( int j = 0; j < 10 ; ++j)
{
column_def.column_group_id_ = static_cast<uint16_t>(j);
column_def.column_name_id_ = 2;
column_def.column_value_type_ = ObDoubleType;
column_def.rowkey_seq_ = 0;
schema.add_column_def(column_def);
column_def.column_name_id_ = 3;
column_def.column_value_type_ = ObIntType;
column_def.rowkey_seq_ = 0;
schema.add_column_def(column_def);
column_def.column_name_id_ = 4;
column_def.column_value_type_ = ObVarcharType;
column_def.rowkey_seq_ = 0;
schema.add_column_def(column_def);
}
}
//build data
ptr = value_data;
for (int64_t i = 0; i < 128; ++i) {
memcpy(ptr, "testing ", 8);
ptr += 8;
}
ObString value_str(1025, 1025, value_data);
writer.set_dio(true);
int64_t start_time = tbsys::CTimeUtil::getTime();
// create sstable file
if (OB_ERROR == (ret = writer.create_sstable(schema, file_name,
compressor, 2)))
{
TBSYS_LOG(ERROR, "Failed to create sstable file: %s", sstable_path);
}
for (int64_t i = 0; i < 5000000; ++i)
{
row.clear();
table_id = i / 1000000 + 1025;
column_group_id = i % 1000000 / 100000;
row.set_table_id(table_id);
row.set_column_group_id(column_group_id);
sprintf(rowkey_str, "row_key_%08ld", i);
row_key.assign(rowkey_str, 16);
key_obj.set_varchar(row_key);
key.assign(&key_obj, 1);
row.set_rowkey(key);
obj.set_double((double)i);
row.add_obj(obj);
obj.set_int(i);
row.add_obj(obj);
obj.set_varchar(value_str);
row.add_obj(obj);
if (OB_ERROR == (ret = writer.append_row(row, disk_usage)))
{
TBSYS_LOG(ERROR, "add row failed, i=%ld", i);
return;
}
}
if (OB_ERROR == (ret = writer.close_sstable(trailer_offset)))
{
TBSYS_LOG(ERROR, "close sstable failed ------------------");
}
printf("test_write_one_file, run_time=%ld\n",
tbsys::CTimeUtil::getTime() - start_time);
//remove(sstable_path);
}
int write_sstable(const ObCellInfo** cell_infos,
const int64_t row_num, const int64_t col_num, const char* sstable_file_path )
{
int err = OB_SUCCESS;
if (sstable_file_path) unlink(sstable_file_path);
ObSSTableSchema sstable_schema;
ObSSTableSchemaColumnDef column_def;
EXPECT_TRUE(NULL != cell_infos);
EXPECT_TRUE(row_num > 0);
EXPECT_TRUE(col_num > 0);
uint64_t table_id = cell_infos[0][0].table_id_;
for (int64_t i = 0; i < col_num; ++i)
{
column_def.table_id_ = table_id;
column_def.column_group_id_ = 0;
column_def.column_name_id_ = cell_infos[0][i].column_id_;
column_def.column_value_type_ = cell_infos[0][i].value_.get_type();
sstable_schema.add_column_def(column_def);
}
ObString path;
ObString compress_name;
const char* path_str = NULL;
if (NULL != sstable_file_path)
{
path_str = sstable_file_path;
}
else
{
path_str = DEF_SSTABLE_PATH;
}
path.assign((char*) path_str, strlen(path_str));
compress_name.assign("lzo_1.0", strlen("lzo_1.0"));
ObSSTableWriter writer;
err = writer.create_sstable(sstable_schema, path, compress_name, table_id);
EXPECT_EQ(0, err);
for (int64_t i = 0; i < row_num; ++i)
{
ObSSTableRow row;
err = row.set_row_key(cell_infos[i][0].row_key_);
EXPECT_EQ(0, err);
row.set_table_id(table_id);
row.set_column_group_id(0);
for (int64_t j = 0; j < col_num; ++j)
{
err = row.add_obj(cell_infos[i][j].value_);
EXPECT_EQ(0, err);
}
int64_t space_usage = 0;
err = writer.append_row(row, space_usage);
EXPECT_EQ(0, err);
}
int64_t offset = 0;
err = writer.close_sstable(offset);
EXPECT_EQ(0, err);
return err;
}
