int gen_sst(const gen_param ¶m)
{
ObSSTableSchema *sst_schema = new(std::nothrow)ObSSTableSchema;
ObSSTableWriter sst_writer;
ObSSTableRow sst_row;
tbsys::CConfig config;
ObSchemaManagerV2 *mgr = new(std::nothrow) ObSchemaManagerV2;
int err = OB_SUCCESS;
if ((NULL == mgr) || (NULL == sst_schema))
{
err = OB_ALLOCATE_MEMORY_FAILED;
TBSYS_LOG(WARN,"fail to allocate memory for ObSchemaManagerV2");
}
if ((OB_SUCCESS == err) && (!mgr->parse_from_file(param.schema_,config)))
{
TBSYS_LOG(WARN,"fail to load schema from file [err:%d,schema.ini:%s]", err, param.schema_);
err = OB_INVALID_ARGUMENT;
}
if ((OB_SUCCESS == err) && (OB_SUCCESS != (err = fill_sstable_schema(*mgr,msolap::target_table_id,*sst_schema))))
{
TBSYS_LOG(WARN,"fail to fill sst schema [err:%d, table_id:%lu]", err, msolap::target_table_id);
}
ObTrailerParam trailer_param;
trailer_param.block_size_ = msolap::ups_sst_block_size;
trailer_param.compressor_name_.assign((char*)msolap::ups_sst_compressor_name, static_cast<int32_t>(strlen(msolap::ups_sst_compressor_name)));
trailer_param.frozen_time_ = tbsys::CTimeUtil::getTime();
trailer_param.store_type_ = msolap::ups_sst_store_type;
trailer_param.table_version_ = msolap::ups_sst_version;
ObString sst_path(static_cast<int32_t>(strlen(param.sst_name_)), static_cast<int32_t>(strlen(param.sst_name_)),(char*)param.sst_name_);
if ((OB_SUCCESS == err) && (OB_SUCCESS != sst_writer.create_sstable(*sst_schema,sst_path, trailer_param)))
{
TBSYS_LOG(WARN,"fail to create sst [err:%d,sst_path:%s]", err, param.sst_name_);
}
uint32_t big_endian_row_key = 0;
int64_t space_usage = 0;
ObString row_key_str(sizeof(big_endian_row_key), sizeof(big_endian_row_key), (char*)&big_endian_row_key);
if((OB_SUCCESS == err) && (OB_SUCCESS != (err = sst_row.set_obj_count(msolap::max_column_id - msolap::min_column_id + 1))))
{
TBSYS_LOG(WARN,"fail to call sst_row.set_obj_count [err:%d]", err);
}
for (uint32_t row_key = param.start_include_; (OB_SUCCESS == err) && (row_key <= param.end_include_); row_key++)
{
big_endian_row_key = htonl(row_key);
sst_row.clear();
if ((OB_SUCCESS == err) &&(OB_SUCCESS != (err = sst_row.set_rowkey(TestRowkeyHelper(row_key_str, &allocator_)))))
{
TBSYS_LOG(WARN,"fail to set sst row rowkey [err:%d]", err);
}
if((OB_SUCCESS == err) && (OB_SUCCESS !=(err = sst_row.set_table_id(msolap::target_table_id))))
{
TBSYS_LOG(WARN,"fail to set sst row table id [err:%d]", err);
}
if((OB_SUCCESS == err) && (OB_SUCCESS !=(err = sst_row.set_column_group_id(msolap::ups_sst_column_group_id))))
{
TBSYS_LOG(WARN,"fail to set sst row group id [err:%d]", err);
}
for (uint64_t column_id = msolap::min_column_id; (OB_SUCCESS == err)&&(column_id <= msolap::max_column_id); column_id ++)
{
ObObj val;
val.set_int(msolap::olap_get_column_val(column_id,big_endian_row_key));
if (OB_SUCCESS != (err = sst_row.add_obj(val)))
{
TBSYS_LOG(WARN,"fail to add obj to sst row [err:%d]", err);
}
}
assert(OB_SUCCESS == sst_row.check_schema(*sst_schema));
if((OB_SUCCESS == err) && (OB_SUCCESS != (err = sst_writer.append_row(sst_row,space_usage))))
{
TBSYS_LOG(WARN,"fail to append row to sst writer [err:%d]", err);
}
}
int64_t trailer_offset = 0;
int64_t sst_size = 0;
if((OB_SUCCESS == err) && (OB_SUCCESS != (err = sst_writer.close_sstable(trailer_offset, sst_size))))
{
TBSYS_LOG(WARN,"fail to close sst [err:%d]", err);
}
else
{
TBSYS_LOG(INFO, "sst write success [trailer_offset:%ld,sst_size:%ld]", trailer_offset, sst_size);
}
delete sst_schema;
delete mgr;
return err;
}
TEST(ObMsScanParam, columns)
{
ObSchemaManagerV2 *mgr = new ObSchemaManagerV2();
tbsys::CConfig config;
EXPECT_TRUE(mgr->parse_from_file("schema.ini",config));
ObScanParam scan_param;
ObScanParam decoded_param;
ObStringBuf buf;
init_decode_param(buf,scan_param);
clear_assis();
EXPECT_EQ(ob_decode_scan_param(scan_param,*mgr,decoded_param),OB_SUCCESS);
ObMergerScanParam ms_scan_param;
EXPECT_EQ(ms_scan_param.set_param(decoded_param), OB_SUCCESS);
/// select basic columns
int32_t return_info_idx = 0;
EXPECT_EQ(scan_param.get_column_name_size(), ms_scan_param.get_cs_param()->get_column_id_size());
EXPECT_EQ(scan_param.get_column_name_size(), ms_scan_param.get_ms_param()->get_column_id_size());
for (int32_t i = 0; i < scan_param.get_column_name_size(); i ++, return_info_idx ++)
{
EXPECT_EQ(ms_scan_param.get_cs_param()->get_column_id()[i], ms_scan_param.get_ms_param()->get_column_id()[i]);
EXPECT_TRUE(*ms_scan_param.get_cs_param()->get_return_infos().at(return_info_idx));
EXPECT_EQ(*scan_param.get_return_infos().at(return_info_idx),
*ms_scan_param.get_ms_param()->get_return_infos().at(return_info_idx));
}
/// select composite columns
EXPECT_EQ(scan_param.get_composite_columns_size(), ms_scan_param.get_cs_param()->get_composite_columns_size());
EXPECT_EQ(scan_param.get_composite_columns_size(), ms_scan_param.get_ms_param()->get_composite_columns_size());
for (int32_t i = 0; i < scan_param.get_composite_columns_size();i ++, return_info_idx++)
{
EXPECT_TRUE(*ms_scan_param.get_cs_param()->get_return_infos().at(return_info_idx));
EXPECT_EQ(*scan_param.get_return_infos().at(return_info_idx),
*ms_scan_param.get_ms_param()->get_return_infos().at(return_info_idx));
const ObObj * cs_expr = ms_scan_param.get_cs_param()->get_composite_columns().at(i)->get_postfix_expr();
const ObObj * ms_expr = ms_scan_param.get_ms_param()->get_composite_columns().at(i)->get_postfix_expr();
for (int obj_idx = 0;; obj_idx += 2)
{
int64_t cs_type = 0;
int64_t ms_type = 0;
EXPECT_EQ(cs_expr[obj_idx].get_int(cs_type),OB_SUCCESS);
EXPECT_EQ(ms_expr[obj_idx].get_int(ms_type),OB_SUCCESS);
EXPECT_EQ(cs_type, ms_type);
if (ms_type == ObExpression::END)
{
break;
}
EXPECT_TRUE(cs_expr[obj_idx + 1] == ms_expr[obj_idx + 1]);
}
}
/// where condition
EXPECT_EQ(ms_scan_param.get_ms_param()->get_filter_info().get_count(), 0);
EXPECT_EQ(ms_scan_param.get_cs_param()->get_filter_info().get_count(), scan_param.get_filter_info().get_count());
return_info_idx = 0;
/// groupby::groupby columns
EXPECT_EQ(scan_param.get_group_by_param().get_groupby_columns().get_array_index(),
ms_scan_param.get_cs_param()->get_group_by_param().get_groupby_columns().get_array_index());
EXPECT_EQ(scan_param.get_group_by_param().get_groupby_columns().get_array_index(),
ms_scan_param.get_ms_param()->get_group_by_param().get_groupby_columns().get_array_index());
for (int32_t i = 0; i < scan_param.get_group_by_param().get_groupby_columns().get_array_index(); i ++, return_info_idx ++)
{
EXPECT_TRUE(*ms_scan_param.get_ms_param()->get_group_by_param().get_groupby_columns().at(i) ==
*ms_scan_param.get_ms_param()->get_group_by_param().get_groupby_columns().at(i));
EXPECT_TRUE(*ms_scan_param.get_cs_param()->get_group_by_param().get_return_infos().at(return_info_idx));
EXPECT_EQ(*scan_param.get_group_by_param().get_return_infos().at(return_info_idx),
*ms_scan_param.get_ms_param()->get_group_by_param().get_return_infos().at(return_info_idx));
}
/// groupby::aggregate columns
EXPECT_EQ(scan_param.get_group_by_param().get_aggregate_columns().get_array_index(),
ms_scan_param.get_cs_param()->get_group_by_param().get_aggregate_columns().get_array_index());
EXPECT_EQ(scan_param.get_group_by_param().get_aggregate_columns().get_array_index(),
ms_scan_param.get_ms_param()->get_group_by_param().get_aggregate_columns().get_array_index());
for (int32_t i = 0; i < scan_param.get_group_by_param().get_aggregate_columns().get_array_index(); i ++, return_info_idx ++)
{
EXPECT_TRUE(*ms_scan_param.get_ms_param()->get_group_by_param().get_aggregate_columns().at(i) ==
*ms_scan_param.get_ms_param()->get_group_by_param().get_aggregate_columns().at(i));
EXPECT_TRUE(*ms_scan_param.get_cs_param()->get_group_by_param().get_return_infos().at(return_info_idx));
EXPECT_EQ(*scan_param.get_group_by_param().get_return_infos().at(return_info_idx),
*ms_scan_param.get_ms_param()->get_group_by_param().get_return_infos().at(return_info_idx));
}
/// groupby::composite columns columns
EXPECT_EQ(scan_param.get_group_by_param().get_return_infos().get_array_index(),
ms_scan_param.get_cs_param()->get_group_by_param().get_return_infos().get_array_index());
EXPECT_EQ(scan_param.get_group_by_param().get_return_infos().get_array_index(),
ms_scan_param.get_ms_param()->get_group_by_param().get_return_infos().get_array_index());
for (int32_t i = 0; i < scan_param.get_group_by_param().get_composite_columns().get_array_index();i ++, return_info_idx++)
{
EXPECT_TRUE(*ms_scan_param.get_cs_param()->get_group_by_param().get_return_infos().at(return_info_idx));
EXPECT_EQ(*scan_param.get_group_by_param().get_return_infos().at(return_info_idx),
*ms_scan_param.get_ms_param()->get_group_by_param().get_return_infos().at(return_info_idx));
const ObObj * cs_expr = ms_scan_param.get_cs_param()->get_group_by_param().get_composite_columns().at(i)->get_postfix_expr();
const ObObj * ms_expr = ms_scan_param.get_ms_param()->get_group_by_param().get_composite_columns().at(i)->get_postfix_expr();
for (int obj_idx = 0;; obj_idx += 2)
{
int64_t cs_type = 0;
int64_t ms_type = 0;
EXPECT_EQ(cs_expr[obj_idx].get_int(cs_type),OB_SUCCESS);
EXPECT_EQ(ms_expr[obj_idx].get_int(ms_type),OB_SUCCESS);
EXPECT_EQ(cs_type, ms_type);
if (ms_type == ObExpression::END)
{
break;
}
EXPECT_TRUE(cs_expr[obj_idx + 1] == ms_expr[obj_idx + 1]);
}
}
/// having condition
EXPECT_EQ(ms_scan_param.get_cs_param()->get_group_by_param().get_having_condition().get_count(), 0);
EXPECT_EQ(ms_scan_param.get_ms_param()->get_group_by_param().get_having_condition().get_count(),
scan_param.get_group_by_param().get_having_condition().get_count());
/// orderby
EXPECT_EQ(ms_scan_param.get_cs_param()->get_orderby_column_size(),
((scan_param.get_group_by_param().get_aggregate_row_width() > 0) && (scan_param.get_orderby_column_size() == 0))?
1: scan_param.get_orderby_column_size());
EXPECT_EQ(ms_scan_param.get_ms_param()->get_orderby_column_size(),
scan_param.get_orderby_column_size());
/// limit info witout sharding_minimum preicession
scan_param.set_limit_info(0,100);
clear_assis();
EXPECT_EQ(ob_decode_scan_param(scan_param,*mgr,decoded_param),OB_SUCCESS);
EXPECT_EQ(ms_scan_param.set_param(decoded_param), OB_SUCCESS);
/// orderby
EXPECT_EQ(ms_scan_param.get_cs_param()->get_orderby_column_size(),
((scan_param.get_group_by_param().get_aggregate_row_width() > 0) && (scan_param.get_orderby_column_size() == 0))?
1: scan_param.get_orderby_column_size());
EXPECT_EQ(ms_scan_param.get_ms_param()->get_orderby_column_size(),
scan_param.get_orderby_column_size());
int64_t limit_offset;
int64_t limit_count;
int64_t sharding_minimum;
double topk_precision;
ms_scan_param.get_cs_param()->get_limit_info(limit_offset,limit_count);
ms_scan_param.get_cs_param()->get_topk_precision(sharding_minimum,topk_precision);
EXPECT_EQ(limit_offset,0);
EXPECT_EQ(limit_count,0);
EXPECT_EQ(sharding_minimum,0);
EXPECT_EQ(topk_precision,0);
ms_scan_param.get_ms_param()->get_limit_info(limit_offset,limit_count);
ms_scan_param.get_ms_param()->get_topk_precision(sharding_minimum,topk_precision);
EXPECT_EQ(limit_offset,0);
EXPECT_EQ(limit_count,100);
EXPECT_EQ(sharding_minimum,0);
EXPECT_EQ(topk_precision,0);
/// limit info witout sharding_minimum preicession
scan_param.set_limit_info(0,100);
scan_param.set_topk_precision(1000, 0.5);
clear_assis();
EXPECT_EQ(ob_decode_scan_param(scan_param,*mgr,decoded_param),OB_SUCCESS);
EXPECT_EQ(ms_scan_param.set_param(decoded_param), OB_SUCCESS);
/// orderby
EXPECT_EQ(ms_scan_param.get_cs_param()->get_orderby_column_size(), scan_param.get_orderby_column_size());
EXPECT_EQ(ms_scan_param.get_ms_param()->get_orderby_column_size(),
scan_param.get_orderby_column_size());
ms_scan_param.get_cs_param()->get_limit_info(limit_offset,limit_count);
ms_scan_param.get_cs_param()->get_topk_precision(sharding_minimum,topk_precision);
EXPECT_EQ(limit_offset,0);
EXPECT_EQ(limit_count,0);
EXPECT_EQ(sharding_minimum,1000);
EXPECT_EQ(topk_precision,0.5);
ms_scan_param.get_ms_param()->get_limit_info(limit_offset,limit_count);
ms_scan_param.get_ms_param()->get_topk_precision(sharding_minimum,topk_precision);
EXPECT_EQ(limit_offset,0);
EXPECT_EQ(limit_count,100);
EXPECT_EQ(sharding_minimum,0);
EXPECT_EQ(topk_precision,0);
delete mgr;
}
