void print_schema(const ObSchemaManagerV2 &sm)
{
uint64_t cur_table_id = OB_INVALID_ID;
const ObColumnSchemaV2 *iter = NULL;
for (iter = sm.column_begin(); iter != sm.column_end(); iter++)
{
if (NULL != iter)
{
if (iter->get_table_id() != cur_table_id)
{
const ObTableSchema *table_schema = sm.get_table_schema(iter->get_table_id());
if (NULL != table_schema)
{
fprintf(stdout, "[TABLE_SCHEMA] table_id=%lu table_type=%d table_name=%s split_pos=%d rowkey_max_length=%d\n",
iter->get_table_id(), table_schema->get_table_type(), table_schema->get_table_name(),
table_schema->get_split_pos(), table_schema->get_rowkey_max_length());
}
else
{
fprintf(stdout, "[TABLE_SCHEMA] table_id=%lu\n", iter->get_table_id());
}
cur_table_id = iter->get_table_id();
}
fprintf(stdout, " [COLUMN_SCHEMA] column_id=%lu column_name=%s column_type=%d size=%ld\n",
iter->get_id(), iter->get_name(), iter->get_type(), iter->get_size());
}
}
}
void print_schema(const char *fname)
{
int fd = open(fname, O_RDONLY);
if (-1 == fd)
{
fprintf(stderr, "open file [%s] fail errno=%u\n", fname, errno);
}
else
{
struct stat st;
fstat(fd, &st);
char *buffer = new char[st.st_size];
if (NULL == buffer)
{
fprintf(stderr, "new buffer fail size=%ld\n", st.st_size);
}
else
{
int64_t read_ret = read(fd, buffer, st.st_size);
if (st.st_size != read_ret)
{
fprintf(stderr, "read file fail ret=%ld size=%ld errno=%u\n", read_ret, st.st_size, errno);
}
else
{
ObSchemaManagerV2 *sm = new(std::nothrow) ObSchemaManagerV2();
if (NULL == sm)
{
fprintf(stdout, "[%s] new ObSchemaManagerV2 fail\n", __FUNCTION__);
}
else
{
int64_t pos = 0;
int ret = sm->deserialize(buffer, st.st_size, pos);
if (OB_SUCCESS != ret)
{
fprintf(stderr, "deserialize fail ret=%d\n", ret);
}
else
{
print_schema(*sm);
}
delete sm;
}
}
delete[] buffer;
}
close(fd);
}
}
int ObClientServerStub::fetch_schema(ObSchemaManagerV2 &schema)
{
int ret = OB_SUCCESS;
const int64_t timeout = 1000000;
ObDataBuffer data_buff;
ret = get_frame_buffer(data_buff);
// step 1. serialize timestamp to data_buff
if (OB_SUCCESS == ret)
{
ret = serialization::encode_vi64(data_buff.get_data(),
data_buff.get_capacity(), data_buff.get_position(), DEFAULT_VERSION);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(ERROR, "serialize timestamp failed:version[%ld], ret[%d]",
DEFAULT_VERSION, ret);
}
}
// step 2. send get update server info request
if (OB_SUCCESS == ret)
{
ret = rpc_frame_->send_request(root_server_, OB_FETCH_SCHEMA, DEFAULT_VERSION,
timeout, data_buff);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "send request to root server for getting schema");
}
}
// step 3. deserialize restult code
int64_t pos = 0;
if (OB_SUCCESS == ret)
{
ObResultCode result_code;
ret = result_code.deserialize(data_buff.get_data(), data_buff.get_position(), pos);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(ERROR, "deserialize result_code failed:pos[%ld], ret[%d]", pos, ret);
}
else
{
ret = result_code.result_code_;
}
}
// step 4. deserialize update server addr
if (OB_SUCCESS == ret)
{
ret = schema.deserialize(data_buff.get_data(), data_buff.get_position(), pos);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "deserialize server failed:pos[%ld], ret[%d]", pos, ret);
}
}
return ret;
}
int ObParamDecoder::decode_cell(const bool is_ext, const ObCellInfo & org_cell,
const ObSchemaManagerV2 & schema, ObCellInfo & decoded_cell)
{
int ret = OB_SUCCESS;
if (false == is_ext)
{
const ObColumnSchemaV2 * column_schema = schema.get_column_schema(org_cell.table_name_,
org_cell.column_name_);
if (NULL == column_schema)
{
ret = OB_SCHEMA_ERROR;
TBSYS_LOG(WARN, "check column failed:table[%.*s], column[%.*s], ret[%d]",
org_cell.table_name_.length(), org_cell.table_name_.ptr(),
org_cell.column_name_.length(), org_cell.column_name_.ptr(), ret);
}
else
{
decoded_cell.table_id_ = column_schema->get_table_id();
decoded_cell.column_id_ = column_schema->get_id();
decoded_cell.row_key_ = org_cell.row_key_;
decoded_cell.value_ = org_cell.value_;
}
}
else
{
const ObTableSchema * table_schema = schema.get_table_schema(org_cell.table_name_);
if (NULL == table_schema)
{
ret = OB_SCHEMA_ERROR;
TBSYS_LOG(WARN, "check table failed:table[%.*s], ret[%d]", org_cell.table_name_.length(),
org_cell.table_name_.ptr(), ret);
}
else
{
decoded_cell.table_id_ = table_schema->get_table_id();
// delete row is already using this column id
// using OB_ALL_MAX_COLUMN_ID for exist type
decoded_cell.column_id_ = OB_ALL_MAX_COLUMN_ID;
decoded_cell.row_key_ = org_cell.row_key_;
decoded_cell.value_ = org_cell.value_;
}
}
return ret;
}
TEST_F(TestRpcStub, test_fetch_schema)
{
ObMergerRpcStub stub;
ThreadSpecificBuffer buffer;
ObPacketFactory factory;
tbnet::Transport transport;
tbnet::DefaultPacketStreamer streamer;
streamer.setPacketFactory(&factory);
transport.start();
ObClientManager client_manager;
EXPECT_TRUE(OB_SUCCESS == client_manager.initialize(&transport, &streamer));
EXPECT_TRUE(OB_SUCCESS == stub.init(&buffer, &client_manager));
// self server
ObServer root_server;
root_server.set_ipv4_addr(addr, MockRootServer::ROOT_SERVER_PORT);
// start root server
MockRootServer server;
MockServerRunner test_root_server(server);
tbsys::CThread root_server_thread;
root_server_thread.start(&test_root_server, NULL);
sleep(2);
ObSchemaManagerV2 manager;
// wrong version
int64_t timestamp = 1023;
EXPECT_TRUE(OB_SUCCESS != stub.fetch_schema(timeout, root_server, timestamp, manager));
timestamp = 1024;
EXPECT_TRUE(OB_SUCCESS == stub.fetch_schema(timeout, root_server, timestamp, manager));
EXPECT_TRUE(manager.get_version() == 1025);
transport.stop();
server.stop();
sleep(10);
}
int fill_sstable_schema(ObSchemaManagerV2 &mgr, const uint64_t table_id, ObSSTableSchema& sstable_schema)
{
int ret = OB_SUCCESS;
int32_t cols = 0;
int32_t size = 0;
ObSSTableSchemaColumnDef column_def;
sstable_schema.reset();
const ObColumnSchemaV2 *col = mgr.get_table_schema( table_id,size);
if (NULL == col || size <= 0)
{
TBSYS_LOG(ERROR,"cann't find this table:%lu",table_id);
ret = OB_ERROR;
}
else
{
for (int col_index = 0; col_index < size && OB_SUCCESS == ret; ++col_index)
{
memset(&column_def,0,sizeof(column_def));
column_def.table_id_ = static_cast<uint32_t>(table_id);
column_def.column_group_id_ = static_cast<uint16_t>((col + col_index)->get_column_group_id());
column_def.column_name_id_ = static_cast<uint16_t>((col + col_index)->get_id());
column_def.column_value_type_ = (col + col_index)->get_type();
if ( (ret = sstable_schema.add_column_def(column_def)) != OB_SUCCESS )
{
TBSYS_LOG(ERROR,"add column_def(%u,%u,%u) failed col_index : %d",column_def.table_id_,
column_def.column_group_id_,column_def.column_name_id_,col_index);
}
++cols;
}
}
if ( 0 == cols && OB_SUCCESS == ret ) //this table has moved to updateserver
{
ret = OB_CS_TABLE_HAS_DELETED;
}
return ret;
}
int ObServerRpc::fetch_schema(const ObServer& root_server,
const int64_t timestap,
ObSchemaManagerV2& schema,
const int64_t timeout)
{
int ret = OB_SUCCESS;
int64_t pos = 0;
ObResultCode result_code;
ObDataBuffer data_buff;
if (root_server.get_ipv4() == 0 || root_server.get_port() == 0
|| timestap < 0 || timeout <= 0)
{
TBSYS_LOG(WARN, "invalid param, ip=%d, port=%d, timestap=%ld, "
"timeout=%ld",
root_server.get_ipv4(), root_server.get_port(),
timestap, timeout);
ret = OB_ERROR;
}
else if (NULL == rpc_frame_)
{
TBSYS_LOG(WARN, "server rpc doesn't init.");
ret = OB_ERROR;
}
if (OB_SUCCESS == ret)
{
ret = get_frame_buffer(data_buff);
}
// step 1. serialize timestap to data_buff
if (OB_SUCCESS == ret)
{
ret = encode_vi64(data_buff.get_data(),
data_buff.get_capacity(),
data_buff.get_position(), timestap);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(WARN, "serialize timestap failed, timestap=%ld, ret=%d.",
timestap, ret);
}
}
// step 2. send request for fetch new schema
if (OB_SUCCESS == ret)
{
ret = rpc_frame_->send_request(root_server,
OB_FETCH_SCHEMA, DEFAULT_VERSION,
timeout, data_buff);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(WARN, "send request to root server for fetch schema failed, "
"timestap=%ld, ret=%d.", timestap, ret);
}
}
// step 3. deserialize the response code
if (OB_SUCCESS == ret)
{
ret = result_code.deserialize(data_buff.get_data(),
data_buff.get_position(), pos);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(WARN, "deserialize result_code failed, pos=%ld, ret=%d.",
pos, ret);
}
else
{
ret = result_code.result_code_;
if (OB_SUCCESS != ret)
{
TBSYS_LOG(WARN, "get response from root server failed, "
"timeout=%ld, ret=%d.", timeout, ret);
}
}
}
// step 4. deserialize the table schema
if (OB_SUCCESS == ret)
{
ret = schema.deserialize(data_buff.get_data(),
data_buff.get_position(), pos);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(WARN, "deserialize schema from buffer failed, "
"timestap=%ld, pos=%ld, ret=%d",
timestap, pos, ret);
}
}
return ret;
}
int gen_sst(const gen_param ¶m)
{
ObSSTableSchema *sst_schema = new(std::nothrow)ObSSTableSchema;
ObSSTableWriter sst_writer;
ObSSTableRow sst_row;
tbsys::CConfig config;
ObSchemaManagerV2 *mgr = new(std::nothrow) ObSchemaManagerV2;
int err = OB_SUCCESS;
if ((NULL == mgr) || (NULL == sst_schema))
{
err = OB_ALLOCATE_MEMORY_FAILED;
TBSYS_LOG(WARN,"fail to allocate memory for ObSchemaManagerV2");
}
if ((OB_SUCCESS == err) && (!mgr->parse_from_file(param.schema_,config)))
{
TBSYS_LOG(WARN,"fail to load schema from file [err:%d,schema.ini:%s]", err, param.schema_);
err = OB_INVALID_ARGUMENT;
}
if ((OB_SUCCESS == err) && (OB_SUCCESS != (err = fill_sstable_schema(*mgr,msolap::target_table_id,*sst_schema))))
{
TBSYS_LOG(WARN,"fail to fill sst schema [err:%d, table_id:%lu]", err, msolap::target_table_id);
}
ObTrailerParam trailer_param;
trailer_param.block_size_ = msolap::ups_sst_block_size;
trailer_param.compressor_name_.assign((char*)msolap::ups_sst_compressor_name, static_cast<int32_t>(strlen(msolap::ups_sst_compressor_name)));
trailer_param.frozen_time_ = tbsys::CTimeUtil::getTime();
trailer_param.store_type_ = msolap::ups_sst_store_type;
trailer_param.table_version_ = msolap::ups_sst_version;
ObString sst_path(static_cast<int32_t>(strlen(param.sst_name_)), static_cast<int32_t>(strlen(param.sst_name_)),(char*)param.sst_name_);
if ((OB_SUCCESS == err) && (OB_SUCCESS != sst_writer.create_sstable(*sst_schema,sst_path, trailer_param)))
{
TBSYS_LOG(WARN,"fail to create sst [err:%d,sst_path:%s]", err, param.sst_name_);
}
uint32_t big_endian_row_key = 0;
int64_t space_usage = 0;
ObString row_key_str(sizeof(big_endian_row_key), sizeof(big_endian_row_key), (char*)&big_endian_row_key);
if((OB_SUCCESS == err) && (OB_SUCCESS != (err = sst_row.set_obj_count(msolap::max_column_id - msolap::min_column_id + 1))))
{
TBSYS_LOG(WARN,"fail to call sst_row.set_obj_count [err:%d]", err);
}
for (uint32_t row_key = param.start_include_; (OB_SUCCESS == err) && (row_key <= param.end_include_); row_key++)
{
big_endian_row_key = htonl(row_key);
sst_row.clear();
if ((OB_SUCCESS == err) &&(OB_SUCCESS != (err = sst_row.set_rowkey(TestRowkeyHelper(row_key_str, &allocator_)))))
{
TBSYS_LOG(WARN,"fail to set sst row rowkey [err:%d]", err);
}
if((OB_SUCCESS == err) && (OB_SUCCESS !=(err = sst_row.set_table_id(msolap::target_table_id))))
{
TBSYS_LOG(WARN,"fail to set sst row table id [err:%d]", err);
}
if((OB_SUCCESS == err) && (OB_SUCCESS !=(err = sst_row.set_column_group_id(msolap::ups_sst_column_group_id))))
{
TBSYS_LOG(WARN,"fail to set sst row group id [err:%d]", err);
}
for (uint64_t column_id = msolap::min_column_id; (OB_SUCCESS == err)&&(column_id <= msolap::max_column_id); column_id ++)
{
ObObj val;
val.set_int(msolap::olap_get_column_val(column_id,big_endian_row_key));
if (OB_SUCCESS != (err = sst_row.add_obj(val)))
{
TBSYS_LOG(WARN,"fail to add obj to sst row [err:%d]", err);
}
}
assert(OB_SUCCESS == sst_row.check_schema(*sst_schema));
if((OB_SUCCESS == err) && (OB_SUCCESS != (err = sst_writer.append_row(sst_row,space_usage))))
{
TBSYS_LOG(WARN,"fail to append row to sst writer [err:%d]", err);
}
}
int64_t trailer_offset = 0;
int64_t sst_size = 0;
if((OB_SUCCESS == err) && (OB_SUCCESS != (err = sst_writer.close_sstable(trailer_offset, sst_size))))
{
TBSYS_LOG(WARN,"fail to close sst [err:%d]", err);
}
else
{
TBSYS_LOG(INFO, "sst write success [trailer_offset:%ld,sst_size:%ld]", trailer_offset, sst_size);
}
delete sst_schema;
delete mgr;
return err;
}
TEST(ObMsScanParam, columns)
{
ObSchemaManagerV2 *mgr = new ObSchemaManagerV2();
tbsys::CConfig config;
EXPECT_TRUE(mgr->parse_from_file("schema.ini",config));
ObScanParam scan_param;
ObScanParam decoded_param;
ObStringBuf buf;
init_decode_param(buf,scan_param);
clear_assis();
EXPECT_EQ(ob_decode_scan_param(scan_param,*mgr,decoded_param),OB_SUCCESS);
ObMergerScanParam ms_scan_param;
EXPECT_EQ(ms_scan_param.set_param(decoded_param), OB_SUCCESS);
/// select basic columns
int32_t return_info_idx = 0;
EXPECT_EQ(scan_param.get_column_name_size(), ms_scan_param.get_cs_param()->get_column_id_size());
EXPECT_EQ(scan_param.get_column_name_size(), ms_scan_param.get_ms_param()->get_column_id_size());
for (int32_t i = 0; i < scan_param.get_column_name_size(); i ++, return_info_idx ++)
{
EXPECT_EQ(ms_scan_param.get_cs_param()->get_column_id()[i], ms_scan_param.get_ms_param()->get_column_id()[i]);
EXPECT_TRUE(*ms_scan_param.get_cs_param()->get_return_infos().at(return_info_idx));
EXPECT_EQ(*scan_param.get_return_infos().at(return_info_idx),
*ms_scan_param.get_ms_param()->get_return_infos().at(return_info_idx));
}
/// select composite columns
EXPECT_EQ(scan_param.get_composite_columns_size(), ms_scan_param.get_cs_param()->get_composite_columns_size());
EXPECT_EQ(scan_param.get_composite_columns_size(), ms_scan_param.get_ms_param()->get_composite_columns_size());
for (int32_t i = 0; i < scan_param.get_composite_columns_size();i ++, return_info_idx++)
{
EXPECT_TRUE(*ms_scan_param.get_cs_param()->get_return_infos().at(return_info_idx));
EXPECT_EQ(*scan_param.get_return_infos().at(return_info_idx),
*ms_scan_param.get_ms_param()->get_return_infos().at(return_info_idx));
const ObObj * cs_expr = ms_scan_param.get_cs_param()->get_composite_columns().at(i)->get_postfix_expr();
const ObObj * ms_expr = ms_scan_param.get_ms_param()->get_composite_columns().at(i)->get_postfix_expr();
for (int obj_idx = 0;; obj_idx += 2)
{
int64_t cs_type = 0;
int64_t ms_type = 0;
EXPECT_EQ(cs_expr[obj_idx].get_int(cs_type),OB_SUCCESS);
EXPECT_EQ(ms_expr[obj_idx].get_int(ms_type),OB_SUCCESS);
EXPECT_EQ(cs_type, ms_type);
if (ms_type == ObExpression::END)
{
break;
}
EXPECT_TRUE(cs_expr[obj_idx + 1] == ms_expr[obj_idx + 1]);
}
}
/// where condition
EXPECT_EQ(ms_scan_param.get_ms_param()->get_filter_info().get_count(), 0);
EXPECT_EQ(ms_scan_param.get_cs_param()->get_filter_info().get_count(), scan_param.get_filter_info().get_count());
return_info_idx = 0;
/// groupby::groupby columns
EXPECT_EQ(scan_param.get_group_by_param().get_groupby_columns().get_array_index(),
ms_scan_param.get_cs_param()->get_group_by_param().get_groupby_columns().get_array_index());
EXPECT_EQ(scan_param.get_group_by_param().get_groupby_columns().get_array_index(),
ms_scan_param.get_ms_param()->get_group_by_param().get_groupby_columns().get_array_index());
for (int32_t i = 0; i < scan_param.get_group_by_param().get_groupby_columns().get_array_index(); i ++, return_info_idx ++)
{
EXPECT_TRUE(*ms_scan_param.get_ms_param()->get_group_by_param().get_groupby_columns().at(i) ==
*ms_scan_param.get_ms_param()->get_group_by_param().get_groupby_columns().at(i));
EXPECT_TRUE(*ms_scan_param.get_cs_param()->get_group_by_param().get_return_infos().at(return_info_idx));
EXPECT_EQ(*scan_param.get_group_by_param().get_return_infos().at(return_info_idx),
*ms_scan_param.get_ms_param()->get_group_by_param().get_return_infos().at(return_info_idx));
}
/// groupby::aggregate columns
EXPECT_EQ(scan_param.get_group_by_param().get_aggregate_columns().get_array_index(),
ms_scan_param.get_cs_param()->get_group_by_param().get_aggregate_columns().get_array_index());
EXPECT_EQ(scan_param.get_group_by_param().get_aggregate_columns().get_array_index(),
ms_scan_param.get_ms_param()->get_group_by_param().get_aggregate_columns().get_array_index());
for (int32_t i = 0; i < scan_param.get_group_by_param().get_aggregate_columns().get_array_index(); i ++, return_info_idx ++)
{
EXPECT_TRUE(*ms_scan_param.get_ms_param()->get_group_by_param().get_aggregate_columns().at(i) ==
*ms_scan_param.get_ms_param()->get_group_by_param().get_aggregate_columns().at(i));
EXPECT_TRUE(*ms_scan_param.get_cs_param()->get_group_by_param().get_return_infos().at(return_info_idx));
EXPECT_EQ(*scan_param.get_group_by_param().get_return_infos().at(return_info_idx),
*ms_scan_param.get_ms_param()->get_group_by_param().get_return_infos().at(return_info_idx));
}
/// groupby::composite columns columns
EXPECT_EQ(scan_param.get_group_by_param().get_return_infos().get_array_index(),
ms_scan_param.get_cs_param()->get_group_by_param().get_return_infos().get_array_index());
EXPECT_EQ(scan_param.get_group_by_param().get_return_infos().get_array_index(),
ms_scan_param.get_ms_param()->get_group_by_param().get_return_infos().get_array_index());
for (int32_t i = 0; i < scan_param.get_group_by_param().get_composite_columns().get_array_index();i ++, return_info_idx++)
{
EXPECT_TRUE(*ms_scan_param.get_cs_param()->get_group_by_param().get_return_infos().at(return_info_idx));
EXPECT_EQ(*scan_param.get_group_by_param().get_return_infos().at(return_info_idx),
*ms_scan_param.get_ms_param()->get_group_by_param().get_return_infos().at(return_info_idx));
const ObObj * cs_expr = ms_scan_param.get_cs_param()->get_group_by_param().get_composite_columns().at(i)->get_postfix_expr();
const ObObj * ms_expr = ms_scan_param.get_ms_param()->get_group_by_param().get_composite_columns().at(i)->get_postfix_expr();
for (int obj_idx = 0;; obj_idx += 2)
{
int64_t cs_type = 0;
int64_t ms_type = 0;
EXPECT_EQ(cs_expr[obj_idx].get_int(cs_type),OB_SUCCESS);
EXPECT_EQ(ms_expr[obj_idx].get_int(ms_type),OB_SUCCESS);
EXPECT_EQ(cs_type, ms_type);
if (ms_type == ObExpression::END)
{
break;
}
EXPECT_TRUE(cs_expr[obj_idx + 1] == ms_expr[obj_idx + 1]);
}
}
/// having condition
EXPECT_EQ(ms_scan_param.get_cs_param()->get_group_by_param().get_having_condition().get_count(), 0);
EXPECT_EQ(ms_scan_param.get_ms_param()->get_group_by_param().get_having_condition().get_count(),
scan_param.get_group_by_param().get_having_condition().get_count());
/// orderby
EXPECT_EQ(ms_scan_param.get_cs_param()->get_orderby_column_size(),
((scan_param.get_group_by_param().get_aggregate_row_width() > 0) && (scan_param.get_orderby_column_size() == 0))?
1: scan_param.get_orderby_column_size());
EXPECT_EQ(ms_scan_param.get_ms_param()->get_orderby_column_size(),
scan_param.get_orderby_column_size());
/// limit info witout sharding_minimum preicession
scan_param.set_limit_info(0,100);
clear_assis();
EXPECT_EQ(ob_decode_scan_param(scan_param,*mgr,decoded_param),OB_SUCCESS);
EXPECT_EQ(ms_scan_param.set_param(decoded_param), OB_SUCCESS);
/// orderby
EXPECT_EQ(ms_scan_param.get_cs_param()->get_orderby_column_size(),
((scan_param.get_group_by_param().get_aggregate_row_width() > 0) && (scan_param.get_orderby_column_size() == 0))?
1: scan_param.get_orderby_column_size());
EXPECT_EQ(ms_scan_param.get_ms_param()->get_orderby_column_size(),
scan_param.get_orderby_column_size());
int64_t limit_offset;
int64_t limit_count;
int64_t sharding_minimum;
double topk_precision;
ms_scan_param.get_cs_param()->get_limit_info(limit_offset,limit_count);
ms_scan_param.get_cs_param()->get_topk_precision(sharding_minimum,topk_precision);
EXPECT_EQ(limit_offset,0);
EXPECT_EQ(limit_count,0);
EXPECT_EQ(sharding_minimum,0);
EXPECT_EQ(topk_precision,0);
ms_scan_param.get_ms_param()->get_limit_info(limit_offset,limit_count);
ms_scan_param.get_ms_param()->get_topk_precision(sharding_minimum,topk_precision);
EXPECT_EQ(limit_offset,0);
EXPECT_EQ(limit_count,100);
EXPECT_EQ(sharding_minimum,0);
EXPECT_EQ(topk_precision,0);
/// limit info witout sharding_minimum preicession
scan_param.set_limit_info(0,100);
scan_param.set_topk_precision(1000, 0.5);
clear_assis();
EXPECT_EQ(ob_decode_scan_param(scan_param,*mgr,decoded_param),OB_SUCCESS);
EXPECT_EQ(ms_scan_param.set_param(decoded_param), OB_SUCCESS);
/// orderby
EXPECT_EQ(ms_scan_param.get_cs_param()->get_orderby_column_size(), scan_param.get_orderby_column_size());
EXPECT_EQ(ms_scan_param.get_ms_param()->get_orderby_column_size(),
scan_param.get_orderby_column_size());
ms_scan_param.get_cs_param()->get_limit_info(limit_offset,limit_count);
ms_scan_param.get_cs_param()->get_topk_precision(sharding_minimum,topk_precision);
EXPECT_EQ(limit_offset,0);
EXPECT_EQ(limit_count,0);
EXPECT_EQ(sharding_minimum,1000);
EXPECT_EQ(topk_precision,0.5);
ms_scan_param.get_ms_param()->get_limit_info(limit_offset,limit_count);
ms_scan_param.get_ms_param()->get_topk_precision(sharding_minimum,topk_precision);
EXPECT_EQ(limit_offset,0);
EXPECT_EQ(limit_count,100);
EXPECT_EQ(sharding_minimum,0);
EXPECT_EQ(topk_precision,0);
delete mgr;
}
int ObMergerRootRpcProxy::fetch_newest_schema(ObMergerSchemaManager * schema_manager,
const ObSchemaManagerV2 ** manager)
{
/// fetch new schema
int ret = OB_SUCCESS;
ObSchemaManagerV2 * schema = NULL;
if (!check_inner_stat())
{
TBSYS_LOG(ERROR, "%s", "check inner stat failed");
ret = OB_INNER_STAT_ERROR;
}
char * temp = NULL;
if(OB_SUCCESS == ret)
{
temp = (char *)ob_malloc(sizeof(ObSchemaManagerV2), ObModIds::OB_MS_RPC);
if (NULL == temp)
{
TBSYS_LOG(ERROR, "%s", "check ob malloc failed");
ret = OB_ALLOCATE_MEMORY_FAILED;
}
else
{
schema = new(temp) ObSchemaManagerV2;
if (NULL == schema)
{
TBSYS_LOG(ERROR, "check replacement new schema failed:schema[%p]", schema);
ret = OB_INNER_STAT_ERROR;
}
else
{
ret = rpc_stub_->fetch_schema(rpc_timeout_, root_server_, 0, *schema);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(WARN, "rpc fetch schema failed:ret[%d]", ret);
}
}
}
}
if (OB_SUCCESS == ret)
{
ret = schema_manager->add_schema(*schema, manager);
// maybe failed because of timer thread fetch and add it already
if (OB_SUCCESS != ret)
{
TBSYS_LOG(WARN, "add new schema failed:version[%ld], ret[%d]", schema->get_version(), ret);
ret = OB_SUCCESS;
*manager = schema_manager->get_schema(0);
if (NULL == *manager)
{
TBSYS_LOG(WARN, "get latest schema failed:schema[%p], latest[%ld]",
*manager, schema_manager->get_latest_version());
ret = OB_INNER_STAT_ERROR;
}
}
else
{
TBSYS_LOG(DEBUG, "fetch and add new schema succ:version[%ld]", schema->get_version());
}
}
if (schema != NULL)
{
schema->~ObSchemaManagerV2();
}
if (temp != NULL)
{
ob_free(temp);
temp = NULL;
}
ms_get_counter_set().inc(ObMergerCounterIds::C_FETCH_SCHEMA);
return ret;
}
int ObRootServerRpcStub::fetch_schema(const int64_t timestamp, ObSchemaManagerV2 & schema)
{
int ret = OB_SUCCESS;
// send_request timeout us
const int64_t timeout = ObChunkServerMain::get_instance()->get_chunk_server().get_param().get_network_time_out();
ObDataBuffer data_buff;
ret = get_frame_buffer(data_buff);
// step 1. serialize timestamp to data_buff
if (OB_SUCCESS == ret)
{
ret = common::serialization::encode_vi64(data_buff.get_data(),
data_buff.get_capacity(), data_buff.get_position(), timestamp);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(ERROR, "serialize timestamp failed:timestamp[%ld], ret[%d].",
timestamp, ret);
}
}
// step 2. send request for fetch new schema
if (OB_SUCCESS == ret)
{
ret = rpc_frame_->send_request(root_server_, OB_FETCH_SCHEMA, DEFAULT_VERSION, timeout, data_buff);
if (ret != OB_SUCCESS)
{
const int32_t MAX_SERVER_ADDR_SIZE = 128;
char server_addr[MAX_SERVER_ADDR_SIZE];
root_server_.to_string(server_addr, MAX_SERVER_ADDR_SIZE);
TBSYS_LOG(ERROR, "send request to root server(%s) for fetch schema failed"
":timestamp[%ld], ret[%d].",server_addr, timestamp, ret);
}
}
// step 3. deserialize the response code
int64_t pos = 0;
if (OB_SUCCESS == ret)
{
ObResultCode result_code;
ret = result_code.deserialize(data_buff.get_data(), data_buff.get_position(), pos);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(ERROR, "deserialize result_code failed:pos[%ld], ret[%d].", pos, ret);
}
else
{
ret = result_code.result_code_;
}
}
// step 4. deserialize the table schema
if (OB_SUCCESS == ret)
{
ret = schema.deserialize(data_buff.get_data(), data_buff.get_position(), pos);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(ERROR, "deserialize schema from buff failed,"
"timestamp[%ld], pos[%ld], ret[%d].", timestamp, pos, ret);
}
}
return ret;
}
int ObMergerRpcStub::fetch_schema(
const int64_t timeout, const ObServer & root_server,
const int64_t version, ObSchemaManagerV2 & schema) const
{
int ret = OB_SUCCESS;
ObDataBuffer data_buff;
ret = get_rpc_buffer(data_buff);
// step 1. serialize timestamp to data_buff
if (OB_SUCCESS == ret)
{
ret = serialization::encode_vi64(data_buff.get_data(),
data_buff.get_capacity(), data_buff.get_position(), version);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(WARN, "serialize timestamp failed:version[%ld], ret[%d]",
version, ret);
}
}
// step 2. send request for fetch new schema
if (OB_SUCCESS == ret)
{
ret = rpc_frame_->send_request(root_server, OB_FETCH_SCHEMA, DEFAULT_VERSION,
timeout, data_buff);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(WARN, "send request to root server[%s] for fetch schema failed:"
"version[%ld], ret[%d]", root_server.to_cstring(), version, ret);
}
}
// step 3. deserialize the response code
int64_t pos = 0;
if (OB_SUCCESS == ret)
{
ObResultCode result_code;
ret = result_code.deserialize(data_buff.get_data(), data_buff.get_position(), pos);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(WARN, "deserialize result_code failed:pos[%ld], ret[%d]", pos, ret);
}
else
{
ret = result_code.result_code_;
}
}
// step 4. deserialize the table schema
if (OB_SUCCESS == ret)
{
ret = schema.deserialize(data_buff.get_data(), data_buff.get_position(), pos);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(WARN, "deserialize schema from buff failed:"
"version[%ld], pos[%ld], ret[%d]", version, pos, ret);
}
else
{
TBSYS_LOG(DEBUG, "fetch schema succ:version[%ld]", schema.get_version());
//schema.print_info();
}
}
return ret;
}
int ObCSCreatePlan::get_basic_column_and_join_info(
const ObProject &project,
const ObSchemaManagerV2 &schema_mgr,
const uint64_t table_id,
const uint64_t renamed_table_id,
ObArray<uint64_t> &basic_columns,
ObTabletJoin::TableJoinInfo &table_join_info)
{
int ret = OB_SUCCESS;
ObSqlExpression expr;
uint64_t column_id = OB_INVALID_ID;
const ObColumnSchemaV2 *column_schema = NULL;
const ObColumnSchemaV2::ObJoinInfo *join_info = NULL;
const ObTableSchema *table_schema = NULL;
ObTabletJoin::JoinInfo join_info_item;
bool is_cid = false;
column_ids_.clear();
//找到所有的需要join的普通列
for(int32_t i=0;OB_SUCCESS == ret && i<project.get_output_columns().count();i++)
{
if(OB_SUCCESS != (ret = project.get_output_columns().at(i, expr)))
{
TBSYS_LOG(WARN, "get expression from out columns fail:ret[%d] i[%d]", ret, i);
}
if(OB_SUCCESS == ret)
{
column_id = expr.get_column_id();
if(OB_INVALID_ID == expr.get_table_id()
|| table_id == expr.get_table_id()
|| renamed_table_id == expr.get_table_id())
{
//do nothing 复合列 or OB_ACTION_FLAG_COLUMN_ID
}
else
{
ret = OB_ERR_UNEXPECTED;
TBSYS_LOG(WARN, "Unknown expression id, table_id[%lu], renamed_table_id[%lu], expr_table_id[%lu]",
table_id, renamed_table_id, expr.get_table_id());
}
}
if(OB_SUCCESS == ret)
{
//如果是普通列, 复合列不交给下层的物理操作符
//if( (OB_SUCCESS == (ret = expr.is_column_index_expr(is_cid))) && is_cid)
uint64_t tid = OB_INVALID_ID;
if ( (OB_SUCCESS == (ret = expr.get_column_index_expr(tid, column_id, is_cid))) && is_cid && common::OB_ACTION_FLAG_COLUMN_ID != column_id)
{
if (column_ids_.has_member((int32_t)column_id))
{
TBSYS_LOG(DEBUG, "duplicate column [%lu]", column_id);
continue; //如果发现重复列，就直接跳过
}
else if (!column_ids_.add_member((int32_t)column_id))
{
ret = OB_ERROR;
TBSYS_LOG(WARN, "fail to add member[%lu]", column_id);
}
else if(NULL == (column_schema = schema_mgr.get_column_schema(table_id, column_id)))
{
ret = OB_ERROR;
TBSYS_LOG(WARN, "get column schema fail:tid[%lu], table_id[%lu] column_id[%lu]", tid, table_id, column_id);
}
else
{
// erase duplicate columns;
if(OB_SUCCESS != (ret = basic_columns.push_back(column_id)))
{
TBSYS_LOG(WARN, "fail to push item:ret[%d], column_id[%lu]", ret, column_id);
}
}
//构造对应的join_info信息
if(OB_SUCCESS == ret)
{
join_info = column_schema->get_join_info();
if(NULL != join_info)
{
if(OB_INVALID_ID == table_join_info.left_table_id_)
{
table_join_info.left_table_id_ = table_id;
}
else if(table_join_info.left_table_id_ != table_id)
{
ret = OB_ERROR;
TBSYS_LOG(WARN, "left table id cannot change:t1[%lu], t2[%lu]", table_join_info.left_table_id_, table_id);
}
if(OB_SUCCESS == ret)
{
if(table_join_info.join_condition_.count() == 0)
{
uint64_t left_rowkey_column_id = OB_INVALID_ID;
for(uint64_t left_col_idx=0;left_col_idx<join_info->left_column_count_;left_col_idx++)
{
int64_t col_idx = join_info->left_column_offset_array_[left_col_idx];
if(NULL == (table_schema = schema_mgr.get_table_schema(table_id)))
{
ret = OB_ERROR;
TBSYS_LOG(WARN, "get table schema fail:table_id[%lu]", table_id);
}
else if(OB_SUCCESS != (ret = table_schema->get_rowkey_info().get_column_id(col_idx, left_rowkey_column_id)))
{
TBSYS_LOG(WARN, "get left rowkey column id fail:ret[%d], col_idx[%ld]", ret, col_idx);
}
else if(OB_SUCCESS != (ret = table_join_info.join_condition_.push_back(left_rowkey_column_id)))
{
TBSYS_LOG(WARN, "push left rowkey column id fail:ret[%d], left_rowkey_column_id[%ld]", ret, left_rowkey_column_id);
}
}
}
else if((uint64_t)table_join_info.join_condition_.count() != join_info->left_column_count_)
{
ret = OB_ERROR;
TBSYS_LOG(WARN, "join info count change");
}
}
if(OB_SUCCESS == ret)
{
table_join_info.right_table_id_ = join_info->join_table_;
join_info_item.left_column_id_ = column_schema->get_id();
join_info_item.right_column_id_ = join_info->correlated_column_;
if(OB_SUCCESS != (ret = table_join_info.join_column_.push_back(join_info_item)))
{
TBSYS_LOG(WARN, "add join info item fail:ret[%d]", ret);
}
}
}
}
}
}
}
if(OB_SUCCESS == ret)
{
//如果存在需要join的列
if(table_join_info.join_column_.count() > 0)
{
//添加join需要的列id
if(OB_SUCCESS == ret)
{
uint64_t left_column_id = OB_INVALID_ID;
for(int32_t i=0;OB_SUCCESS == ret && i<table_join_info.join_condition_.count();i++)
{
if(OB_SUCCESS != (ret = table_join_info.join_condition_.at(i, left_column_id)))
{
TBSYS_LOG(WARN, "get join cond fail:ret[%d] i[%d]", ret, i);
}
else if (column_ids_.has_member((int32_t)left_column_id))
{
//do nothing
}
else if (!column_ids_.add_member((int32_t)left_column_id))
{
ret = OB_ERROR;
TBSYS_LOG(WARN, "fail to add member[%lu]", column_id);
}
else
{
// erase duplicate columns;
if(OB_SUCCESS != (ret = basic_columns.push_back(left_column_id)))
{
TBSYS_LOG(WARN, "fail to push item:ret[%d], column_id[%lu]", ret, left_column_id);
}
}
}
}
}
}
return ret;
}
