int ObGetScanProxy::get_compact_scanner(const ObGetParam& get_param,
                                            ObScanner& orig_scanner,ObScanner& compact_scanner)
    {
      ObTabletManager::ObGetThreadContext*& get_context = tablet_manager_.get_cur_thread_get_contex();
      const ObGetParam::ObRowIndex* row_index           = get_param.get_row_index();      
      int64_t row_num                                   = orig_scanner.get_row_num();
      ColumnFilter* cf                                  = GET_TSI_MULT(ColumnFilter,TSI_CS_COLUMNFILTER_1);      
      ObTablet* tablet                                  = NULL;
      int ret                                           = OB_SUCCESS;      
      ObRowkey rowkey;

      if ((NULL == get_context) || (NULL == row_index) || (NULL == cf))
      {
        TBSYS_LOG(WARN,"get thread context of get failed");
        ret = OB_ERROR;
      }
      else
      {
        int64_t compactsstable_version = get_context->min_compactsstable_version_;
        if (ObVersion::compare(compactsstable_version,orig_scanner.get_data_version()) > 0)
        {
          compact_scanner.set_data_version(compactsstable_version);
          for(int64_t i=0; (i < row_num) && (OB_SUCCESS == ret); ++i)
          {
            tablet = get_context->tablets_[i];
            rowkey = get_param[row_index[i].offset_]->row_key_;

            build_get_column_filter(get_param,row_index[i].offset_,row_index[i].size_,*cf);

            if ((tablet != NULL) &&
                (ret = get_compact_row(*tablet,rowkey,compactsstable_version,cf,compact_scanner)) != OB_SUCCESS)
            {
              TBSYS_LOG(WARN,"get data from compactsstable failed,ret=%d",ret);
            }
          }
        }
      }
      return ret;
    }
Example #2
0
TEST(TestObScanner, get_row)
{
  ObScanner scanner;
  const int table_num = 10;
  const int row_num = 20;
  const int column_num = 10;
  buffer table_name[table_num];
  buffer row_key[row_num];
  buffer column_name[column_num];
  for (int i = 0; i < table_num; i++)
  {
    table_name[i].assigne("T").appende(i);
  }
  for (int i = 0; i < row_num; i++)
  {
    row_key[i].assigne("R").appende(i);
  }
  for (int i = 0; i < column_num; i++)
  {
    column_name[i].assigne("C").appende(i);
  }

  for (int i = 0; i < table_num; i++)
  {
    for (int j = 0; j < row_num; j++)
    {
      for (int k = 0; k < column_num; k++)
      {
        ObCellInfo cell;
        cell.table_name_ = table_name[i].get_obstring();
        cell.row_key_ = row_key[j].get_obstring();
        cell.column_name_ = column_name[k].get_obstring();
        cell.value_.set_int(1);
        scanner.add_cell(cell);
      }
    }
  }

  ObScanner::RowIterator it = scanner.row_begin();
  ASSERT_EQ(it ==  scanner.row_begin(), true);

  for (int i = 0; i < table_num; i++)
  {
    for (int j = 0; j < row_num; j++)
    {
      ObCellInfo *row;
      int64_t num = 0;
      ASSERT_EQ(OB_SUCCESS, scanner.next_row());
      ASSERT_EQ(OB_SUCCESS, scanner.get_row(&row, &num));
      ASSERT_EQ(column_num, num);
    }
  }
  ASSERT_EQ(OB_ITER_END, scanner.next_row());

  buffer buf(1 << 21);
  scanner.serialize(buf.ptre(), buf.capacity(), buf.length());
  printf("%ld\n", buf.length());
  ObScanner dscanner;
  int64_t pos = 0;
  dscanner.deserialize(buf.ptre(), buf.length(), pos);

  for (int i = 0; i < table_num; i++)
  {
    for (int j = 0; j < row_num; j++)
    {
      ObCellInfo *row;
      int64_t num = 0;
      ASSERT_EQ(OB_SUCCESS, dscanner.next_row());
      ASSERT_EQ(OB_SUCCESS, dscanner.get_row(&row, &num));
      ASSERT_EQ(column_num, num);
    }
  }
  ASSERT_EQ(OB_ITER_END, dscanner.next_row());

  scanner.clear();
  scanner.set_mem_size_limit(256);
  int j = 0, k = 0;

  int err = OB_SUCCESS;
  for (; j < row_num; j++)
  {
    for (k = 0; k < column_num; k++)
    {
      ObCellInfo cell;
      cell.table_name_ = table_name[0].get_obstring();
      cell.row_key_ = row_key[j].get_obstring();
      cell.column_name_ = column_name[k].get_obstring();
      cell.value_.set_int(1);
      if (OB_SUCCESS != (err = scanner.add_cell(cell)))
      {
        break;
      }
    }
    if (OB_SUCCESS != err)
    {
      break;
    }
  }
  if (OB_SUCCESS != err)
  {
    scanner.rollback();
  }

  for (int i = 0; i < j; i++)
  {
    ObCellInfo *row;
    int64_t num = 0;
    ASSERT_EQ(OB_SUCCESS, scanner.next_row());
    ASSERT_EQ(OB_SUCCESS, scanner.get_row(&row, &num));
    ASSERT_EQ(column_num, num);
  }
  ASSERT_EQ(OB_ITER_END, scanner.next_row());

  buf.length() = 0;
  scanner.serialize(buf.ptre(), buf.capacity(), buf.length());
  printf("%ld\n", buf.length());
  dscanner.reset();
  pos = 0;
  dscanner.deserialize(buf.ptre(), buf.length(), pos);
  printf("row_num=%ld\n", dscanner.get_row_num());

  for (int i = 0; i < j; i++)
  {
    ObCellInfo *row;
    int64_t num = 0;
    ASSERT_EQ(OB_SUCCESS, dscanner.next_row());
    ASSERT_EQ(OB_SUCCESS, dscanner.get_row(&row, &num));
    ASSERT_EQ(column_num, num);
  }
  ASSERT_EQ(OB_ITER_END, dscanner.next_row());
}
Example #3
0
TEST(TestObScanner, row_num)
{
  ObScanner scanner;
  const int table_num = 3;
  const int row_num = 20;
  const int column_num = 5;
  buffer table_name[table_num];
  buffer row_key[row_num];
  buffer column_name[column_num];
  for (int i = 0; i < table_num; i++)
  {
    table_name[i].assigne("T").appende(i);
  }
  for (int i = 0; i < row_num; i++)
  {
    row_key[i].assigne("R").appende(i);
  }
  for (int i = 0; i < column_num; i++)
  {
    column_name[i].assigne("C").appende(i);
  }

  for (int i = 0; i < table_num; i++)
  {
    for (int j = 0; j < row_num; j++)
    {
      for (int k = 0; k < column_num; k++)
      {
        ObCellInfo cell;
        cell.table_name_ = table_name[i].get_obstring();
        cell.row_key_ = row_key[j].get_obstring();
        cell.column_name_ = column_name[k].get_obstring();
        cell.value_.set_int(1);
        scanner.add_cell(cell);
      }
    }
  }

  ASSERT_EQ(scanner.get_row_num(), table_num * row_num);
  ASSERT_EQ(scanner.get_cell_num(), table_num * row_num * column_num);
  scanner.set_whole_result_row_num(1000);

  buffer buf(1 << 21);
  scanner.serialize(buf.ptre(), buf.capacity(), buf.length());
  printf("%ld\n", buf.length());
  ObScanner dscanner;
  int64_t pos = 0;
  dscanner.deserialize(buf.ptre(), buf.length(), pos);

  EXPECT_EQ(dscanner.get_row_num(), table_num * row_num);
  EXPECT_EQ(dscanner.get_cell_num(), table_num * row_num * column_num);
  EXPECT_EQ(dscanner.get_whole_result_row_num(), 1000);

  scanner.clear();
  scanner.set_mem_size_limit(256);
  int j = 0, k = 0;

  int err = OB_SUCCESS;
  for (; j < row_num; j++)
  {
    k = 0;
    for (; k < column_num; k++)
    {
      ObCellInfo cell;
      cell.table_name_ = table_name[0].get_obstring();
      cell.row_key_ = row_key[j].get_obstring();
      cell.column_name_ = column_name[k].get_obstring();
      cell.value_.set_int(1);
      if (OB_SUCCESS != (err = scanner.add_cell(cell)))
      {
        break;
      }
    }
    if (OB_SUCCESS != err)
    {
      break;
    }
  }
  if (OB_SUCCESS != err)
  {
    scanner.rollback();
  }

  EXPECT_EQ(scanner.get_row_num(), j);
  EXPECT_EQ(scanner.get_cell_num(), j * column_num);

  buf.length() = 0;
  scanner.serialize(buf.ptre(), buf.capacity(), buf.length());
  printf("%ld\n", buf.length());
  dscanner.reset();
  pos = 0;
  dscanner.deserialize(buf.ptre(), buf.length(), pos);

  EXPECT_EQ(dscanner.get_row_num(), j);
  EXPECT_EQ(dscanner.get_cell_num(), j * column_num);
}
int ObMergerSortedOperator::add_sharding_result(ObScanner & sharding_res, const ObNewRange &query_range,
  bool &is_finish)
{
  int err = OB_SUCCESS;
  if ((OB_SUCCESS == err) && (NULL == scan_param_))
  {
    TBSYS_LOG(WARN,"operator was not initialized yet [scan_param_:%p]", scan_param_);
    err = OB_INVALID_ARGUMENT;
  }
  if ((OB_SUCCESS == err) && (sharding_result_count_ >= MAX_SHARDING_RESULT_COUNT))
  {
    TBSYS_LOG(WARN,"array is full [MAX_SHARDING_RESULT_COUNT:%ld,sharding_result_count_:%ld]",
      MAX_SHARDING_RESULT_COUNT, sharding_result_count_);
    err = OB_ARRAY_OUT_OF_RANGE;
  }
  /// @todo (wushi [email protected]) check fullfill item number and size property
  bool is_fullfilled = false;
  int64_t fullfilled_item_num = 0;
  ObRowkey last_row_key;
  ObNewRange cs_tablet_range;
  if ((OB_SUCCESS == err) && (OB_SUCCESS != (err = sharding_res.get_is_req_fullfilled(is_fullfilled,fullfilled_item_num))))
  {
    TBSYS_LOG(WARN,"fail to get fullfilled info from sharding result [err:%d]", err);
  }
  if (OB_SUCCESS == err)
  {
    if (OB_SUCCESS == (err = sharding_res.get_last_row_key(last_row_key)))
    {
    }
    else if (OB_ENTRY_NOT_EXIST == err)
    {
      err = OB_SUCCESS;
    }
    else
    {
      TBSYS_LOG(WARN,"fail to get last rowkey from sharding result [err:%d]", err);
    }
  }

  if (OB_SUCCESS == err)
  {
    if (is_fullfilled)
    {
      if ((OB_SUCCESS == err) && (OB_SUCCESS != (err = sharding_res.get_range(cs_tablet_range))))
      {
        TBSYS_LOG(WARN,"fail to get tablet range from sharding result [err:%d]", err);
      }
      if (OB_SUCCESS == err)
      {
        if (ScanFlag::FORWARD == scan_param_->get_scan_direction())
        {
          last_row_key = cs_tablet_range.end_key_;
        }
        else
        {
          last_row_key = cs_tablet_range.start_key_;
        }
      }
    }
    if (OB_SUCCESS == err)
    {
      fullfilled_item_num = sharding_res.get_row_num();
      sharding_result_arr_[sharding_result_count_].init(sharding_res,query_range,*scan_param_, last_row_key, fullfilled_item_num);
      sharding_result_count_ ++;
    }
  }
  is_finish = false;
  if (OB_SUCCESS == err)
  {
    sort(is_finish, &sharding_res);
  }
  TBSYS_LOG(DEBUG, "add sharding result. is_finish=%d, err=%d", is_finish, err);
  return err;
}