size_t SSPAggrBgWorker::ReadTableOpLogMetaUpToClock( int32_t table_id, ClientTable *table, int32_t clock_to_push, TableOpLogMeta *table_oplog_meta, GetSerializedRowOpLogSizeFunc GetSerializedRowOpLogSize, BgOpLogPartition *bg_table_oplog) { if (clock_to_push < 0) return 0; size_t accum_table_oplog_bytes = 0; AbstractOpLog &table_oplog = table->get_oplog(); int32_t row_id; row_id = table_oplog_meta->InitGetUptoClock(clock_to_push); while (row_id >= 0) { AbstractRowOpLog* row_oplog = 0; bool found = table_oplog.GetEraseOpLog(row_id, &row_oplog); if (found && row_oplog != 0) { size_t serialized_oplog_size = CountRowOpLogToSend( row_id, row_oplog, &table_num_bytes_by_server_, bg_table_oplog, GetSerializedRowOpLogSize); accum_table_oplog_bytes += serialized_oplog_size; } row_id = table_oplog_meta->GetAndClearNextUptoClock(); } return accum_table_oplog_bytes; }
size_t SSPAggrBgWorker::ReadTableOpLogMetaUpToCapacity( int32_t table_id, ClientTable *table, size_t bytes_accumulated, TableOpLogMeta *table_oplog_meta, GetSerializedRowOpLogSizeFunc GetSerializedRowOpLogSize, BgOpLogPartition *bg_table_oplog) { size_t accum_table_oplog_bytes = bytes_accumulated; AbstractOpLog &table_oplog = table->get_oplog(); table_oplog_meta->Sort(); int32_t row_id; row_id = table_oplog_meta->GetAndClearNextInOrder(); while (row_id >= 0) { AbstractRowOpLog* row_oplog = 0; bool found = table_oplog.GetEraseOpLog(row_id, &row_oplog); if (found && row_oplog != 0) { size_t serialized_oplog_size = CountRowOpLogToSend( row_id, row_oplog, &table_num_bytes_by_server_, bg_table_oplog, GetSerializedRowOpLogSize); accum_table_oplog_bytes += serialized_oplog_size; if (accum_table_oplog_bytes >= GlobalContext::get_oplog_push_upper_bound_kb()*k1_Ki) break; } row_id = table_oplog_meta->GetAndClearNextInOrder(); } return accum_table_oplog_bytes; }
BgOpLogPartition *SSPBgWorker::PrepareOpLogsNormal( int32_t table_id, ClientTable *table) { AbstractOpLog &table_oplog = table->get_oplog(); GetSerializedRowOpLogSizeFunc GetSerializedRowOpLogSize; if (table->oplog_dense_serialized()) { GetSerializedRowOpLogSize = GetDenseSerializedRowOpLogSize; } else { GetSerializedRowOpLogSize = GetSparseSerializedRowOpLogSize; } // Get OpLog index cuckoohash_map<int32_t, bool> *new_table_oplog_index_ptr = table->GetAndResetOpLogIndex(my_comm_channel_idx_); size_t table_update_size = table->get_sample_row()->get_update_size(); BgOpLogPartition *bg_table_oplog = new BgOpLogPartition( table_id, table_update_size, my_comm_channel_idx_); for (const auto &server_id : server_ids_) { // Reset size to 0 table_num_bytes_by_server_[server_id] = 0; } for (auto oplog_index_iter = new_table_oplog_index_ptr->cbegin(); !oplog_index_iter.is_end(); oplog_index_iter++) { int32_t row_id = oplog_index_iter->first; AbstractRowOpLog *row_oplog = 0; bool found = GetRowOpLog(table_oplog, row_id, &row_oplog); if (!found) continue; if (found && (row_oplog == 0)) continue; CountRowOpLogToSend(row_id, row_oplog, &table_num_bytes_by_server_, bg_table_oplog, GetSerializedRowOpLogSize); STATS_BG_ACCUM_TABLE_OPLOG_SENT(table_id, row_id, 1); } delete new_table_oplog_index_ptr; return bg_table_oplog; }
BgOpLogPartition *SSPBgWorker::PrepareOpLogsAppendOnly( int32_t table_id, ClientTable *table) { GetSerializedRowOpLogSizeFunc GetSerializedRowOpLogSize; if (table->oplog_dense_serialized()) { GetSerializedRowOpLogSize = GetDenseSerializedRowOpLogSize; } else { GetSerializedRowOpLogSize = GetSparseSerializedRowOpLogSize; } size_t table_update_size = table->get_sample_row()->get_update_size(); BgOpLogPartition *bg_table_oplog = new BgOpLogPartition( table_id, table_update_size, my_comm_channel_idx_); for (const auto &server_id : server_ids_) { // Reset size to 0 table_num_bytes_by_server_[server_id] = 0; } auto buff_iter = append_only_row_oplog_buffer_map_.find(table_id); if (buff_iter != append_only_row_oplog_buffer_map_.end()) { AppendOnlyRowOpLogBuffer *append_only_row_oplog_buffer = buff_iter->second; append_only_row_oplog_buffer->MergeTmpOpLog(); int32_t row_id; AbstractRowOpLog *row_oplog = append_only_row_oplog_buffer->InitReadRmOpLog(&row_id); while (row_oplog != 0) { CountRowOpLogToSend(row_id, row_oplog, &table_num_bytes_by_server_, bg_table_oplog, GetSerializedRowOpLogSize); STATS_BG_ACCUM_TABLE_OPLOG_SENT(table_id, row_id, 1); row_oplog = append_only_row_oplog_buffer->NextReadRmOpLog(&row_id); } } return bg_table_oplog; }