TEST(cursor,sdbCursor_get_over_then_close)
{
INT32 rc = SDB_OK ;
rc = initEnv( HOST, SERVER, USER, PASSWD ) ;
ASSERT_EQ( SDB_OK, rc ) ;
sdbConnectionHandle connection = 0 ;
sdbCollectionHandle collection = 0 ;
sdbCursorHandle cursor = 0 ;
INT32 NUM = 1 ;
bson obj ;
bson obj1 ;
bson obj2 ;
rc = sdbConnect ( HOST, SERVER, USER, PASSWD, &connection ) ;
ASSERT_EQ( SDB_OK, rc ) ;
rc = getCollection ( connection,
COLLECTION_FULL_NAME,
&collection ) ;
ASSERT_EQ( SDB_OK, rc ) ;
insertRecords ( collection, NUM ) ;
rc = sdbQuery( collection, NULL, NULL, NULL, NULL, 0, -1, &cursor ) ;
ASSERT_EQ( SDB_OK, rc ) ;
bson_init(&obj);
bson_init(&obj1);
bson_init(&obj2);
rc = sdbNext ( cursor, &obj ) ;
CHECK_MSG("%s%d\n","rc = ",rc) ;
ASSERT_EQ( SDB_OK, rc ) ;
printf( "SdbNext record is:\n" ) ;
bson_print( &obj ) ;
printf("\n") ;
rc = sdbNext ( cursor, &obj1 ) ;
CHECK_MSG("%s%d\n","rc = ",rc) ;
ASSERT_EQ( SDB_DMS_EOC, rc ) ;
rc = sdbNext ( cursor, &obj1 ) ;
CHECK_MSG("%s%d\n","rc = ",rc) ;
ASSERT_EQ( SDB_DMS_CONTEXT_IS_CLOSE, rc ) ;
bson_destroy( &obj ) ;
bson_destroy( &obj1 ) ;
bson_destroy( &obj2 ) ;
rc = sdbCloseCursor( cursor ) ;
CHECK_MSG("%s%d\n","rc = ",rc) ;
ASSERT_EQ( SDB_OK, rc ) ;
sdbDisconnect ( connection ) ;
sdbReleaseCursor ( cursor ) ;
sdbReleaseCollection ( collection ) ;
sdbReleaseConnection ( connection ) ;
}
TEST(cursor,sdbNext)
{
INT32 rc = SDB_OK ;
rc = initEnv( HOST, SERVER, USER, PASSWD ) ;
ASSERT_EQ( SDB_OK, rc ) ;
sdbConnectionHandle connection = 0 ;
sdbCollectionHandle collection = 0 ;
sdbCursorHandle cursor = 0 ;
INT32 NUM = 10 ;
SINT64 count = 0 ;
bson obj ;
rc = sdbConnect ( HOST, SERVER, USER, PASSWD, &connection ) ;
ASSERT_EQ( SDB_OK, rc ) ;
rc = getCollection ( connection,
COLLECTION_FULL_NAME,
&collection ) ;
ASSERT_EQ( SDB_OK, rc ) ;
insertRecords( collection, NUM ) ;
rc = sdbQuery ( collection, NULL, NULL, NULL, NULL, 0, -1, &cursor ) ;
ASSERT_EQ( SDB_OK, rc ) ;
bson_init(&obj);
rc = sdbCurrent( cursor, &obj ) ;
printf( "Current record is:\n" ) ;
bson_print( &obj ) ;
printf("\n") ;
bson_destroy( &obj ) ;
bson_init(&obj);
rc = sdbNext( cursor, &obj ) ;
ASSERT_EQ( SDB_OK, rc ) ;
printf( "Next record is:\n" ) ;
bson_print( &obj ) ;
printf("\n") ;
bson_destroy(&obj);
rc = sdbGetCount( collection, NULL, &count ) ;
CHECK_MSG("%s%d\n","rc = ",rc) ;
CHECK_MSG("%s%d%s%lld\n", "NUM = ", NUM, " count = ", count) ;
ASSERT_EQ ( NUM, count ) ;
sdbDisconnect ( connection ) ;
sdbReleaseCursor ( cursor ) ;
sdbReleaseCollection ( collection ) ;
sdbReleaseConnection ( connection ) ;
}
TEST(cursor,sdbCloseCursor)
{
INT32 rc = SDB_OK ;
rc = initEnv( HOST, SERVER, USER, PASSWD ) ;
ASSERT_EQ( SDB_OK, rc ) ;
sdbConnectionHandle connection = 0 ;
sdbCollectionHandle collection = 0 ;
sdbCursorHandle cursor = 0 ;
INT32 NUM = 10 ;
bson obj ;
bson obj1 ;
rc = sdbConnect ( HOST, SERVER, USER, PASSWD, &connection ) ;
ASSERT_EQ( SDB_OK, rc ) ;
rc = getCollection ( connection,
COLLECTION_FULL_NAME,
&collection ) ;
ASSERT_EQ( SDB_OK, rc ) ;
insertRecords ( collection, NUM ) ;
rc = sdbQuery( collection, NULL, NULL, NULL, NULL, 0, -1, &cursor ) ;
bson_init(&obj);
rc = sdbCurrent( cursor, &obj ) ;
ASSERT_EQ( SDB_OK, rc ) ;
printf( "Current record is:\n" ) ;
bson_print( &obj ) ;
printf("\n") ;
bson_destroy( &obj ) ;
rc = sdbCloseCursor( cursor ) ;
ASSERT_EQ( SDB_OK, rc ) ;
bson_init( &obj1 ) ;
rc = sdbCurrent( cursor, &obj1 ) ;
ASSERT_EQ( rc, SDB_DMS_CONTEXT_IS_CLOSE ) ;
sdbDisconnect ( connection ) ;
sdbReleaseCursor ( cursor ) ;
sdbReleaseCollection ( collection ) ;
sdbReleaseConnection ( connection ) ;
}
bool HdfsLoader::append(std::string tuple_string)
{
#ifdef HDFS_LOAD
if(Config::local_disk_mode) {
connector_ = new LocalDiskConnector(writepath);
}
else {
connector_ = new HdfsConnector(writepath);
}
connector_->openFiles(open_flag_);
#endif
cout << "\n\n\n--------------------------Insert Begin!--------------------------\n";
while (tuple_string.length() != 0)
{
int current = 0;
s_record.clear();
while (current < tuple_string.length() && tuple_string[current] != '\n')
s_record += tuple_string[current++];
current++;
if (current < tuple_string.length())
tuple_string = tuple_string.substr(current, tuple_string.length()-current);
else
tuple_string = "\0";
insertRecords();
}
//flush the last block which is not full of 64*1024Byte
for(int i = 0; i < table_descriptor_->getNumberOfProjection(); i++)
{
for(int j = 0; j < table_descriptor_->getProjectoin(i)->getPartitioner()->getNumberOfPartitions(); j++)
{
if(!pj_buffer[i][j]->Empty())
{
pj_buffer[i][j]->serialize(*sblock);
#ifdef HDFS_LOAD
tSize num_written_every64k_bytes = connector_->flush(i, j, (void*)sblock->getBlock(), sblock->getsize());
// hdfsWrite(connector_->get_fs(), connector_->get_writefile()[i][j], (void*)sblock->getBlock(), sblock->getsize());
#endif
cout << row_id << "The last block has written to HDFS!\n";
blocks_per_partition[i][j]++;
pj_buffer[i][j]->setEmpty();
}
}
}
#ifdef HDFS_LOAD
if(connector_->closeFiles() == false)
{
cout << "Failed to close HDFS." << endl;
exit(-1);
}else
cout << "HDFS close successfully." << endl;
#endif
//register the number of rows in table to catalog
table_descriptor_->setRowNumber(row_id);
//register the partition information to catalog
for(int i = 0; i < table_descriptor_->getNumberOfProjection(); i++)
{
for(int j = 0; j < table_descriptor_->getProjectoin(i)->getPartitioner()->getNumberOfPartitions(); j++)
{
printf("Table %d, projection %d ::Partition info:%lx\n", i, j, table_descriptor_->getProjectoin(i)->getPartitioner());
table_descriptor_->getProjectoin(i)->getPartitioner()->RegisterPartitionWithNumberOfBlocks(j, blocks_per_partition[i][j]);
table_descriptor_->getProjectoin(i)->getPartitioner()->UpdatePartitionWithNumberOfChunksToBlockManager(j, blocks_per_partition[i][j]);
cout << "Number of blocks " << i << "\t" << j << "\t: " << blocks_per_partition[i][j] << endl;
}
}
cout << "\n\n\n--------------------------Insert End!--------------------------\n";
return true;
}
bool HdfsLoader::load(double sample_rate){
#ifdef HDFS_LOAD
if(Config::local_disk_mode) {
connector_ = new LocalDiskConnector(writepath);
}
else {
connector_ = new HdfsConnector(writepath);
}
connector_->openFiles(open_flag_);
#endif
if (open_flag_ == CREATE)
{
//TODO: for reloading after select, unbinding the original data
if (table_descriptor_->getProjectoin(0)->getPartitioner()->allPartitionBound())
{
for(int i = 0; i < table_descriptor_->getNumberOfProjection(); i++)
{
for(int j = 0; j < table_descriptor_->getProjectoin(i)->getPartitioner()->getNumberOfPartitions(); j++)
{
Catalog::getInstance()->getBindingModele()->UnbindingEntireProjection(table_descriptor_->getProjectoin(i)->getPartitioner());
}
}
}
cout << "\n\n\n--------------------------Load Begin!--------------------------\n";
}
else
cout << "\n\n\n--------------------------Append Begin!--------------------------\n";
vector<string>::iterator iter;
/*for testing*/ int t_count = 1;
for (iter = file_list.begin(); iter != file_list.end(); iter++)
{
ifstream InFile((*iter).c_str());
if(!InFile.good()){
printf("Cannot open source file:%s , reason: %s\n",(*iter).c_str(),strerror(errno));
return false;
}
while(!InFile.eof()/* for testing && t_count++ < 500 */)
{
s_record.clear();
getline(InFile,s_record,row_separator);
if((double)rand()/RAND_MAX>sample_rate)
continue;
///*for testing*/ if(row_id/t_count > 100000)
///*for testing*/ break;
//TODO(yukai): get the return value and handle it
insertRecords();
}
InFile.close();
/*for testing*/ t_count++;
}
cout << "----------------------------"<<endl;
//flush the last block which is not full of 64*1024Byte
for(int i = 0; i < table_descriptor_->getNumberOfProjection(); i++)
{
for(int j = 0; j < table_descriptor_->getProjectoin(i)->getPartitioner()->getNumberOfPartitions(); j++)
{
if(!pj_buffer[i][j]->Empty())
{
pj_buffer[i][j]->serialize(*sblock);
#ifdef HDFS_LOAD
tSize num_written_every64k_bytes = connector_->flush(i, j, sblock->getBlock(), sblock->getsize());
//hdfsWrite(connector_->get_fs(), connector_->get_writefile()[i][j], (void*)sblock->getBlock(), sblock->getsize());
#endif
cout << row_id << "\tThe last block has written to HDFS!\n";
blocks_per_partition[i][j]++;
pj_buffer[i][j]->setEmpty();
}
}
}
///*for testing begin*/
// for(int i = 0; i < table_descriptor_->getNumberOfProjection(); i++)
// {
// for(int j = 0; j < table_descriptor_->getProjectoin(i)->getPartitioner()->getNumberOfPartitions(); j++)
// {
// cout << blocks_per_partition[i][j] << " ";
// }
// cout << endl;
// }
///*for testing end*/
#ifdef HDFS_LOAD
if(connector_->closeFiles() == false)
{
cout << "Failed to close HDFS." << endl;
exit(-1);
}else
cout << "HDFS close successfully." << endl;
#endif
//register the number of rows in table to catalog
table_descriptor_->setRowNumber(row_id);
//register the partition information to catalog
for(int i = 0; i < table_descriptor_->getNumberOfProjection(); i++)
{
for(int j = 0; j < table_descriptor_->getProjectoin(i)->getPartitioner()->getNumberOfPartitions(); j++)
{
// table_descriptor_->getProjectoin(i)->getPartitioner()->RegisterPartitionWithNumberOfBlocks(j, blocks_per_partition[i][j],tuples_per_partition[i][j]);
table_descriptor_->getProjectoin(i)->getPartitioner()->RegisterPartitionWithNumberOfBlocks(j, blocks_per_partition[i][j]);
if (open_flag_ == APPEND)
{
table_descriptor_->getProjectoin(i)->getPartitioner()->UpdatePartitionWithNumberOfChunksToBlockManager(j, blocks_per_partition[i][j]);
}
cout << "Number of blocks " << i << "\t" << j << "\t: " << blocks_per_partition[i][j] << endl;
}
}
if (open_flag_ == CREATE)
cout << "\n\n\n--------------------------Load End!--------------------------\n";
else
cout << "\n\n\n--------------------------Append End!--------------------------\n";
return true;
}
void dlgcd_refresh2(void *mp1, void *mp2)
{
int i;
HWND hwnd = (HWND) mp1;
CDTRACKRECORD *record;
CDTRACKRECORD *firstRecord, *posRecord;
RECORDINSERT recordInfo;
char buffer[256];
char buffer2[256];
USHORT bitrate = getBitRate();
if(!cdDrive)
return;
if(cddbSocket)
{
setText(hwnd,EF_ARTIST,cddbSocket->get_disc_title(0));
setText(hwnd,EF_TITLE,cddbSocket->get_disc_title(1));
setText(hwnd,EF_COMMENTS,cddbSocket->get_disc_title(2));
setText(hwnd,EF_GENRE,queryData.category);
}
/* delete all current records */
record = (CDTRACKRECORD *) enumRecords(hwnd, CT_TRACK, NULL, CMA_FIRST);
while(record && record != (CDTRACKRECORD *) -1)
{
free(record->record.pszIcon);
record = (CDTRACKRECORD *) enumRecords(hwnd, CT_TRACK, (RECORDCORE *) record, CMA_NEXT);
}
removeRecords(hwnd, CT_TRACK, NULL, 0);
/* allocate and fill container with CD Track Info */
firstRecord = posRecord = (CDTRACKRECORD *)
allocaRecords(hwnd, CT_TRACK, cdDrive->getCount(), sizeof(CDTRACKRECORD) - sizeof(RECORDCORE));
for (i = 0; i < cdDrive->getCount(); i++)
{
UCHAR track = cdDrive->getCDInfo()->firstTrack+i;
/* setting kludges and other weird things */
posRecord->timepointer = posRecord->time;
posRecord->typepointer = posRecord->type;
posRecord->titleptr = posRecord->title;
posRecord->artistptr = posRecord->artist;
posRecord->albumptr = posRecord->album;
posRecord->yearptr = posRecord->year;
posRecord->genreptr = posRecord->genre;
posRecord->commentptr = posRecord->comment;
posRecord->record.hptrMiniIcon = 0; /* ask God */
/* loading info fields */
if(cdDrive->getTrackInfo(track)->data)
{
strcat(posRecord->time,"N/A");
strcat(posRecord->type,"Data");
posRecord->mp3size = 0;
posRecord->record.hptrIcon = dataIco;
}
else
{
posRecord->minutes = cdDrive->getTrackInfo(track)->length.minute;
posRecord->seconds = cdDrive->getTrackInfo(track)->length.second;
sprintf(posRecord->time,"%d:%.2d",cdDrive->getTrackInfo(track)->length.minute,
cdDrive->getTrackInfo(track)->length.second);
strcat(posRecord->type,"Audio");
if(bitrate)
posRecord->mp3size = cdDrive->getTrackInfo(track)->size /
((cdDrive->getTrackInfo(track)->channels*2*44100) /
((double) bitrate * 1000/8)) + 0.5;
else
posRecord->mp3size = 0;
posRecord->record.hptrIcon = trackIco;
}
posRecord->track = cdDrive->getTrackInfo(track)->number;
posRecord->size = cdDrive->getTrackInfo(track)->size;
posRecord->channels = cdDrive->getTrackInfo(track)->channels;
posRecord->data = cdDrive->getTrackInfo(track)->data;
if(cddbSocket)
{
strncpy(posRecord->title, cddbSocket->get_track_title(i,0), sizeof(posRecord->title)-1);
strncpy(posRecord->artist, cddbSocket->get_disc_title(0), sizeof(posRecord->artist)-1);
strncpy(posRecord->album, cddbSocket->get_disc_title(1), sizeof(posRecord->album)-1);
strncpy(posRecord->comment, cddbSocket->get_track_title(i,1), sizeof(posRecord->comment)-1);
strncpy(posRecord->genre, queryData.category, sizeof(posRecord->genre)-1);
// cddbSocket->get_disc_title(2)
getFilenameFormat(buffer2,sizeof(buffer2));
}
else
{
getText(hwnd,EF_ARTIST,buffer,sizeof(buffer));
strncpy(posRecord->artist, buffer, sizeof(posRecord->artist)-1);
getText(hwnd,EF_TITLE,buffer,sizeof(buffer));
strncpy(posRecord->album, buffer, sizeof(posRecord->album)-1);
getText(hwnd,EF_COMMENTS,buffer,sizeof(buffer));
strncpy(posRecord->comment, buffer, sizeof(posRecord->comment)-1);
getText(hwnd,EF_GENRE,buffer,sizeof(buffer));
strncpy(posRecord->genre, buffer, sizeof(posRecord->genre)-1);
getFallbackFF(buffer2, sizeof(buffer2));
}
formatFilename(buffer,sizeof(buffer),buffer2,posRecord,hwnd);
posRecord->record.pszIcon = posRecord->record.pszText =
posRecord->record.pszName = posRecord->record.pszTree =
(char*) malloc(strlen(buffer)+1);
strcpy(posRecord->record.pszIcon,makeValidLFN(buffer));
/* "next!" */
posRecord = (CDTRACKRECORD *) posRecord->record.preccNextRecord;
}
recordInfo.cb = sizeof(recordInfo);
recordInfo.pRecordOrder = (RECORDCORE *) CMA_END;
recordInfo.pRecordParent = NULL;
recordInfo.fInvalidateRecord = TRUE;
recordInfo.zOrder = CMA_TOP;
recordInfo.cRecordsInsert = cdDrive->getCount();
insertRecords(hwnd, CT_TRACK, (RECORDCORE *) firstRecord, &recordInfo);
delete cdDrive; cdDrive = NULL;
delete cddbSocket; cddbSocket = NULL;
}
