bool Dictionaries::addFromFile(std::string* filename)
{
IDictionaryParser* parser = new DictionaryParser;
IFileReader* reader = new FileReader;//FIX ME
IFiledataValidator* validator = new FiledataValidator;
std::vector<std::string*>* filedata = reader->Read(filename);
ReturnedData data(filedata);
bool is_Success = validator->is_valid(&data);
if (!is_Success)
{
delete parser;
delete reader;//FIX ME
delete validator;
return false;
}
Dictionaries* dict = parser->pars((filedata));
InsertDict(dict);
delete parser;
delete reader;//FIX ME
delete validator;
return true;
}
bool Dictionaries::initFromFile(std::string* filename)
{
IDictionaryParser* parser = new DictionaryParser;
IFileReader* reader = new FileReader;//FIX ME
IFiledataValidator* validator = new FiledataValidator;
std::vector<std::string*>* filedata = reader->Read(filename);
ReturnedData data(filedata);
bool is_Success = validator->is_valid(&data);
if (!is_Success)
{
delete parser;
delete reader;//FIX ME
delete validator;
return false;
}
Dictionaries* dict = parser->pars((filedata));
this->mDictionaries = dict->mDictionaries;
this->mLanguages = dict->mLanguages;
delete parser;
delete reader;//FIX ME
delete validator;
dict->ClearData();
delete dict;
return true;
}
void MainWindow::on_actionOpen_triggered()
{
if (!promptUnsavedWork()) return;
QString fileName =
QFileDialog::getOpenFileName(this,
tr("Open Canvas"),
QDir::currentPath(),
tr("Json (*.json);;All File Types (*.*)"));
if (fileName.isEmpty()) return;
QString ext = QFileInfo(fileName).suffix();
IFileReader *reader = FileReaderFactory::create(ext.toStdString());
if (reader == nullptr) {
QMessageBox::critical(this,
"Unsupported file type",
QString("I can't read *.%1 files.").arg(ext));
return;
}
Group *readCanvas = new Group();
reader->setup(readCanvas);
bool canRead = reader->read(fileName.toStdString());
delete reader;
if (canRead) {
m_canvas->setMainGroup(readCanvas);
ui->statusBar->showMessage(QString("Opened file: \"%1\"").arg(fileName));
} else {
delete readCanvas;
QMessageBox::critical(this,
"File Read Error",
"I can't read this file ):");
return;
}
setCanvasFile(fileName);
resetCommandStack();
m_canvas->repaint();
}
//------------------------------------------------------------------------
bool CFileReadSLK::LoadFromFile( const char* pFileName )
//------------------------------------------------------------------------
{
assert( NULL == m_data );
assert( NULL == m_column );
//OutputDebugString(pFileName);
//OutputDebugString("\n==================================\n");
IFileReader* fileReader = NULL;
if( !CreateFileReader( &fileReader ) )
return false;
bool isLoadOk = true;
for( unsigned int i = 0; i < _TSLKValueArray.size(); i++ )
{
_TSLKValue* p = _TSLKValueArray[i];
SAFE_DELETE( p );
}
_TSLKValueArray.clear();
char* buffer = NULL;
uInt32 bufferSize = 0;
if(fileReader)
{
if( !(fileReader->OpenFile( pFileName )) )
{
isLoadOk = false;
goto OPEN_FILE_FAIL;
}
bufferSize = fileReader->GetSize();
assert( bufferSize > 0 );
buffer = new char[bufferSize+1];
memset( buffer, 0, (bufferSize + 1) * sizeof(char) );
char tmpBuffer[1024];
memset( tmpBuffer, 0, 1024 * sizeof(char) );
char* ptrBuffer = buffer;
size_t size = fileReader->Read( (void*)tmpBuffer, 1024 );
assert( "出现异常错误！" && size <= 1024 );
size_t count = 0;
while( size > 0 )
{
memcpy( ptrBuffer, tmpBuffer, size );
ptrBuffer += size;
count += size;
memset( tmpBuffer, 0, 1024 * sizeof(char) );
size = fileReader->Read( tmpBuffer, 1024 );
assert( "出现异常错误！" && size <= 1024 );
}
assert( "文件大小错误！" && bufferSize == count );
if( bufferSize != count )
isLoadOk = false;
else if( !GetInfoFromMemory( buffer, bufferSize ) )
isLoadOk = false;
SAFE_DELETE_ARRAY( buffer );
}
OPEN_FILE_FAIL:
if(fileReader)
fileReader->Close();
SAFE_DELETE( fileReader );
for( unsigned int i = 0; i < _TSLKValueArray.size(); i++ )
{
_TSLKValue* p = _TSLKValueArray[i];
SAFE_DELETE( p );
}
_TSLKValueArray.clear();
return isLoadOk;
}
static void png_read_callback(png_structp png, png_bytep data, png_size_t size) {
IFileReader* file = (IFileReader*)png_get_io_ptr( png );
if ( file->ReadByte( data, (int)size ) == false ) {
LOG_ERROR( "Error while reading PNG file" );
}
}
void multi_thread_process_loop(vector<string> files1,
vector<string> files2) {
MultithreadData mtdata(threads);
list<pthread_t> satellite_threads;
pthread_t writer_thread;
if (files1.size() == 0) return;
for (size_t i = 0; i < threads; ++i) {
pthread_t id;
if (pthread_create(&id, NULL, satellite_process_consumer_thread,
reinterpret_cast<void*>(&mtdata))) {
PrintMessageDieOnError("Failed to create threads", ERROR);
}
satellite_threads.push_back(id);
}
if (pthread_create(&writer_thread, NULL, output_writer_thread,
reinterpret_cast<void*>(&mtdata))) {
PrintMessageDieOnError("Failed to create output writer threads", ERROR);
}
size_t counter = 1;
std::string file1;
std::string file2;
for (size_t i = 0; i < files1.size(); i++) {
file1 = files1.at(i);
if (paired && !bam) {
file2 = files2.at(i);
PrintMessageDieOnError("Processing files " + file1 + " and " + file2, PROGRESS);
if (using_s3) {
const std::string s3cmd1 = GenerateS3Command(s3bucket,
file1,
s3cmd_configfile);
const std::string s3cmd2 = GenerateS3Command(s3bucket,
file2,
s3cmd_configfile);
if (s3debug) {
PrintMessageDieOnError("S3 debug: " + s3cmd1, PROGRESS);
PrintMessageDieOnError("S3 debug: " + s3cmd2, PROGRESS);
} else {
if (system(s3cmd1.c_str()) != 0) {
PrintMessageDieOnError("Problem fetching file1 from S3", ERROR);
}
if (system(s3cmd2.c_str()) != 0) {
PrintMessageDieOnError("Problem fetching file2 from S3", ERROR);
}
}
file1 = "/mnt/lobstr/"+file1;
file2 = "/mnt/lobstr/"+file2;
}
if (!(fexists(file1.c_str()) && fexists(file2.c_str()))) {
PrintMessageDieOnError("File " + file1 + " or " + file2 + " does not exist", WARNING);
continue;
}
} else {
PrintMessageDieOnError("Processing file " + file1, PROGRESS);
if (using_s3) {
const std::string s3cmd = GenerateS3Command(s3bucket,
file1,
s3cmd_configfile);
if (s3debug) {
PrintMessageDieOnError("S3 debug: " + s3cmd, PROGRESS);
} else {
if (system(s3cmd.c_str()) != 0) {
PrintMessageDieOnError("Problem fetching file from S3", ERROR);
}
}
file1 = "/mnt/lobstr/"+file1;
file2 = "/mnt/lobstr/"+file2;
}
if (!fexists(file1.c_str())) {
PrintMessageDieOnError("File " + file1 + " or " + file2 + " does not exist", WARNING);
continue;
}
}
IFileReader *pReader = create_file_reader(file1, file2);
do {
ReadPair *pRecord = new ReadPair;
pRecord->read_count = counter;
if (counter % READPROGRESS == 0) {
stringstream msg;
msg << "Processed " << counter << ' ' << unit_name;
PrintMessageDieOnError(msg.str(), PROGRESS);
}
if (!pReader->GetNextRecord(pRecord))
break; // no more reads
counter++;
mtdata.increment_input_counter();
mtdata.post_new_input_read(pRecord);
pRecord = NULL; // the consumers will take it from here, and free it
} while (1);
delete pReader;
if (using_s3) {
string rmcmd = "rm " + file1;
if (paired && !bam) {
rmcmd += "; rm ";
rmcmd += file2;
}
if (s3debug) {
PrintMessageDieOnError("S3 debug: " + rmcmd, PROGRESS);
} else {
if (system(rmcmd.c_str()) != 0) {
PrintMessageDieOnError("Problem deleting file", ERROR);
}
}
}
}
run_info.num_processed_units = counter;
#ifdef DEBUG_THREADS
PrintMessageDieOnError("No more input, waiting for alignment threads completion", PROGRESS);
#endif
//Send a 'poison pill' to the alignment threads
for (size_t i = 0; i < threads; ++i)
mtdata.post_new_input_read(NULL);
for (list<pthread_t>::const_iterator it = satellite_threads.begin();
it != satellite_threads.end(); ++it) {
int i = pthread_join(*it,NULL);
if (i != 0) {
stringstream msg;
msg << "Failed to join alignment thread " << (*it) <<
"error code = " << i ;
PrintMessageDieOnError(msg.str(), WARNING);
}
}
//Send a 'poison pill' to the writer thread
#ifdef DEBUG_THREADS
PrintMessageDieOnError("waiting for writer thread completion", PROGRESS);
#endif
mtdata.post_new_output_read(NULL);
int i = pthread_join(writer_thread,NULL);
if (i != 0) {
stringstream msg;
msg << "Failed to join writer thread " << (writer_thread) <<
"error code = " << i ;
PrintMessageDieOnError(msg.str(), WARNING);
}
#ifdef DEBUG_THREADS
PrintMessageDieOnError("All thread terminated.", PROGRESS);
#endif
}
/*
* process read in single thread
*/
void single_thread_process_loop(const vector<string>& files1,
const vector<string>& files2) {
ReadPair read_pair;
SamFileWriter samWriter(output_prefix + ".aligned.bam", chrom_sizes);
STRDetector *pDetector = new STRDetector();
BWAReadAligner *pAligner = new BWAReadAligner(&bwt_references,
&bnt_annotations,
&ref_sequences, opts);
std::string file1;
std::string file2;
size_t num_reads_processed = 0;
for (size_t i = 0; i < files1.size(); i++) {
file1 = files1.at(i);
if (paired && !bam) {
file2 = files2.at(i);
PrintMessageDieOnError("Processing files " + file1 + " and " + file2, PROGRESS);
if (using_s3) {
const std::string s3cmd1 = GenerateS3Command(s3bucket,
file1,
s3cmd_configfile);
const std::string s3cmd2 = GenerateS3Command(s3bucket,
file2,
s3cmd_configfile);
if (s3debug) {
PrintMessageDieOnError("S3 debug: " + s3cmd1, PROGRESS);
PrintMessageDieOnError("S3 debug: " + s3cmd2, PROGRESS);
} else {
if (system(s3cmd1.c_str()) != 0) {
PrintMessageDieOnError("Problem fetching file1 from S3", ERROR);
}
if (system(s3cmd2.c_str()) != 0) {
PrintMessageDieOnError("Problem fetching file2 from S3", ERROR);
}
}
file1 = "/mnt/lobstr/"+file1;
file2 = "/mnt/lobstr/"+file2;
}
if (!(fexists(file1.c_str()) && fexists(file2.c_str()))) {
PrintMessageDieOnError("File " + file1 + " or " + file2 + " does not exist", WARNING);
continue;
}
} else {
PrintMessageDieOnError("Processing file " + file1, PROGRESS);
if (using_s3) {
const std::string s3cmd = GenerateS3Command(s3bucket,
file1,
s3cmd_configfile);
if (s3debug) {
PrintMessageDieOnError("S3 debug: " + s3cmd, PROGRESS);
} else {
if (system(s3cmd.c_str()) != 0) {
PrintMessageDieOnError("Problem fetching file from S3", ERROR);
}
}
file1 = "/mnt/lobstr/"+file1;
}
if (!fexists(file1.c_str())) {
PrintMessageDieOnError("File " + file1 + " does not exist", WARNING);
continue;
}
}
IFileReader* pReader = create_file_reader(file1, file2);
int aligned = false;
std::string repseq = "";
while (pReader->GetNextRecord(&read_pair)) {
aligned = false;
num_reads_processed += 1;
if (num_reads_processed % READPROGRESS == 0) {
stringstream msg;
msg << "Processed " << num_reads_processed << ' ' << unit_name;
PrintMessageDieOnError(msg.str(), PROGRESS);
}
read_pair.read_count = num_reads_processed;
// reset fields
read_pair.reads.at(0).repseq = "";
read_pair.reads.at(0).ms_repeat_best_period = 0;
read_pair.reads.at(0).ms_repeat_next_best_period = 0;
if (read_pair.reads.at(0).paired) {
read_pair.reads.at(1).repseq = "";
read_pair.reads.at(1).ms_repeat_best_period = 0;
read_pair.reads.at(1).ms_repeat_next_best_period = 0;
}
// Check read length
if (!(read_pair.reads.at(0).nucleotides.length() >= min_read_length) &&
(read_pair.reads.at(0).nucleotides.length() <= max_read_length)) {
continue;
}
if (read_pair.reads.at(0).paired) {
if (!(read_pair.reads.at(1).nucleotides.length() >= min_read_length) &&
(read_pair.reads.at(1).nucleotides.length() <= max_read_length)) {
continue;
}
}
bases += read_pair.reads.at(0).nucleotides.length();
if (read_pair.reads.at(0).paired) bases += read_pair.reads.at(1).nucleotides.length();
// STEP 1: Sensing
string det_err, det_messages;
if (!pDetector->ProcessReadPair(&read_pair, &det_err, &det_messages)) {
if (debug) {
PrintMessageDieOnError(GetReadDebug(read_pair, det_err, det_messages, "NA", "NA") + " (detection-fail)", DEBUG);
}
continue;
}
// STEP 2: Alignment
string aln_err, aln_messages;
if (pAligner->ProcessReadPair(&read_pair, &aln_err, &aln_messages)) {
aligned = true;
if (debug) { // if aligned, what was the repseq we aligned to
PrintMessageDieOnError(GetReadDebug(read_pair, det_err, det_messages, aln_err, aln_messages)+ " (aligned-round-1)", DEBUG);
}
} else {
read_pair.read1_passed_detection = false;
read_pair.read2_passed_detection = false;
// Try second best period for each read
if (read_pair.reads.at(0).ms_repeat_next_best_period != 0) {
read_pair.reads.at(0).ms_repeat_best_period =
read_pair.reads.at(0).ms_repeat_next_best_period;
string err, second_best_repseq;
if (getMSSeq(read_pair.reads.at(0).detected_ms_region_nuc,
read_pair.reads.at(0).ms_repeat_best_period, &repseq, &second_best_repseq, &err)) {
read_pair.reads.at(0).repseq = repseq;
read_pair.read1_passed_detection = true;
}
}
if (read_pair.reads.at(0).paired) {
if (read_pair.reads.at(1).ms_repeat_next_best_period != 0) {
read_pair.reads.at(1).ms_repeat_best_period =
read_pair.reads.at(1).ms_repeat_next_best_period;
string err, second_best_repseq;
if (getMSSeq(read_pair.reads.at(1).detected_ms_region_nuc,
read_pair.reads.at(1).ms_repeat_best_period,
&repseq, &second_best_repseq, &err)) {
read_pair.reads.at(1).repseq = repseq;
read_pair.read2_passed_detection = true;
}
}
}
if (read_pair.read1_passed_detection ||
read_pair.read2_passed_detection) {
if (pAligner->ProcessReadPair(&read_pair, &aln_err, &aln_messages)) {
aligned = true;
if (debug) { // if aligned, what was the repseq we aligned to
PrintMessageDieOnError(GetReadDebug(read_pair, det_err, det_messages, aln_err, aln_messages)+ " (aligned-round-2)", DEBUG);
}
}
}
}
if (aligned) {
samWriter.WriteRecord(read_pair);
} else {
if (debug) { // if didn't align, print this
PrintMessageDieOnError(GetReadDebug(read_pair, det_err, det_messages, aln_err, aln_messages)+ " (not-aligned)", DEBUG);
}
}
}
delete pReader;
stringstream msg;
msg << "Processed " << num_reads_processed << ' ' << unit_name;
PrintMessageDieOnError(msg.str(), PROGRESS);
if (using_s3) {
string rmcmd = "rm " + file1;
if (paired && !bam) {
rmcmd += "; rm ";
rmcmd += file2;
}
if (s3debug) {
PrintMessageDieOnError("S3 debug: " + rmcmd, PROGRESS);
} else {
if (system(rmcmd.c_str()) != 0) {
PrintMessageDieOnError("Problem deleting file", ERROR);
}
}
}
}
delete pDetector;
delete pAligner;
run_info.num_processed_units = num_reads_processed;
}