int main() { const int rounds = 10000; std::vector<FlowTests::FlowRecord> records(1000); boost::random::mt19937 gen; for(int i=0; i < 1000; i++) records[i].Fill(gen); FlowTests::FlowRecord f; f.Fill(gen); int maxVal; boost::chrono::steady_clock::time_point start = boost::chrono::steady_clock::now(); for(int i=0; i < rounds; i++) { boost::lockfree::queue<int> q(records.size()); for(auto iter = records.begin(); iter != records.end(); ++iter) { q.push(iter->someInfo()); } while(!q.empty()) { int v; if(q.pop(v) && maxVal < v) maxVal = v; } } boost::chrono::duration<double> sec = boost::chrono::steady_clock::now() - start; std::cout << "running " << rounds << " times of 1000 records took " << sec.count() << " seconds and it resulted in " << maxVal << std::endl; return 0; }
void CSVWorld::ScriptEdit::dropEvent (QDropEvent* event) { const CSMWorld::TableMimeData* mime = dynamic_cast<const CSMWorld::TableMimeData*> (event->mimeData()); if (!mime) // May happen when non-records (e.g. plain text) are dragged and dropped { QPlainTextEdit::dropEvent(event); return; } setTextCursor (cursorForPosition (event->pos())); if (mime->fromDocument (mDocument)) { std::vector<CSMWorld::UniversalId> records (mime->getData()); for (std::vector<CSMWorld::UniversalId>::iterator it = records.begin(); it != records.end(); ++it) { if (mAllowedTypes.contains (it->getType())) { if (stringNeedsQuote(it->getId())) { insertPlainText(QString::fromUtf8 (('"' + it->getId() + '"').c_str())); } else { insertPlainText(QString::fromUtf8 (it->getId().c_str())); } } } } }
std::vector<CountH5Record> count(const std::string chr, const std::string cell_type, const unsigned int bin_size, const size_t length, const unsigned int extension, const char strand, const std::string bam_file) { const std::string bam_file_name = file_name_from_path(bam_file); int bins = std::ceil(length / (double) bin_size); int max_base_pair = bins * bin_size; const std::vector<double> bin_counts = liquidate(bam_file, chr, 0, max_base_pair, strand, bins, extension); CountH5Record record; record.bin_number = 0; strncpy(record.cell_type, cell_type.c_str(), sizeof(CountH5Record::cell_type)); strncpy(record.chromosome, chr.c_str(), sizeof(CountH5Record::chromosome)); strncpy(record.file_name, bam_file_name.c_str(), sizeof(CountH5Record::file_name)); std::vector<CountH5Record> records(bin_counts.size(), record); for (size_t bin=0; bin < bin_counts.size(); ++bin) { records[bin].bin_number = bin; records[bin].count = bin_counts[bin]; } return records; }
void WifiDialog::appendStoredAPs(WifiProfiles & list) { sys::SystemConfig conf; WifiProfiles stored_aps = records(conf); for(int i = 0; i < stored_aps.size(); i++) { if (!stored_aps[i].ssid().isEmpty()) { bool found = false; foreach(WifiProfile profile, list) { if (stored_aps[i].ssid() == profile.ssid()) { found = true; break; } } if (!found) { stored_aps[i].setPresent(false); ODataPtr d(new OData(stored_aps[i])); datas_.push_back(d); } } }
void DlgRecord::setData(QHostAddress saddr, quint16 port, quint16 vodport) { if(_pctlc == NULL) { if(vodport == 0) { _vodport = 9001; //default port } else { _vodport = vodport; } _pctlc = new ControlClient(saddr, port); connect(_pctlc, SIGNAL(routeOut(QString)), this, SLOT(records(QString))); _vodprefix = "http://" + saddr.toString() + ":" + QString::number(_vodport); } }
vector<double> extended_pred_symbol::getTimedAchievers(Environment * f,const proposition * prop) const { vector<double> tms; if(records()->get(f,prop)) { tms.push_back(0); }; for(map<double,PropStore *>::const_iterator i = timedInitials.begin();i != timedInitials.end();++i) { if(i->second->get(f,prop)) { tms.push_back(i->first); }; }; return tms; };
QList<IoslotValueRecord> IoslotValueTable::median(const QDateTime &from, const QDateTime &to, int limit) { int count = recordCount(from, to); qint64 w1 = from.toMSecsSinceEpoch() - to.toMSecsSinceEpoch(); qint64 w2 = double(limit) / double(count) * w1; qint16 dw = (w1 - w2) / 2; QDateTime dt1 = from; QDateTime dt2 = to; if (dw > 0) { dt1 = QDateTime::fromMSecsSinceEpoch(dt1.toMSecsSinceEpoch() + dw); dt2 = QDateTime::fromMSecsSinceEpoch(dt2.toMSecsSinceEpoch() - dw); } return records(dt1, dt2, limit); }
json QueryRunner::get_result_as_json() { json result(new json_object()); result["id"] = new json_integer(id); result["query_num"] = new json_integer(query_num); size_t success_count = 0; json records(new json_array()); for(size_t i = 0; i < query_num; ++i ) { if ( results[i].err_code == 0 ) { records.add( new json_float( results[i].query_time.elapsed_time_msec()) ); ++success_count; } } result["success_num"] = new json_integer(success_count); result["error_num"] = new json_integer(query_num-success_count); result["records"] = records; return result; }
int menu() { records(); //leitura do arquivo onde estao as pontuacoes. int opcao,i; printf("********************************************************************************\n"); printf("\t ** * * * ** * * ** *** * * * * * * \n "); printf("\t* * * ** * * * * * * * * * * * * * \n"); printf("\t* * * * * ** * * * * * * * * * * * \n"); printf("\t ** * * * * ** * ** * * * * * * * \n"); printf("********************************************************************************\n"); printf("\n\t\t\t Menu:\t \n\n \t\t\tJogar(1)\t \n \t\t\tSair(2)\t\n \t\t\tPontuacao(3)\t\n\t\t\tDicas(4)\t \n\n \t\t\tEscolha: "); scanf("%d",&opcao); switch(opcao) //opcoes do menu. { case 1: //começa o jogo. printf("Informe seu nome: "); scanf("%s", jogadores[5].nome); printf("\n\n"); jogo(jogadores); break; case 2: //finaliza o jogo. printf("\t\n Voce finalizou o jogo\n\n\tQue pena T-T\n\n"); return -1; case 3: //mostra as pontuaçoes. i = 0; printf("\n\n\t\tMelhores pontuacoes:\n\n"); while (i < 5) { printf("\t\t%s\t:%d\n", jogadores[i].nome, jogadores[i].p); i++; } printf("\n\n"); menu(); break; case 4: //Mostra as instruçoes. instrucoes(); break; } return 0; }
int maximalRectangle(vector<vector<char>>& matrix) { if(matrix.size() == 0) return 0; vector<vector<int>> records(matrix.size(), vector<int>(matrix[0].size(), 0)); for(int i = 0; i < matrix.size(); ++ i){ for(int j = 0; j < matrix[i].size(); ++ j){ if(matrix[i][j] == '0') records[i][j] = 0; else records[i][j] = i > 0 ? records[i - 1][j] + 1 : 1; } } int area = 0; for(int i = 0; i < records.size(); ++ i){ int temp = maxArea(records[i]); if(temp > area) area = temp; } return area; }
void MainWindow::createActions() { newGameChaosAct = gameMenu->addAction( tr("Нова гра (Хаос)") ); newGameStraightAct = gameMenu->addAction( tr("Нова гра (По прямій)") ); resetLevelAction = gameMenu->addAction( tr("Скинути поточний рівень") ); recordsAction = gameMenu->addAction( tr("Рекорди") ); gameMenu->addSeparator(); // Линия в меню quitAction = gameMenu->addAction( tr("Вихід") ); helpAction = helpMenu->addAction( tr("Як грати?") ); helpMenu->addSeparator(); aboutAction = helpMenu->addAction( tr("Про гру") ); // hot keys newGameChaosAct->setShortcut ( QKeySequence(Qt::CTRL | Qt::Key_C) ); newGameStraightAct->setShortcut ( QKeySequence(Qt::CTRL | Qt::Key_S) ); resetLevelAction->setShortcut ( QKeySequence(Qt::Key_Space) ); quitAction->setShortcut ( QKeySequence(Qt::CTRL | Qt::Key_Q) ); helpAction->setShortcut ( QKeySequence(Qt::Key_F1) ); // Connect connect( newGameChaosAct, SIGNAL(triggered()), this, SLOT( newGameChaos())); connect( newGameStraightAct, SIGNAL(triggered()), this, SLOT( newGameStraight())); connect( resetLevelAction, SIGNAL(triggered()), this, SLOT( resetGame())); connect( recordsAction, SIGNAL(triggered()), this, SLOT( records())); connect( quitAction, SIGNAL(triggered()), this, SLOT( close())); connect( helpAction, SIGNAL(triggered()), this, SLOT( manualGame())); connect( aboutAction, SIGNAL(triggered()), this, SLOT( about())); }
std::vector<CountH5Record> count_placeholders( const std::vector<std::pair<std::string, size_t>>& chromosome_lengths, const std::string& cell_type, const std::string& bam_file_path, const unsigned int bin_size) { size_t num_records = 0; for (auto& chr_length : chromosome_lengths) { int bins = std::ceil(chr_length.second / (double) bin_size); num_records += bins; } const std::string bam_file_name = file_name_from_path(bam_file_path); CountH5Record empty_record; empty_record.bin_number = 0; empty_record.count = 0; strncpy(empty_record.cell_type, cell_type.c_str(), sizeof(CountH5Record::cell_type)); strncpy(empty_record.chromosome, "", sizeof(CountH5Record::chromosome)); strncpy(empty_record.file_name, bam_file_name.c_str(), sizeof(CountH5Record::file_name)); std::vector<CountH5Record> records(num_records, empty_record); size_t i=0; for (auto& chr_length : chromosome_lengths) { int bins = std::ceil(chr_length.second / (double) bin_size); for (size_t j=0; j < bins; ++j, ++i) { records[i].bin_number = j; strncpy(records[i].chromosome, chr_length.first.c_str(), sizeof(CountH5Record::chromosome)); } } return records; }
void ListView::answerReceived(int id, const QList<QByteArray>& answer) { /* We received something, so the sensor is probably ok. */ sensorError(id, false); switch (id) { case 100: { /* We have received the answer to a '?' command that contains * the information about the table headers. */ if (answer.count() != 2) { kWarning(1215) << "wrong number of lines"; return; } KSGRD::SensorTokenizer headers(answer[0], '\t'); KSGRD::SensorTokenizer colTypes(answer[1], '\t'); /* add the new columns */ mModel.clear(); QStringList translatedHeaders; for (uint i = 0; i < headers.count(); i++) { translatedHeaders.append( i18nc("heading from daemon", headers[i]) ); } for(uint i =0 ; i < colTypes.count(); i++) { ColumnType type = convertColumnType(colTypes[i]); mColumnTypes.append(type); if (type == Text || type == DiskStat) mModel.addColumnAlignment(Qt::AlignLeft); else mModel.addColumnAlignment(Qt::AlignRight); } mModel.setHorizontalHeaderLabels(translatedHeaders); //If we have some header settings to restore, we can do so now if(!mHeaderSettings.isEmpty()) { mView->header()->restoreState(mHeaderSettings); mModel.sort( mView->header()->sortIndicatorSection(), mView->header()->sortIndicatorOrder() ); } break; } case 19: { for (int i = 0; i < answer.count(); i++) { KSGRD::SensorTokenizer records(answer[i], '\t'); for (uint j = 0; j < records.count() && j < mColumnTypes.count(); j++) { QStandardItem *item = new QStandardItem(); item->setEditable(false); switch( mColumnTypes[j] ) { case Int: item->setData(records[j].toLongLong(), Qt::UserRole); item->setText(records[j]); break; case Percentage: item->setData(records[j].toInt(), Qt::UserRole); item->setText(records[j] + "%"); break; case Float: item->setData(records[j].toFloat(), Qt::DisplayRole); item->setData(records[j].toFloat(), Qt::UserRole); break; case Time: item->setData(QTime::fromString(records[j]), Qt::DisplayRole); item->setData(QTime::fromString(records[j]), Qt::UserRole); break; case KByte: { item->setData(records[j].toInt(), Qt::UserRole); item->setText(formatByteSize(records[j].toLongLong(), mUnits)); break; } case DiskStat: case Text: default: item->setText(records[j]); item->setData(records[j], Qt::UserRole); } mModel.setItem(i, j, item); } } mModel.setRowCount(answer.count()); mModel.sort( mView->header()->sortIndicatorSection(), mView->header()->sortIndicatorOrder() ); break; } } }
void EvalDetectionLayer<Dtype>::Forward_cpu( const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) { //const Dtype* input_data = bottom[0]->cpu_data();// 网络输出 N*13*13*125 const Dtype* label_data = bottom[1]->cpu_data(); // 真实标签数据 N*30*5 //LOG(INFO) << bottom[0]->data_at(0,0,0,0) << " " << bottom[0]->data_at(0,0,0,1); Blob<Dtype> swap;// 网络输出数据 N * 13* 13* 125 // 变形为 N * (13 * 13) * 5 * 25 // N*(5*(5+num_class_))*13*13 -> N * (13*13) * 5 * (5+num_class_) swap.Reshape(bottom[0]->num(), bottom[0]->height()*bottom[0]->width(), num_object_, bottom[0]->channels()/num_object_); Dtype* swap_data = swap.mutable_cpu_data();// cpu上的数据 caffe_set(swap.count(), Dtype(0.0), swap_data);// 设置为0 int index = 0; for (int b = 0; b < bottom[0]->num(); ++b)// 图片数量 for (int h = 0; h < bottom[0]->height(); ++h) // 格子 13 for (int w = 0; w < bottom[0]->width(); ++w)// 格子 13 for (int c = 0; c < bottom[0]->channels(); ++c)// 5*25=125 { swap_data[index++] = bottom[0]->data_at(b,c,h,w); } //*******************************************************************************// //caffe_set(swap.count(), Dtype(0.0), swap_data);// 设置为0 //int p_index = (7*13+4)*125; //swap_data[p_index]=-0.1020; //swap_data[p_index+1]=2.0867; //swap_data[p_index+2]=1.612; //swap_data[p_index+3]=1.0515; //swap_data[p_index+4]=1.0; //swap_data[p_index+5+11]=100; //*******************************************************************************// Dtype* top_data = top[0]->mutable_cpu_data();// 层输出 cpu数据 caffe_set(top[0]->count(), Dtype(0), top_data);// 设置为0 Dtype all_mAP = 0.0;// 总 batch 的 mAP for (int i = 0; i < bottom[0]->num(); ++i) { // N 图片数量 int input_index = i * bottom[0]->count(1);// 网络输出标签 i * 13*13*125 int true_index = i * bottom[1]->count(1);// 真实标签 i * 30*5 int top_index = i * top[0]->count(1); // 输出数据 // i * ( 20 + 13*13*5*4 + 1) -> i * (13*13*5*4 + 1) // 前面20个为 真实标签 物体类别出现的次数 // 获取真实边框 ========================================= map<int, vector<BoxData> > gt_boxes; // 从 对应图片的标签数据中 获取 真实边框 label_ + score_ + box_ // 返回一张图像的 标签 + 多边框数据的 标签 GetGTBox(side_, label_data + true_index, >_boxes); // 在输出数据中 记录 真实标签 物体类别出现的次数======================= for (std::map<int, vector<BoxData > >::iterator it = gt_boxes.begin(); it != gt_boxes.end(); ++it) { // 遍历 每一个 真实的标签 //int label = it->first;// 标签 类别 vector<BoxData>& g_boxes = it->second;// BoxData: label_ + score_ + box_ for (int j = 0; j < g_boxes.size(); ++j) {// 边框数量 // 输出数据中的前 20个================================================ // ======================================= // top_data[top_index + label] += 1; // 真实标签 物体类别出现的次数 } } // 获取预测边框 ============================================= map<int, vector<BoxData> > pred_boxes; // 获取预测边框 13*13*5 -> NMS抑制+置信度抑制 -> pred_boxes(数量少很多) //GetPredBox(side_, num_object_, num_class_, input_data + input_index, &pred_boxes, sqrt_, constriant_, score_type_, nms_); GetPredBox(side_, num_object_, num_class_, swap_data + input_index, &pred_boxes, score_type_, nms_, biases_); // 输出数据 后面 的 13*13*5*4 ============================= // int index = top_index + num_class_ + 1;// 20 + 1 之后为 上面的 (label + score + tp + fp) 参数 //============================= int index = top_index + 1;// 1个 map 之后为 上面的 (label + score + tp + fp) 参数 int pred_count(0);// Dtype mAP = 0.0; int pre_clas_num=0; //float AP = 0.0; // int tp // 遍历预测值的 每一类,与最合适的标签边框计算AP,在计算总类别的mAP============ for (std::map<int, vector<BoxData> >::iterator it = pred_boxes.begin(); it != pred_boxes.end(); ++it) { Dtype AP = 0.0;// 该类AP int tp=0;// 预测正确 int fp=0;// 预测错误 ++pre_clas_num;// 该图片预测的总类别数量 int label = it->first;// 预测 边框标签 vector<BoxData>& p_boxes = it->second;// 多个边框 vector<BoxData> // 真实标签中 未找到该 类别======================================= if (gt_boxes.find(label) == gt_boxes.end()) { for (int b = 0; b < p_boxes.size(); ++b) {// 该类别下的每一个预测边框 top_data[index + pred_count * 4 + 0] = p_boxes[b].label_;// 标签 top_data[index + pred_count * 4 + 1] = p_boxes[b].score_;// 得分 top_data[index + pred_count * 4 + 2] = 0; //tp top_data[index + pred_count * 4 + 3] = 1; //fp 错误的预测为正确的 ++pred_count; ++fp;// 预测错误============================================ } if(tp + fp) AP = tp / (tp + fp);// 计算该类别的 平均准确度 这里等于0 mAP += AP;// 所有类别的总 AP 这里可以省略 因为 AP为0 continue;// 跳过预测错误的,只记录fp } // 真实标签找到了该预测的类别====================================== vector<BoxData>& g_boxes = gt_boxes[label];// 真实标签中该 类别的 多个真实边框===== vector<bool> records(g_boxes.size(), false);// 记录 真实类别的每一个 物体边框 是否已经被预测过 for (int k = 0; k < p_boxes.size(); ++k) { // 遍历 每个 预测边框============== top_data[index + pred_count * 4 + 0] = p_boxes[k].label_;// 标签 top_data[index + pred_count * 4 + 1] = p_boxes[k].score_;// 得分 Dtype max_iou(-1);// 预测边框最接近的 真实边框 iou int idx(-1);// 对应的 真实边框id // 遍历每个真实边框 找到 预测边框最接近的 真实边框=========== for (int g = 0; g < g_boxes.size(); ++g) { Dtype iou = Calc_iou(p_boxes[k].box_, g_boxes[g].box_);// 计算交并比 if (iou > max_iou) { max_iou = iou;// 记录 每个预测边框 最接近的 真实边框 的 IOU idx = g;// 对应的 真实边框id } } // 根据 交并比 确定判断 预测 正确/错误 if (max_iou >= threshold_) { if ( !records[idx] ) { records[idx] = true;// 对应的 真实边框id top_data[index + pred_count * 4 + 2] = 1; // tp 正->正确 top_data[index + pred_count * 4 + 3] = 0; // fp ++tp;// 预测正确 ================================= } else {// 同一个位置的物体之前已经被预测过,又一个框来预测,则认为是错误的 top_data[index + pred_count * 4 + 2] = 0; top_data[index + pred_count * 4 + 3] = 1;// 错误 -> 正确 ++fp;// 预测错误============================================ } } ++pred_count; } if(tp + fp) AP = tp / (tp + fp);// 计算该类别的 平均准确度 mAP += AP;// 所有类别的总 AP } if(pre_clas_num){ mAP /= pre_clas_num; // mAP /= num_class_; // 计算 mAP 是除以总类别数,还是总预测类别数 } else mAP = 0.0; // 输出对应图片的mAP top_data[ index - 1 ] = mAP; all_mAP += mAP; } if(bottom[0]->num()) all_mAP /= (bottom[0]->num()); top_data[0] = all_mAP; }
int methyltest_main(int argc, char** argv) { parse_methyltest_options(argc, argv); omp_set_num_threads(opt::num_threads); Fast5Map name_map(opt::reads_file); ModelMap models = read_models_fofn(opt::models_fofn, mtest_alphabet); // Open the BAM and iterate over reads // load bam file htsFile* bam_fh = sam_open(opt::bam_file.c_str(), "r"); assert(bam_fh != NULL); // load bam index file std::string index_filename = opt::bam_file + ".bai"; hts_idx_t* bam_idx = bam_index_load(index_filename.c_str()); assert(bam_idx != NULL); // read the bam header bam_hdr_t* hdr = sam_hdr_read(bam_fh); // load reference fai file faidx_t *fai = fai_load(opt::genome_file.c_str()); hts_itr_t* itr; // If processing a region of the genome, only emit events aligned to this window int clip_start = -1; int clip_end = -1; if(opt::region.empty()) { // TODO: is this valid? itr = sam_itr_queryi(bam_idx, HTS_IDX_START, 0, 0); } else { fprintf(stderr, "Region: %s\n", opt::region.c_str()); itr = sam_itr_querys(bam_idx, hdr, opt::region.c_str()); hts_parse_reg(opt::region.c_str(), &clip_start, &clip_end); } #ifndef H5_HAVE_THREADSAFE if(opt::num_threads > 1) { fprintf(stderr, "You enabled multi-threading but you do not have a threadsafe HDF5\n"); fprintf(stderr, "Please recompile nanopolish's built-in libhdf5 or run with -t 1\n"); exit(1); } #endif // Initialize writers OutputHandles handles; handles.site_writer = fopen(std::string(opt::bam_file + ".methyltest.sites.bed").c_str(), "w"); handles.read_writer = fopen(std::string(opt::bam_file + ".methyltest.reads.tsv").c_str(), "w"); handles.strand_writer = fopen(std::string(opt::bam_file + ".methyltest.strand.tsv").c_str(), "w"); // Write a header to the reads.tsv file fprintf(handles.read_writer, "name\tsum_ll_ratio\tn_cpg\tcomplement_model\ttags\n"); // strand header fprintf(handles.strand_writer, "name\tsum_ll_ratio\tn_cpg\tmodel\n"); // Initialize iteration std::vector<bam1_t*> records(opt::batch_size, NULL); for(size_t i = 0; i < records.size(); ++i) { records[i] = bam_init1(); } int result; size_t num_reads_processed = 0; size_t num_records_buffered = 0; Progress progress("[methyltest]"); do { assert(num_records_buffered < records.size()); // read a record into the next slot in the buffer result = sam_itr_next(bam_fh, itr, records[num_records_buffered]); num_records_buffered += result >= 0; // realign if we've hit the max buffer size or reached the end of file if(num_records_buffered == records.size() || result < 0) { #pragma omp parallel for for(size_t i = 0; i < num_records_buffered; ++i) { bam1_t* record = records[i]; size_t read_idx = num_reads_processed + i; if( (record->core.flag & BAM_FUNMAP) == 0) { calculate_methylation_for_read(models, name_map, fai, hdr, record, read_idx, handles); } } num_reads_processed += num_records_buffered; num_records_buffered = 0; } } while(result >= 0); assert(num_records_buffered == 0); progress.end(); // cleanup records for(size_t i = 0; i < records.size(); ++i) { bam_destroy1(records[i]); } // cleanup fclose(handles.site_writer); fclose(handles.read_writer); fclose(handles.strand_writer); sam_itr_destroy(itr); bam_hdr_destroy(hdr); fai_destroy(fai); sam_close(bam_fh); hts_idx_destroy(bam_idx); return EXIT_SUCCESS; }
void train_one_round(const Fast5Map& name_map, size_t round) { const PoreModelMap& current_models = PoreModelSet::get_models(opt::trained_model_type); // Initialize the training summary stats for each kmer for each model ModelTrainingMap model_training_data; for(auto current_model_iter = current_models.begin(); current_model_iter != current_models.end(); current_model_iter++) { // one summary entry per kmer in the model std::vector<StateSummary> summaries(current_model_iter->second.get_num_states()); model_training_data[current_model_iter->first] = summaries; } // Open the BAM and iterate over reads // load bam file htsFile* bam_fh = sam_open(opt::bam_file.c_str(), "r"); assert(bam_fh != NULL); // load bam index file std::string index_filename = opt::bam_file + ".bai"; hts_idx_t* bam_idx = bam_index_load(index_filename.c_str()); assert(bam_idx != NULL); // read the bam header bam_hdr_t* hdr = sam_hdr_read(bam_fh); // load reference fai file faidx_t *fai = fai_load(opt::genome_file.c_str()); hts_itr_t* itr; // If processing a region of the genome, only emit events aligned to this window int clip_start = -1; int clip_end = -1; if(opt::region.empty()) { // TODO: is this valid? itr = sam_itr_queryi(bam_idx, HTS_IDX_START, 0, 0); } else { fprintf(stderr, "Region: %s\n", opt::region.c_str()); itr = sam_itr_querys(bam_idx, hdr, opt::region.c_str()); hts_parse_reg(opt::region.c_str(), &clip_start, &clip_end); } #ifndef H5_HAVE_THREADSAFE if(opt::num_threads > 1) { fprintf(stderr, "You enabled multi-threading but you do not have a threadsafe HDF5\n"); fprintf(stderr, "Please recompile nanopolish's built-in libhdf5 or run with -t 1\n"); exit(1); } #endif // Initialize iteration std::vector<bam1_t*> records(opt::batch_size, NULL); for(size_t i = 0; i < records.size(); ++i) { records[i] = bam_init1(); } int result; size_t num_reads_realigned = 0; size_t num_records_buffered = 0; Progress progress("[methyltrain]"); do { assert(num_records_buffered < records.size()); // read a record into the next slot in the buffer result = sam_itr_next(bam_fh, itr, records[num_records_buffered]); num_records_buffered += result >= 0; // realign if we've hit the max buffer size or reached the end of file if(num_records_buffered == records.size() || result < 0) { #pragma omp parallel for for(size_t i = 0; i < num_records_buffered; ++i) { bam1_t* record = records[i]; size_t read_idx = num_reads_realigned + i; if( (record->core.flag & BAM_FUNMAP) == 0) { add_aligned_events(name_map, fai, hdr, record, read_idx, clip_start, clip_end, round, model_training_data); } } num_reads_realigned += num_records_buffered; num_records_buffered = 0; } if(opt::progress) { fprintf(stderr, "Realigned %zu reads in %.1lfs\r", num_reads_realigned, progress.get_elapsed_seconds()); } } while(result >= 0); assert(num_records_buffered == 0); progress.end(); // open the summary file std::stringstream summary_fn; summary_fn << "methyltrain" << opt::out_suffix << ".summary"; FILE* summary_fp = fopen(summary_fn.str().c_str(), "w"); fprintf(summary_fp, "model_short_name\tkmer\tnum_matches\tnum_skips\t" "num_stays\tnum_events_for_training\twas_trained\t" "trained_level_mean\ttrained_level_stdv\n"); // open the tsv file with the raw training data std::stringstream training_fn; training_fn << "methyltrain" << opt::out_suffix << ".round" << round << ".events.tsv"; std::ofstream training_ofs(training_fn.str()); // write out a header for the training data StateTrainingData::write_header(training_ofs); // iterate over models: template, complement_pop1, complement_pop2 for(auto model_training_iter = model_training_data.begin(); model_training_iter != model_training_data.end(); model_training_iter++) { // Initialize the trained model from the input model auto current_model_iter = current_models.find(model_training_iter->first); assert(current_model_iter != current_models.end()); std::string model_name = model_training_iter->first; std::string model_short_name = current_model_iter->second.metadata.get_short_name(); // Initialize the new model from the current model PoreModel updated_model = current_model_iter->second; uint32_t k = updated_model.k; const std::vector<StateSummary>& summaries = model_training_iter->second; // Generate the complete set of kmers std::string gen_kmer(k, 'A'); std::vector<std::string> all_kmers; for(size_t ki = 0; ki < summaries.size(); ++ki) { all_kmers.push_back(gen_kmer); mtrain_alphabet->lexicographic_next(gen_kmer); } assert(gen_kmer == std::string(k, 'A')); assert(all_kmers.front() == std::string(k, 'A')); assert(all_kmers.back() == std::string(k, 'T')); // Update means for each kmer #pragma omp parallel for for(size_t ki = 0; ki < summaries.size(); ++ki) { assert(ki < all_kmers.size()); std::string kmer = all_kmers[ki]; // write the observed values to a tsv file #pragma omp critical { for(size_t ei = 0; ei < summaries[ki].events.size(); ++ei) { summaries[ki].events[ei].write_tsv(training_ofs, model_short_name, kmer); } } bool is_m_kmer = kmer.find('M') != std::string::npos; bool update_kmer = opt::training_target == TT_ALL_KMERS || (is_m_kmer && opt::training_target == TT_METHYLATED_KMERS) || (!is_m_kmer && opt::training_target == TT_UNMETHYLATED_KMERS); bool trained = false; // only train if there are a sufficient number of events for this kmer if(update_kmer && summaries[ki].events.size() >= opt::min_number_of_events_to_train) { // train a mixture model where a minority of k-mers aren't methylated ParamMixture mixture; float incomplete_methylation_rate = 0.05f; std::string um_kmer = mtrain_alphabet->unmethylate(kmer); size_t um_ki = mtrain_alphabet->kmer_rank(um_kmer.c_str(), k); // Initialize the training parameters. If this is a kmer containing // a methylation site we train a two component mixture, otherwise // just fit a gaussian float major_weight = is_m_kmer ? 1 - incomplete_methylation_rate : 1.0f; mixture.log_weights.push_back(log(major_weight)); mixture.params.push_back(current_model_iter->second.get_parameters(ki)); if(is_m_kmer) { // add second unmethylated component mixture.log_weights.push_back(std::log(incomplete_methylation_rate)); mixture.params.push_back(current_model_iter->second.get_parameters(um_ki)); } if(opt::verbose > 1) { fprintf(stderr, "INIT__MIX %s\t%s\t[%.2lf %.2lf %.2lf]\t[%.2lf %.2lf %.2lf]\n", model_training_iter->first.c_str(), kmer.c_str(), std::exp(mixture.log_weights[0]), mixture.params[0].level_mean, mixture.params[0].level_stdv, std::exp(mixture.log_weights[1]), mixture.params[1].level_mean, mixture.params[1].level_stdv); } ParamMixture trained_mixture = train_gaussian_mixture(summaries[ki].events, mixture); if(opt::verbose > 1) { fprintf(stderr, "TRAIN_MIX %s\t%s\t[%.2lf %.2lf %.2lf]\t[%.2lf %.2lf %.2lf]\n", model_training_iter->first.c_str(), kmer.c_str(), std::exp(trained_mixture.log_weights[0]), trained_mixture.params[0].level_mean, trained_mixture.params[0].level_stdv, std::exp(trained_mixture.log_weights[1]), trained_mixture.params[1].level_mean, trained_mixture.params[1].level_stdv); } #pragma omp critical updated_model.states[ki] = trained_mixture.params[0]; if (model_stdv()) { ParamMixture ig_mixture; // weights ig_mixture.log_weights = trained_mixture.log_weights; // states ig_mixture.params.emplace_back(trained_mixture.params[0]); if(is_m_kmer) { ig_mixture.params.emplace_back(current_model_iter->second.get_parameters(um_ki)); } // run training auto trained_ig_mixture = train_invgaussian_mixture(summaries[ki].events, ig_mixture); LOG("methyltrain", debug) << "IG_INIT__MIX " << model_training_iter->first.c_str() << " " << kmer.c_str() << " [" << std::fixed << std::setprecision(5) << ig_mixture.params[0].sd_mean << " " << ig_mixture.params[1].sd_mean << "]" << std::endl << "IG_TRAIN_MIX " << model_training_iter->first.c_str() << " " << kmer.c_str() << " [" << trained_ig_mixture.params[0].sd_mean << " " << trained_ig_mixture.params[1].sd_mean << "]" << std::endl; // update state #pragma omp critical { updated_model.states[ki] = trained_ig_mixture.params[0]; } } trained = true; } #pragma omp critical { fprintf(summary_fp, "%s\t%s\t%d\t%d\t%d\t%zu\t%d\t%.2lf\t%.2lf\n", model_short_name.c_str(), kmer.c_str(), summaries[ki].num_matches, summaries[ki].num_skips, summaries[ki].num_stays, summaries[ki].events.size(), trained, updated_model.states[ki].level_mean, updated_model.states[ki].level_stdv); } // add the updated model into the collection (or replace what is already there) PoreModelSet::insert_model(opt::trained_model_type, updated_model); } } // cleanup records for(size_t i = 0; i < records.size(); ++i) { bam_destroy1(records[i]); } // cleanup sam_itr_destroy(itr); bam_hdr_destroy(hdr); fai_destroy(fai); sam_close(bam_fh); hts_idx_destroy(bam_idx); fclose(summary_fp); }
rsurf_iso_contouring::rsurf_iso_contouring(const rsurf_data& data,const std::size_t& rsIndex) :iso_contouring_data(new t_iso_contouring_data()) { //Initialisation std::size_t tsSize(0); std::size_t nodesSize(0); std::size_t nbtimestep(0); std::size_t nbfaces(0); std::size_t idstep(0); std::size_t vertices[3]={0,0,0}; std::size_t records(0); float timestep(0.f); float energy(0.f); float totenergy(0.f); std::string rs_name; std::string recordType; int rs_xmlid(0); bool converttodb=false; data.GetRsInfo(rsIndex,nbfaces,rs_name,rs_xmlid); data.GetFileInfos(tsSize,nodesSize,nbtimestep,timestep,recordType); if(recordType=="SPL_STANDART" || recordType=="SPL_GAIN") converttodb=true; //Passage de valeur de surface vers des valeurs de sommets std::vector<float> vertices_lvl(nodesSize,0.f); std::vector<unsigned int> vertices_countlinkedfaces(nodesSize,0); std::vector<bool> face_useforcalculation(nbfaces,true); for(std::size_t idface=0;idface<nbfaces;idface++) { data.GetFaceInfo(rsIndex,idface,vertices[0],vertices[1],vertices[2],records); //Cumul de l'energie sur le temps totenergy=0; for(std::size_t idrecord=0;idrecord<records;idrecord++) { data.GetFaceEnergy(rsIndex,idface,idrecord,idstep,energy); totenergy+=energy; } if(totenergy==0 || totenergy==std::numeric_limits<float>::infinity() || -totenergy==std::numeric_limits<float>::infinity()) { face_useforcalculation[idface]=false; }else{ vertices_countlinkedfaces[vertices[0]]+=1; vertices_countlinkedfaces[vertices[1]]+=1; vertices_countlinkedfaces[vertices[2]]+=1; vertices_lvl[vertices[0]]+=totenergy; vertices_lvl[vertices[1]]+=totenergy; vertices_lvl[vertices[2]]+=totenergy; } } //Division des valeurs par leurs nombre de contributeurs et conversion en dB for(std::size_t idvert=0;idvert<nodesSize;idvert++) { if(vertices_countlinkedfaces[idvert]>0) vertices_lvl[idvert]/=vertices_countlinkedfaces[idvert]; if(converttodb && vertices_lvl[idvert]>0) vertices_lvl[idvert]=10*log10(vertices_lvl[idvert]/pow(10.f,-12.f)); } //Initialisation des variables de la classe iso_contouring_data->minValue=vertices_lvl[vertices[0]]; iso_contouring_data->maxValue=iso_contouring_data->minValue; vec3 A,B,C; for(std::size_t idface=0;idface<nbfaces;idface++) { data.GetFaceInfo(rsIndex,idface,vertices[0],vertices[1],vertices[2],records); data.GetNodePositionValue(vertices[0],A.x,A.y,A.z); data.GetNodePositionValue(vertices[1],B.x,B.y,B.z); data.GetNodePositionValue(vertices[2],C.x,C.y,C.z); t_tri_data newtri(A,B,C,vertices_lvl[vertices[0]],vertices_lvl[vertices[1]],vertices_lvl[vertices[2]]); newtri.useInIsoLines=face_useforcalculation[idface]; iso_contouring_data->tri_grid.push_back(newtri); FINDMINMAX_RSURF(vertices_lvl[vertices[0]],vertices_lvl[vertices[1]],vertices_lvl[vertices[2]],iso_contouring_data->minValue,iso_contouring_data->maxValue); } }
int scorereads_main(int argc, char** argv) { parse_scorereads_options(argc, argv); omp_set_num_threads(opt::num_threads); Fast5Map name_map(opt::reads_file); ModelMap models; if (!opt::models_fofn.empty()) models = read_models_fofn(opt::models_fofn); // Open the BAM and iterate over reads // load bam file htsFile* bam_fh = sam_open(opt::bam_file.c_str(), "r"); assert(bam_fh != NULL); // load bam index file std::string index_filename = opt::bam_file + ".bai"; hts_idx_t* bam_idx = bam_index_load(index_filename.c_str()); assert(bam_idx != NULL); // read the bam header bam_hdr_t* hdr = sam_hdr_read(bam_fh); // load reference fai file faidx_t *fai = fai_load(opt::genome_file.c_str()); hts_itr_t* itr; // If processing a region of the genome, only emit events aligned to this window int clip_start = -1; int clip_end = -1; if(opt::region.empty()) { // TODO: is this valid? itr = sam_itr_queryi(bam_idx, HTS_IDX_START, 0, 0); } else { fprintf(stderr, "Region: %s\n", opt::region.c_str()); itr = sam_itr_querys(bam_idx, hdr, opt::region.c_str()); hts_parse_reg(opt::region.c_str(), &clip_start, &clip_end); } #ifndef H5_HAVE_THREADSAFE if(opt::num_threads > 1) { fprintf(stderr, "You enabled multi-threading but you do not have a threadsafe HDF5\n"); fprintf(stderr, "Please recompile nanopolish's built-in libhdf5 or run with -t 1\n"); exit(1); } #endif // Initialize iteration std::vector<bam1_t*> records(opt::batch_size, NULL); for(size_t i = 0; i < records.size(); ++i) { records[i] = bam_init1(); } int result; size_t num_reads_realigned = 0; size_t num_records_buffered = 0; do { assert(num_records_buffered < records.size()); // read a record into the next slot in the buffer result = sam_itr_next(bam_fh, itr, records[num_records_buffered]); num_records_buffered += result >= 0; // realign if we've hit the max buffer size or reached the end of file if(num_records_buffered == records.size() || result < 0) { #pragma omp parallel for schedule(dynamic) for(size_t i = 0; i < num_records_buffered; ++i) { bam1_t* record = records[i]; size_t read_idx = num_reads_realigned + i; if( (record->core.flag & BAM_FUNMAP) == 0) { //load read std::string read_name = bam_get_qname(record); std::string fast5_path = name_map.get_path(read_name); SquiggleRead sr(read_name, fast5_path); // TODO: early exit when have processed all of the reads in readnames if (!opt::readnames.empty() && std::find(opt::readnames.begin(), opt::readnames.end(), read_name) == opt::readnames.end() ) continue; for(size_t strand_idx = 0; strand_idx < NUM_STRANDS; ++strand_idx) { std::vector<EventAlignment> ao = alignment_from_read(sr, strand_idx, read_idx, models, fai, hdr, record, clip_start, clip_end); if (ao.size() == 0) continue; // Update pore model based on alignment if ( opt::calibrate ) recalibrate_model(sr, strand_idx, ao, false); double score = model_score(sr, strand_idx, fai, ao, 500); if (score > 0) continue; #pragma omp critical(print) std::cout << read_name << " " << ( strand_idx ? "complement" : "template" ) << " " << sr.pore_model[strand_idx].name << " " << score << std::endl; } } } num_reads_realigned += num_records_buffered; num_records_buffered = 0; } } while(result >= 0); // cleanup records for(size_t i = 0; i < records.size(); ++i) { bam_destroy1(records[i]); } // cleanup sam_itr_destroy(itr); bam_hdr_destroy(hdr); fai_destroy(fai); sam_close(bam_fh); hts_idx_destroy(bam_idx); return 0; }