int main() {
const int rounds = 10000;
std::vector<FlowTests::FlowRecord> records(1000);
boost::random::mt19937 gen;
for(int i=0; i < 1000; i++)
records[i].Fill(gen);
FlowTests::FlowRecord f;
f.Fill(gen);
int maxVal;
boost::chrono::steady_clock::time_point start = boost::chrono::steady_clock::now();
for(int i=0; i < rounds; i++)
{
boost::lockfree::queue<int> q(records.size());
for(auto iter = records.begin(); iter != records.end(); ++iter)
{
q.push(iter->someInfo());
}
while(!q.empty())
{
int v;
if(q.pop(v) && maxVal < v)
maxVal = v;
}
}
boost::chrono::duration<double> sec = boost::chrono::steady_clock::now() - start;
std::cout << "running " << rounds << " times of 1000 records took " << sec.count() << " seconds and it resulted in " << maxVal << std::endl;
return 0;
}
void CSVWorld::ScriptEdit::dropEvent (QDropEvent* event)
{
const CSMWorld::TableMimeData* mime = dynamic_cast<const CSMWorld::TableMimeData*> (event->mimeData());
if (!mime) // May happen when non-records (e.g. plain text) are dragged and dropped
{
QPlainTextEdit::dropEvent(event);
return;
}
setTextCursor (cursorForPosition (event->pos()));
if (mime->fromDocument (mDocument))
{
std::vector<CSMWorld::UniversalId> records (mime->getData());
for (std::vector<CSMWorld::UniversalId>::iterator it = records.begin(); it != records.end(); ++it)
{
if (mAllowedTypes.contains (it->getType()))
{
if (stringNeedsQuote(it->getId()))
{
insertPlainText(QString::fromUtf8 (('"' + it->getId() + '"').c_str()));
} else {
insertPlainText(QString::fromUtf8 (it->getId().c_str()));
}
}
}
}
}
std::vector<CountH5Record> count(const std::string chr,
const std::string cell_type,
const unsigned int bin_size,
const size_t length,
const unsigned int extension,
const char strand,
const std::string bam_file)
{
const std::string bam_file_name = file_name_from_path(bam_file);
int bins = std::ceil(length / (double) bin_size);
int max_base_pair = bins * bin_size;
const std::vector<double> bin_counts = liquidate(bam_file, chr, 0, max_base_pair, strand, bins, extension);
CountH5Record record;
record.bin_number = 0;
strncpy(record.cell_type, cell_type.c_str(), sizeof(CountH5Record::cell_type));
strncpy(record.chromosome, chr.c_str(), sizeof(CountH5Record::chromosome));
strncpy(record.file_name, bam_file_name.c_str(), sizeof(CountH5Record::file_name));
std::vector<CountH5Record> records(bin_counts.size(), record);
for (size_t bin=0; bin < bin_counts.size(); ++bin)
{
records[bin].bin_number = bin;
records[bin].count = bin_counts[bin];
}
return records;
}
void WifiDialog::appendStoredAPs(WifiProfiles & list)
{
sys::SystemConfig conf;
WifiProfiles stored_aps = records(conf);
for(int i = 0; i < stored_aps.size(); i++)
{
if (!stored_aps[i].ssid().isEmpty())
{
bool found = false;
foreach(WifiProfile profile, list)
{
if (stored_aps[i].ssid() == profile.ssid())
{
found = true;
break;
}
}
if (!found)
{
stored_aps[i].setPresent(false);
ODataPtr d(new OData(stored_aps[i]));
datas_.push_back(d);
}
}
}
void DlgRecord::setData(QHostAddress saddr, quint16 port, quint16 vodport) {
if(_pctlc == NULL) {
if(vodport == 0) {
_vodport = 9001; //default port
} else {
_vodport = vodport;
}
_pctlc = new ControlClient(saddr, port);
connect(_pctlc, SIGNAL(routeOut(QString)), this, SLOT(records(QString)));
_vodprefix = "http://" + saddr.toString() + ":" + QString::number(_vodport);
}
}
vector<double> extended_pred_symbol::getTimedAchievers(Environment * f,const proposition * prop) const
{
vector<double> tms;
if(records()->get(f,prop))
{
tms.push_back(0);
};
for(map<double,PropStore *>::const_iterator i = timedInitials.begin();i != timedInitials.end();++i)
{
if(i->second->get(f,prop))
{
tms.push_back(i->first);
};
};
return tms;
};
QList<IoslotValueRecord> IoslotValueTable::median(const QDateTime &from, const QDateTime &to, int limit)
{
int count = recordCount(from, to);
qint64 w1 = from.toMSecsSinceEpoch() - to.toMSecsSinceEpoch();
qint64 w2 = double(limit) / double(count) * w1;
qint16 dw = (w1 - w2) / 2;
QDateTime dt1 = from;
QDateTime dt2 = to;
if (dw > 0) {
dt1 = QDateTime::fromMSecsSinceEpoch(dt1.toMSecsSinceEpoch() + dw);
dt2 = QDateTime::fromMSecsSinceEpoch(dt2.toMSecsSinceEpoch() - dw);
}
return records(dt1, dt2, limit);
}
json QueryRunner::get_result_as_json() {
json result(new json_object());
result["id"] = new json_integer(id);
result["query_num"] = new json_integer(query_num);
size_t success_count = 0;
json records(new json_array());
for(size_t i = 0; i < query_num; ++i ) {
if ( results[i].err_code == 0 ) {
records.add( new json_float( results[i].query_time.elapsed_time_msec()) );
++success_count;
}
}
result["success_num"] = new json_integer(success_count);
result["error_num"] = new json_integer(query_num-success_count);
result["records"] = records;
return result;
}
int menu()
{
records(); //leitura do arquivo onde estao as pontuacoes.
int opcao,i;
printf("********************************************************************************\n");
printf("\t ** * * * ** * * ** *** * * * * * * \n ");
printf("\t* * * ** * * * * * * * * * * * * * \n");
printf("\t* * * * * ** * * * * * * * * * * * \n");
printf("\t ** * * * * ** * ** * * * * * * * \n");
printf("********************************************************************************\n");
printf("\n\t\t\t Menu:\t \n\n \t\t\tJogar(1)\t \n \t\t\tSair(2)\t\n \t\t\tPontuacao(3)\t\n\t\t\tDicas(4)\t \n\n \t\t\tEscolha: ");
scanf("%d",&opcao);
switch(opcao) //opcoes do menu.
{
case 1: //começa o jogo.
printf("Informe seu nome: ");
scanf("%s", jogadores[5].nome);
printf("\n\n");
jogo(jogadores);
break;
case 2: //finaliza o jogo.
printf("\t\n Voce finalizou o jogo\n\n\tQue pena T-T\n\n");
return -1;
case 3: //mostra as pontuaçoes.
i = 0;
printf("\n\n\t\tMelhores pontuacoes:\n\n");
while (i < 5)
{
printf("\t\t%s\t:%d\n", jogadores[i].nome, jogadores[i].p);
i++;
}
printf("\n\n");
menu();
break;
case 4: //Mostra as instruçoes.
instrucoes();
break;
}
return 0;
}
int maximalRectangle(vector<vector<char>>& matrix) {
if(matrix.size() == 0)
return 0;
vector<vector<int>> records(matrix.size(), vector<int>(matrix[0].size(), 0));
for(int i = 0; i < matrix.size(); ++ i){
for(int j = 0; j < matrix[i].size(); ++ j){
if(matrix[i][j] == '0')
records[i][j] = 0;
else
records[i][j] = i > 0 ? records[i - 1][j] + 1 : 1;
}
}
int area = 0;
for(int i = 0; i < records.size(); ++ i){
int temp = maxArea(records[i]);
if(temp > area)
area = temp;
}
return area;
}
void MainWindow::createActions()
{
newGameChaosAct = gameMenu->addAction( tr("Нова гра (Хаос)") );
newGameStraightAct = gameMenu->addAction( tr("Нова гра (По прямій)") );
resetLevelAction = gameMenu->addAction( tr("Скинути поточний рівень") );
recordsAction = gameMenu->addAction( tr("Рекорди") );
gameMenu->addSeparator(); // Линия в меню
quitAction = gameMenu->addAction( tr("Вихід") );
helpAction = helpMenu->addAction( tr("Як грати?") );
helpMenu->addSeparator();
aboutAction = helpMenu->addAction( tr("Про гру") );
// hot keys
newGameChaosAct->setShortcut ( QKeySequence(Qt::CTRL | Qt::Key_C) );
newGameStraightAct->setShortcut ( QKeySequence(Qt::CTRL | Qt::Key_S) );
resetLevelAction->setShortcut ( QKeySequence(Qt::Key_Space) );
quitAction->setShortcut ( QKeySequence(Qt::CTRL | Qt::Key_Q) );
helpAction->setShortcut ( QKeySequence(Qt::Key_F1) );
// Connect
connect( newGameChaosAct, SIGNAL(triggered()),
this, SLOT( newGameChaos()));
connect( newGameStraightAct, SIGNAL(triggered()),
this, SLOT( newGameStraight()));
connect( resetLevelAction, SIGNAL(triggered()),
this, SLOT( resetGame()));
connect( recordsAction, SIGNAL(triggered()),
this, SLOT( records()));
connect( quitAction, SIGNAL(triggered()),
this, SLOT( close()));
connect( helpAction, SIGNAL(triggered()),
this, SLOT( manualGame()));
connect( aboutAction, SIGNAL(triggered()),
this, SLOT( about()));
}
std::vector<CountH5Record> count_placeholders(
const std::vector<std::pair<std::string, size_t>>& chromosome_lengths,
const std::string& cell_type,
const std::string& bam_file_path,
const unsigned int bin_size)
{
size_t num_records = 0;
for (auto& chr_length : chromosome_lengths)
{
int bins = std::ceil(chr_length.second / (double) bin_size);
num_records += bins;
}
const std::string bam_file_name = file_name_from_path(bam_file_path);
CountH5Record empty_record;
empty_record.bin_number = 0;
empty_record.count = 0;
strncpy(empty_record.cell_type, cell_type.c_str(), sizeof(CountH5Record::cell_type));
strncpy(empty_record.chromosome, "", sizeof(CountH5Record::chromosome));
strncpy(empty_record.file_name, bam_file_name.c_str(), sizeof(CountH5Record::file_name));
std::vector<CountH5Record> records(num_records, empty_record);
size_t i=0;
for (auto& chr_length : chromosome_lengths)
{
int bins = std::ceil(chr_length.second / (double) bin_size);
for (size_t j=0; j < bins; ++j, ++i)
{
records[i].bin_number = j;
strncpy(records[i].chromosome, chr_length.first.c_str(), sizeof(CountH5Record::chromosome));
}
}
return records;
}
void
ListView::answerReceived(int id, const QList<QByteArray>& answer)
{
/* We received something, so the sensor is probably ok. */
sensorError(id, false);
switch (id)
{
case 100: {
/* We have received the answer to a '?' command that contains
* the information about the table headers. */
if (answer.count() != 2)
{
kWarning(1215) << "wrong number of lines";
return;
}
KSGRD::SensorTokenizer headers(answer[0], '\t');
KSGRD::SensorTokenizer colTypes(answer[1], '\t');
/* add the new columns */
mModel.clear();
QStringList translatedHeaders;
for (uint i = 0; i < headers.count(); i++) {
translatedHeaders.append( i18nc("heading from daemon", headers[i]) );
}
for(uint i =0 ; i < colTypes.count(); i++) {
ColumnType type = convertColumnType(colTypes[i]);
mColumnTypes.append(type);
if (type == Text || type == DiskStat)
mModel.addColumnAlignment(Qt::AlignLeft);
else
mModel.addColumnAlignment(Qt::AlignRight);
}
mModel.setHorizontalHeaderLabels(translatedHeaders);
//If we have some header settings to restore, we can do so now
if(!mHeaderSettings.isEmpty()) {
mView->header()->restoreState(mHeaderSettings);
mModel.sort( mView->header()->sortIndicatorSection(), mView->header()->sortIndicatorOrder() );
}
break;
}
case 19: {
for (int i = 0; i < answer.count(); i++) {
KSGRD::SensorTokenizer records(answer[i], '\t');
for (uint j = 0; j < records.count() && j < mColumnTypes.count(); j++) {
QStandardItem *item = new QStandardItem();
item->setEditable(false);
switch( mColumnTypes[j] ) {
case Int:
item->setData(records[j].toLongLong(), Qt::UserRole);
item->setText(records[j]);
break;
case Percentage:
item->setData(records[j].toInt(), Qt::UserRole);
item->setText(records[j] + "%");
break;
case Float:
item->setData(records[j].toFloat(), Qt::DisplayRole);
item->setData(records[j].toFloat(), Qt::UserRole);
break;
case Time:
item->setData(QTime::fromString(records[j]), Qt::DisplayRole);
item->setData(QTime::fromString(records[j]), Qt::UserRole);
break;
case KByte: {
item->setData(records[j].toInt(), Qt::UserRole);
item->setText(formatByteSize(records[j].toLongLong(), mUnits));
break;
}
case DiskStat:
case Text:
default:
item->setText(records[j]);
item->setData(records[j], Qt::UserRole);
}
mModel.setItem(i, j, item);
}
}
mModel.setRowCount(answer.count());
mModel.sort( mView->header()->sortIndicatorSection(), mView->header()->sortIndicatorOrder() );
break;
}
}
}
void EvalDetectionLayer<Dtype>::Forward_cpu(
const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top)
{
//const Dtype* input_data = bottom[0]->cpu_data();// 网络输出 N*13*13*125
const Dtype* label_data = bottom[1]->cpu_data(); // 真实标签数据 N*30*5
//LOG(INFO) << bottom[0]->data_at(0,0,0,0) << " " << bottom[0]->data_at(0,0,0,1);
Blob<Dtype> swap;// 网络输出数据 N * 13* 13* 125
// 变形为 N * (13 * 13) * 5 * 25
// N*(5*(5+num_class_))*13*13 -> N * (13*13) * 5 * (5+num_class_)
swap.Reshape(bottom[0]->num(),
bottom[0]->height()*bottom[0]->width(),
num_object_,
bottom[0]->channels()/num_object_);
Dtype* swap_data = swap.mutable_cpu_data();// cpu上的数据
caffe_set(swap.count(), Dtype(0.0), swap_data);// 设置为0
int index = 0;
for (int b = 0; b < bottom[0]->num(); ++b)// 图片数量
for (int h = 0; h < bottom[0]->height(); ++h) // 格子 13
for (int w = 0; w < bottom[0]->width(); ++w)// 格子 13
for (int c = 0; c < bottom[0]->channels(); ++c)// 5*25=125
{
swap_data[index++] = bottom[0]->data_at(b,c,h,w);
}
//*******************************************************************************//
//caffe_set(swap.count(), Dtype(0.0), swap_data);// 设置为0
//int p_index = (7*13+4)*125;
//swap_data[p_index]=-0.1020;
//swap_data[p_index+1]=2.0867;
//swap_data[p_index+2]=1.612;
//swap_data[p_index+3]=1.0515;
//swap_data[p_index+4]=1.0;
//swap_data[p_index+5+11]=100;
//*******************************************************************************//
Dtype* top_data = top[0]->mutable_cpu_data();// 层输出 cpu数据
caffe_set(top[0]->count(), Dtype(0), top_data);// 设置为0
Dtype all_mAP = 0.0;// 总 batch 的 mAP
for (int i = 0; i < bottom[0]->num(); ++i)
{ // N 图片数量
int input_index = i * bottom[0]->count(1);// 网络输出标签 i * 13*13*125
int true_index = i * bottom[1]->count(1);// 真实标签 i * 30*5
int top_index = i * top[0]->count(1); // 输出数据 // i * ( 20 + 13*13*5*4 + 1) -> i * (13*13*5*4 + 1)
// 前面20个为 真实标签 物体类别出现的次数
// 获取真实边框 =========================================
map<int, vector<BoxData> > gt_boxes;
// 从 对应图片的标签数据中 获取 真实边框 label_ + score_ + box_
// 返回一张图像的 标签 + 多边框数据的 标签
GetGTBox(side_, label_data + true_index, >_boxes);
// 在输出数据中 记录 真实标签 物体类别出现的次数=======================
for (std::map<int, vector<BoxData > >::iterator it = gt_boxes.begin(); it != gt_boxes.end(); ++it)
{
// 遍历 每一个 真实的标签
//int label = it->first;// 标签 类别
vector<BoxData>& g_boxes = it->second;// BoxData: label_ + score_ + box_
for (int j = 0; j < g_boxes.size(); ++j)
{// 边框数量
// 输出数据中的前 20个================================================
// =======================================
// top_data[top_index + label] += 1; // 真实标签 物体类别出现的次数
}
}
// 获取预测边框 =============================================
map<int, vector<BoxData> > pred_boxes;
// 获取预测边框 13*13*5 -> NMS抑制+置信度抑制 -> pred_boxes(数量少很多)
//GetPredBox(side_, num_object_, num_class_, input_data + input_index, &pred_boxes, sqrt_, constriant_, score_type_, nms_);
GetPredBox(side_, num_object_, num_class_, swap_data + input_index, &pred_boxes, score_type_, nms_, biases_);
// 输出数据 后面 的 13*13*5*4 =============================
// int index = top_index + num_class_ + 1;// 20 + 1 之后为 上面的 (label + score + tp + fp) 参数
//=============================
int index = top_index + 1;// 1个 map 之后为 上面的 (label + score + tp + fp) 参数
int pred_count(0);//
Dtype mAP = 0.0;
int pre_clas_num=0;
//float AP = 0.0;
// int tp
// 遍历预测值的 每一类，与最合适的标签边框计算AP,在计算总类别的mAP============
for (std::map<int, vector<BoxData> >::iterator it = pred_boxes.begin(); it != pred_boxes.end(); ++it)
{
Dtype AP = 0.0;// 该类AP
int tp=0;// 预测正确
int fp=0;// 预测错误
++pre_clas_num;// 该图片预测的总类别数量
int label = it->first;// 预测 边框标签
vector<BoxData>& p_boxes = it->second;// 多个边框 vector<BoxData>
// 真实标签中 未找到该 类别=======================================
if (gt_boxes.find(label) == gt_boxes.end()) {
for (int b = 0; b < p_boxes.size(); ++b) {// 该类别下的每一个预测边框
top_data[index + pred_count * 4 + 0] = p_boxes[b].label_;// 标签
top_data[index + pred_count * 4 + 1] = p_boxes[b].score_;// 得分
top_data[index + pred_count * 4 + 2] = 0; //tp
top_data[index + pred_count * 4 + 3] = 1; //fp 错误的预测为正确的
++pred_count;
++fp;// 预测错误============================================
}
if(tp + fp)
AP = tp / (tp + fp);// 计算该类别的 平均准确度 这里等于0
mAP += AP;// 所有类别的总 AP 这里可以省略 因为 AP为0
continue;// 跳过预测错误的，只记录fp
}
// 真实标签找到了该预测的类别======================================
vector<BoxData>& g_boxes = gt_boxes[label];// 真实标签中该 类别的 多个真实边框=====
vector<bool> records(g_boxes.size(), false);// 记录 真实类别的每一个 物体边框 是否已经被预测过
for (int k = 0; k < p_boxes.size(); ++k)
{ // 遍历 每个 预测边框==============
top_data[index + pred_count * 4 + 0] = p_boxes[k].label_;// 标签
top_data[index + pred_count * 4 + 1] = p_boxes[k].score_;// 得分
Dtype max_iou(-1);// 预测边框最接近的 真实边框 iou
int idx(-1);// 对应的 真实边框id
// 遍历每个真实边框 找到 预测边框最接近的 真实边框===========
for (int g = 0; g < g_boxes.size(); ++g) {
Dtype iou = Calc_iou(p_boxes[k].box_, g_boxes[g].box_);// 计算交并比
if (iou > max_iou) {
max_iou = iou;// 记录 每个预测边框 最接近的 真实边框 的 IOU
idx = g;// 对应的 真实边框id
}
}
// 根据 交并比 确定判断 预测 正确/错误
if (max_iou >= threshold_) {
if ( !records[idx] ) {
records[idx] = true;// 对应的 真实边框id
top_data[index + pred_count * 4 + 2] = 1; // tp 正->正确
top_data[index + pred_count * 4 + 3] = 0; // fp
++tp;// 预测正确 =================================
}
else
{// 同一个位置的物体之前已经被预测过，又一个框来预测，则认为是错误的
top_data[index + pred_count * 4 + 2] = 0;
top_data[index + pred_count * 4 + 3] = 1;// 错误 -> 正确
++fp;// 预测错误============================================
}
}
++pred_count;
}
if(tp + fp) AP = tp / (tp + fp);// 计算该类别的 平均准确度
mAP += AP;// 所有类别的总 AP
}
if(pre_clas_num){
mAP /= pre_clas_num;
// mAP /= num_class_; // 计算 mAP 是除以总类别数，还是总预测类别数
}
else mAP = 0.0;
// 输出对应图片的mAP
top_data[ index - 1 ] = mAP;
all_mAP += mAP;
}
if(bottom[0]->num()) all_mAP /= (bottom[0]->num());
top_data[0] = all_mAP;
}
int methyltest_main(int argc, char** argv)
{
parse_methyltest_options(argc, argv);
omp_set_num_threads(opt::num_threads);
Fast5Map name_map(opt::reads_file);
ModelMap models = read_models_fofn(opt::models_fofn, mtest_alphabet);
// Open the BAM and iterate over reads
// load bam file
htsFile* bam_fh = sam_open(opt::bam_file.c_str(), "r");
assert(bam_fh != NULL);
// load bam index file
std::string index_filename = opt::bam_file + ".bai";
hts_idx_t* bam_idx = bam_index_load(index_filename.c_str());
assert(bam_idx != NULL);
// read the bam header
bam_hdr_t* hdr = sam_hdr_read(bam_fh);
// load reference fai file
faidx_t *fai = fai_load(opt::genome_file.c_str());
hts_itr_t* itr;
// If processing a region of the genome, only emit events aligned to this window
int clip_start = -1;
int clip_end = -1;
if(opt::region.empty()) {
// TODO: is this valid?
itr = sam_itr_queryi(bam_idx, HTS_IDX_START, 0, 0);
} else {
fprintf(stderr, "Region: %s\n", opt::region.c_str());
itr = sam_itr_querys(bam_idx, hdr, opt::region.c_str());
hts_parse_reg(opt::region.c_str(), &clip_start, &clip_end);
}
#ifndef H5_HAVE_THREADSAFE
if(opt::num_threads > 1) {
fprintf(stderr, "You enabled multi-threading but you do not have a threadsafe HDF5\n");
fprintf(stderr, "Please recompile nanopolish's built-in libhdf5 or run with -t 1\n");
exit(1);
}
#endif
// Initialize writers
OutputHandles handles;
handles.site_writer = fopen(std::string(opt::bam_file + ".methyltest.sites.bed").c_str(), "w");
handles.read_writer = fopen(std::string(opt::bam_file + ".methyltest.reads.tsv").c_str(), "w");
handles.strand_writer = fopen(std::string(opt::bam_file + ".methyltest.strand.tsv").c_str(), "w");
// Write a header to the reads.tsv file
fprintf(handles.read_writer, "name\tsum_ll_ratio\tn_cpg\tcomplement_model\ttags\n");
// strand header
fprintf(handles.strand_writer, "name\tsum_ll_ratio\tn_cpg\tmodel\n");
// Initialize iteration
std::vector<bam1_t*> records(opt::batch_size, NULL);
for(size_t i = 0; i < records.size(); ++i) {
records[i] = bam_init1();
}
int result;
size_t num_reads_processed = 0;
size_t num_records_buffered = 0;
Progress progress("[methyltest]");
do {
assert(num_records_buffered < records.size());
// read a record into the next slot in the buffer
result = sam_itr_next(bam_fh, itr, records[num_records_buffered]);
num_records_buffered += result >= 0;
// realign if we've hit the max buffer size or reached the end of file
if(num_records_buffered == records.size() || result < 0) {
#pragma omp parallel for
for(size_t i = 0; i < num_records_buffered; ++i) {
bam1_t* record = records[i];
size_t read_idx = num_reads_processed + i;
if( (record->core.flag & BAM_FUNMAP) == 0) {
calculate_methylation_for_read(models, name_map, fai, hdr, record, read_idx, handles);
}
}
num_reads_processed += num_records_buffered;
num_records_buffered = 0;
}
} while(result >= 0);
assert(num_records_buffered == 0);
progress.end();
// cleanup records
for(size_t i = 0; i < records.size(); ++i) {
bam_destroy1(records[i]);
}
// cleanup
fclose(handles.site_writer);
fclose(handles.read_writer);
fclose(handles.strand_writer);
sam_itr_destroy(itr);
bam_hdr_destroy(hdr);
fai_destroy(fai);
sam_close(bam_fh);
hts_idx_destroy(bam_idx);
return EXIT_SUCCESS;
}
void train_one_round(const Fast5Map& name_map, size_t round)
{
const PoreModelMap& current_models = PoreModelSet::get_models(opt::trained_model_type);
// Initialize the training summary stats for each kmer for each model
ModelTrainingMap model_training_data;
for(auto current_model_iter = current_models.begin(); current_model_iter != current_models.end(); current_model_iter++) {
// one summary entry per kmer in the model
std::vector<StateSummary> summaries(current_model_iter->second.get_num_states());
model_training_data[current_model_iter->first] = summaries;
}
// Open the BAM and iterate over reads
// load bam file
htsFile* bam_fh = sam_open(opt::bam_file.c_str(), "r");
assert(bam_fh != NULL);
// load bam index file
std::string index_filename = opt::bam_file + ".bai";
hts_idx_t* bam_idx = bam_index_load(index_filename.c_str());
assert(bam_idx != NULL);
// read the bam header
bam_hdr_t* hdr = sam_hdr_read(bam_fh);
// load reference fai file
faidx_t *fai = fai_load(opt::genome_file.c_str());
hts_itr_t* itr;
// If processing a region of the genome, only emit events aligned to this window
int clip_start = -1;
int clip_end = -1;
if(opt::region.empty()) {
// TODO: is this valid?
itr = sam_itr_queryi(bam_idx, HTS_IDX_START, 0, 0);
} else {
fprintf(stderr, "Region: %s\n", opt::region.c_str());
itr = sam_itr_querys(bam_idx, hdr, opt::region.c_str());
hts_parse_reg(opt::region.c_str(), &clip_start, &clip_end);
}
#ifndef H5_HAVE_THREADSAFE
if(opt::num_threads > 1) {
fprintf(stderr, "You enabled multi-threading but you do not have a threadsafe HDF5\n");
fprintf(stderr, "Please recompile nanopolish's built-in libhdf5 or run with -t 1\n");
exit(1);
}
#endif
// Initialize iteration
std::vector<bam1_t*> records(opt::batch_size, NULL);
for(size_t i = 0; i < records.size(); ++i) {
records[i] = bam_init1();
}
int result;
size_t num_reads_realigned = 0;
size_t num_records_buffered = 0;
Progress progress("[methyltrain]");
do {
assert(num_records_buffered < records.size());
// read a record into the next slot in the buffer
result = sam_itr_next(bam_fh, itr, records[num_records_buffered]);
num_records_buffered += result >= 0;
// realign if we've hit the max buffer size or reached the end of file
if(num_records_buffered == records.size() || result < 0) {
#pragma omp parallel for
for(size_t i = 0; i < num_records_buffered; ++i) {
bam1_t* record = records[i];
size_t read_idx = num_reads_realigned + i;
if( (record->core.flag & BAM_FUNMAP) == 0) {
add_aligned_events(name_map, fai, hdr, record, read_idx, clip_start, clip_end, round, model_training_data);
}
}
num_reads_realigned += num_records_buffered;
num_records_buffered = 0;
}
if(opt::progress) {
fprintf(stderr, "Realigned %zu reads in %.1lfs\r", num_reads_realigned, progress.get_elapsed_seconds());
}
} while(result >= 0);
assert(num_records_buffered == 0);
progress.end();
// open the summary file
std::stringstream summary_fn;
summary_fn << "methyltrain" << opt::out_suffix << ".summary";
FILE* summary_fp = fopen(summary_fn.str().c_str(), "w");
fprintf(summary_fp, "model_short_name\tkmer\tnum_matches\tnum_skips\t"
"num_stays\tnum_events_for_training\twas_trained\t"
"trained_level_mean\ttrained_level_stdv\n");
// open the tsv file with the raw training data
std::stringstream training_fn;
training_fn << "methyltrain" << opt::out_suffix << ".round" << round << ".events.tsv";
std::ofstream training_ofs(training_fn.str());
// write out a header for the training data
StateTrainingData::write_header(training_ofs);
// iterate over models: template, complement_pop1, complement_pop2
for(auto model_training_iter = model_training_data.begin();
model_training_iter != model_training_data.end(); model_training_iter++) {
// Initialize the trained model from the input model
auto current_model_iter = current_models.find(model_training_iter->first);
assert(current_model_iter != current_models.end());
std::string model_name = model_training_iter->first;
std::string model_short_name = current_model_iter->second.metadata.get_short_name();
// Initialize the new model from the current model
PoreModel updated_model = current_model_iter->second;
uint32_t k = updated_model.k;
const std::vector<StateSummary>& summaries = model_training_iter->second;
// Generate the complete set of kmers
std::string gen_kmer(k, 'A');
std::vector<std::string> all_kmers;
for(size_t ki = 0; ki < summaries.size(); ++ki) {
all_kmers.push_back(gen_kmer);
mtrain_alphabet->lexicographic_next(gen_kmer);
}
assert(gen_kmer == std::string(k, 'A'));
assert(all_kmers.front() == std::string(k, 'A'));
assert(all_kmers.back() == std::string(k, 'T'));
// Update means for each kmer
#pragma omp parallel for
for(size_t ki = 0; ki < summaries.size(); ++ki) {
assert(ki < all_kmers.size());
std::string kmer = all_kmers[ki];
// write the observed values to a tsv file
#pragma omp critical
{
for(size_t ei = 0; ei < summaries[ki].events.size(); ++ei) {
summaries[ki].events[ei].write_tsv(training_ofs, model_short_name, kmer);
}
}
bool is_m_kmer = kmer.find('M') != std::string::npos;
bool update_kmer = opt::training_target == TT_ALL_KMERS ||
(is_m_kmer && opt::training_target == TT_METHYLATED_KMERS) ||
(!is_m_kmer && opt::training_target == TT_UNMETHYLATED_KMERS);
bool trained = false;
// only train if there are a sufficient number of events for this kmer
if(update_kmer && summaries[ki].events.size() >= opt::min_number_of_events_to_train) {
// train a mixture model where a minority of k-mers aren't methylated
ParamMixture mixture;
float incomplete_methylation_rate = 0.05f;
std::string um_kmer = mtrain_alphabet->unmethylate(kmer);
size_t um_ki = mtrain_alphabet->kmer_rank(um_kmer.c_str(), k);
// Initialize the training parameters. If this is a kmer containing
// a methylation site we train a two component mixture, otherwise
// just fit a gaussian
float major_weight = is_m_kmer ? 1 - incomplete_methylation_rate : 1.0f;
mixture.log_weights.push_back(log(major_weight));
mixture.params.push_back(current_model_iter->second.get_parameters(ki));
if(is_m_kmer) {
// add second unmethylated component
mixture.log_weights.push_back(std::log(incomplete_methylation_rate));
mixture.params.push_back(current_model_iter->second.get_parameters(um_ki));
}
if(opt::verbose > 1) {
fprintf(stderr, "INIT__MIX %s\t%s\t[%.2lf %.2lf %.2lf]\t[%.2lf %.2lf %.2lf]\n", model_training_iter->first.c_str(), kmer.c_str(),
std::exp(mixture.log_weights[0]), mixture.params[0].level_mean, mixture.params[0].level_stdv,
std::exp(mixture.log_weights[1]), mixture.params[1].level_mean, mixture.params[1].level_stdv);
}
ParamMixture trained_mixture = train_gaussian_mixture(summaries[ki].events, mixture);
if(opt::verbose > 1) {
fprintf(stderr, "TRAIN_MIX %s\t%s\t[%.2lf %.2lf %.2lf]\t[%.2lf %.2lf %.2lf]\n", model_training_iter->first.c_str(), kmer.c_str(),
std::exp(trained_mixture.log_weights[0]), trained_mixture.params[0].level_mean, trained_mixture.params[0].level_stdv,
std::exp(trained_mixture.log_weights[1]), trained_mixture.params[1].level_mean, trained_mixture.params[1].level_stdv);
}
#pragma omp critical
updated_model.states[ki] = trained_mixture.params[0];
if (model_stdv()) {
ParamMixture ig_mixture;
// weights
ig_mixture.log_weights = trained_mixture.log_weights;
// states
ig_mixture.params.emplace_back(trained_mixture.params[0]);
if(is_m_kmer) {
ig_mixture.params.emplace_back(current_model_iter->second.get_parameters(um_ki));
}
// run training
auto trained_ig_mixture = train_invgaussian_mixture(summaries[ki].events, ig_mixture);
LOG("methyltrain", debug)
<< "IG_INIT__MIX " << model_training_iter->first.c_str() << " " << kmer.c_str() << " ["
<< std::fixed << std::setprecision(5) << ig_mixture.params[0].sd_mean << " "
<< ig_mixture.params[1].sd_mean << "]" << std::endl
<< "IG_TRAIN_MIX " << model_training_iter->first.c_str() << " " << kmer.c_str() << " ["
<< trained_ig_mixture.params[0].sd_mean << " "
<< trained_ig_mixture.params[1].sd_mean << "]" << std::endl;
// update state
#pragma omp critical
{
updated_model.states[ki] = trained_ig_mixture.params[0];
}
}
trained = true;
}
#pragma omp critical
{
fprintf(summary_fp, "%s\t%s\t%d\t%d\t%d\t%zu\t%d\t%.2lf\t%.2lf\n",
model_short_name.c_str(), kmer.c_str(),
summaries[ki].num_matches, summaries[ki].num_skips, summaries[ki].num_stays,
summaries[ki].events.size(), trained, updated_model.states[ki].level_mean, updated_model.states[ki].level_stdv);
}
// add the updated model into the collection (or replace what is already there)
PoreModelSet::insert_model(opt::trained_model_type, updated_model);
}
}
// cleanup records
for(size_t i = 0; i < records.size(); ++i) {
bam_destroy1(records[i]);
}
// cleanup
sam_itr_destroy(itr);
bam_hdr_destroy(hdr);
fai_destroy(fai);
sam_close(bam_fh);
hts_idx_destroy(bam_idx);
fclose(summary_fp);
}
rsurf_iso_contouring::rsurf_iso_contouring(const rsurf_data& data,const std::size_t& rsIndex)
:iso_contouring_data(new t_iso_contouring_data())
{
//Initialisation
std::size_t tsSize(0);
std::size_t nodesSize(0);
std::size_t nbtimestep(0);
std::size_t nbfaces(0);
std::size_t idstep(0);
std::size_t vertices[3]={0,0,0};
std::size_t records(0);
float timestep(0.f);
float energy(0.f);
float totenergy(0.f);
std::string rs_name;
std::string recordType;
int rs_xmlid(0);
bool converttodb=false;
data.GetRsInfo(rsIndex,nbfaces,rs_name,rs_xmlid);
data.GetFileInfos(tsSize,nodesSize,nbtimestep,timestep,recordType);
if(recordType=="SPL_STANDART" || recordType=="SPL_GAIN")
converttodb=true;
//Passage de valeur de surface vers des valeurs de sommets
std::vector<float> vertices_lvl(nodesSize,0.f);
std::vector<unsigned int> vertices_countlinkedfaces(nodesSize,0);
std::vector<bool> face_useforcalculation(nbfaces,true);
for(std::size_t idface=0;idface<nbfaces;idface++)
{
data.GetFaceInfo(rsIndex,idface,vertices[0],vertices[1],vertices[2],records);
//Cumul de l'energie sur le temps
totenergy=0;
for(std::size_t idrecord=0;idrecord<records;idrecord++)
{
data.GetFaceEnergy(rsIndex,idface,idrecord,idstep,energy);
totenergy+=energy;
}
if(totenergy==0 || totenergy==std::numeric_limits<float>::infinity() || -totenergy==std::numeric_limits<float>::infinity())
{
face_useforcalculation[idface]=false;
}else{
vertices_countlinkedfaces[vertices[0]]+=1;
vertices_countlinkedfaces[vertices[1]]+=1;
vertices_countlinkedfaces[vertices[2]]+=1;
vertices_lvl[vertices[0]]+=totenergy;
vertices_lvl[vertices[1]]+=totenergy;
vertices_lvl[vertices[2]]+=totenergy;
}
}
//Division des valeurs par leurs nombre de contributeurs et conversion en dB
for(std::size_t idvert=0;idvert<nodesSize;idvert++)
{
if(vertices_countlinkedfaces[idvert]>0)
vertices_lvl[idvert]/=vertices_countlinkedfaces[idvert];
if(converttodb && vertices_lvl[idvert]>0)
vertices_lvl[idvert]=10*log10(vertices_lvl[idvert]/pow(10.f,-12.f));
}
//Initialisation des variables de la classe
iso_contouring_data->minValue=vertices_lvl[vertices[0]];
iso_contouring_data->maxValue=iso_contouring_data->minValue;
vec3 A,B,C;
for(std::size_t idface=0;idface<nbfaces;idface++)
{
data.GetFaceInfo(rsIndex,idface,vertices[0],vertices[1],vertices[2],records);
data.GetNodePositionValue(vertices[0],A.x,A.y,A.z);
data.GetNodePositionValue(vertices[1],B.x,B.y,B.z);
data.GetNodePositionValue(vertices[2],C.x,C.y,C.z);
t_tri_data newtri(A,B,C,vertices_lvl[vertices[0]],vertices_lvl[vertices[1]],vertices_lvl[vertices[2]]);
newtri.useInIsoLines=face_useforcalculation[idface];
iso_contouring_data->tri_grid.push_back(newtri);
FINDMINMAX_RSURF(vertices_lvl[vertices[0]],vertices_lvl[vertices[1]],vertices_lvl[vertices[2]],iso_contouring_data->minValue,iso_contouring_data->maxValue);
}
}
int scorereads_main(int argc, char** argv)
{
parse_scorereads_options(argc, argv);
omp_set_num_threads(opt::num_threads);
Fast5Map name_map(opt::reads_file);
ModelMap models;
if (!opt::models_fofn.empty())
models = read_models_fofn(opt::models_fofn);
// Open the BAM and iterate over reads
// load bam file
htsFile* bam_fh = sam_open(opt::bam_file.c_str(), "r");
assert(bam_fh != NULL);
// load bam index file
std::string index_filename = opt::bam_file + ".bai";
hts_idx_t* bam_idx = bam_index_load(index_filename.c_str());
assert(bam_idx != NULL);
// read the bam header
bam_hdr_t* hdr = sam_hdr_read(bam_fh);
// load reference fai file
faidx_t *fai = fai_load(opt::genome_file.c_str());
hts_itr_t* itr;
// If processing a region of the genome, only emit events aligned to this window
int clip_start = -1;
int clip_end = -1;
if(opt::region.empty()) {
// TODO: is this valid?
itr = sam_itr_queryi(bam_idx, HTS_IDX_START, 0, 0);
} else {
fprintf(stderr, "Region: %s\n", opt::region.c_str());
itr = sam_itr_querys(bam_idx, hdr, opt::region.c_str());
hts_parse_reg(opt::region.c_str(), &clip_start, &clip_end);
}
#ifndef H5_HAVE_THREADSAFE
if(opt::num_threads > 1) {
fprintf(stderr, "You enabled multi-threading but you do not have a threadsafe HDF5\n");
fprintf(stderr, "Please recompile nanopolish's built-in libhdf5 or run with -t 1\n");
exit(1);
}
#endif
// Initialize iteration
std::vector<bam1_t*> records(opt::batch_size, NULL);
for(size_t i = 0; i < records.size(); ++i) {
records[i] = bam_init1();
}
int result;
size_t num_reads_realigned = 0;
size_t num_records_buffered = 0;
do {
assert(num_records_buffered < records.size());
// read a record into the next slot in the buffer
result = sam_itr_next(bam_fh, itr, records[num_records_buffered]);
num_records_buffered += result >= 0;
// realign if we've hit the max buffer size or reached the end of file
if(num_records_buffered == records.size() || result < 0) {
#pragma omp parallel for schedule(dynamic)
for(size_t i = 0; i < num_records_buffered; ++i) {
bam1_t* record = records[i];
size_t read_idx = num_reads_realigned + i;
if( (record->core.flag & BAM_FUNMAP) == 0) {
//load read
std::string read_name = bam_get_qname(record);
std::string fast5_path = name_map.get_path(read_name);
SquiggleRead sr(read_name, fast5_path);
// TODO: early exit when have processed all of the reads in readnames
if (!opt::readnames.empty() &&
std::find(opt::readnames.begin(), opt::readnames.end(), read_name) == opt::readnames.end() )
continue;
for(size_t strand_idx = 0; strand_idx < NUM_STRANDS; ++strand_idx) {
std::vector<EventAlignment> ao = alignment_from_read(sr, strand_idx, read_idx,
models, fai, hdr,
record, clip_start, clip_end);
if (ao.size() == 0)
continue;
// Update pore model based on alignment
if ( opt::calibrate )
recalibrate_model(sr, strand_idx, ao, false);
double score = model_score(sr, strand_idx, fai, ao, 500);
if (score > 0)
continue;
#pragma omp critical(print)
std::cout << read_name << " " << ( strand_idx ? "complement" : "template" )
<< " " << sr.pore_model[strand_idx].name << " " << score << std::endl;
}
}
}
num_reads_realigned += num_records_buffered;
num_records_buffered = 0;
}
} while(result >= 0);
// cleanup records
for(size_t i = 0; i < records.size(); ++i) {
bam_destroy1(records[i]);
}
// cleanup
sam_itr_destroy(itr);
bam_hdr_destroy(hdr);
fai_destroy(fai);
sam_close(bam_fh);
hts_idx_destroy(bam_idx);
return 0;
}
