int main(int argc, char **argv) {
try {
// write
std::map<std::string, int> m = { {"a",1}, {"b",2} };
std::map<std::string, std::vector<double>> mv = { {"a",{1.0, 2.0}}, {"b",{2.0, 3.0, 4.0}} };
{
h5::file file1("test_map.h5", H5F_ACC_TRUNC);
h5::group top1(file1);
h5_write(top1, "map_int", m);
h5_write(top1, "map_vec", mv);
}
// read
std::map<std::string, int> mm = { {"c",1} };
std::map<std::string, std::vector<double>> mmv = { {"c",{1.0}} };
h5::file file2("test_map.h5", H5F_ACC_RDONLY);
h5::group top2(file2);
h5_read(top2, "map_int", mm);
h5_read(top2, "map_vec", mmv);
for (auto const & val: mm) std::cout << val.first << " " << val.second << std::endl;
for (auto const & val: mmv) {
std::cout << val.first << std::endl;
for (auto const & x: val.second) std::cout << x << std::endl;
}
}
TRIQS_CATCH_AND_ABORT;
}
template <typename G> static void read(h5::group gr, G &g) {
h5_read(gr, "data", g._data);
h5_read(gr, "singularity", g._singularity);
h5_read(gr, "mesh", g._mesh);
h5_read(gr, "symmetry", g._symmetry);
h5_read(gr, "indices", g._indices);
}
void h5_read ( h5::group F, std::string const & subgroup_name, parameters & p){
auto gr = F.open_group(subgroup_name);
std::vector<std::string> ds_name = F.get_all_dataset_names(subgroup_name), grp_name = F.get_all_subgroup_names(subgroup_name);
for (auto & x : grp_name) {
//std::cerr << " loading group : "<< x <<std::endl;
auto x_grp = gr.open_group(x);
auto triqs_data_scheme = x_grp.read_triqs_hdf5_data_scheme();
if (triqs_data_scheme != "") {
auto type_hash = _object::h5_scheme_to_code[triqs_data_scheme];
auto it = _object::code_to_h5_read_fnts.find(type_hash);
if (it == _object::code_to_h5_read_fnts.end()) TRIQS_RUNTIME_ERROR << "TRIQS_HDF5_data_scheme : ["<< triqs_data_scheme << "] is unknown. Did you register your object ?";
p[x] = it->second(gr,x);
}
else {
parameters p2;
h5_read (gr,x,p2);
p[x] = p2;
}
}
for (auto & x : ds_name) {
//std::cerr << " loading : "<< x <<std::endl;
try {
_object obj;
h5_read(gr,x,obj);
p[x] = obj;
}
catch(H5::Exception const & e) { TRIQS_RUNTIME_ERROR<< "Cannot load "<< x<<"\n H5 error is : \n "<< e.getCDetailMsg();}
}
}
/// Read from HDF5
friend void h5_read(h5::group fg, std::string subgroup_name, matsubara_domain &d) {
h5::group gr = fg.open_group(subgroup_name);
double beta;
std::string statistic;
h5_read(gr, "beta", beta);
h5_read(gr, "statistic", statistic);
d = matsubara_domain(beta, (statistic == "F" ? Fermion : Boson));
}
void h5_read(triqs::h5::group fg, std::string subgroup_name, block_matrix<T> & c){
triqs::h5::group gr = fg.open_group(subgroup_name);
std::vector<std::string> block_names;
std::vector<matrix<T>> matrix_vec;
h5_read(gr,"block_names",block_names);
h5_read(gr,"matrix_vec",matrix_vec);
c = block_matrix<T>(block_names, matrix_vec);
}
/// Read from HDF5
friend void h5_read(h5::group fg, std::string subgroup_name, matsubara_freq_mesh &m) {
h5::group gr = fg.open_group(subgroup_name);
typename matsubara_freq_mesh::domain_t dom;
int L;
bool s = true;
h5_read(gr, "domain", dom);
h5_read(gr, "size", L);
if (gr.has_key("start_at_0")) h5_read(gr, "start_at_0", s);
m = matsubara_freq_mesh{std::move(dom), L, s};
}
/// Read from HDF5
friend void h5_read(h5::group fg, std::string subgroup_name, linear_mesh &m) {
h5::group gr = fg.open_group(subgroup_name);
typename linear_mesh::domain_t dom;
double a, b;
long L;
h5_read(gr, "domain", dom);
h5_read(gr, "min", a);
h5_read(gr, "max", b);
h5_read(gr, "size", L);
m = linear_mesh(std::move(dom), a, b, L);
}
/// Read from HDF5
friend void h5_read(h5::group fg, std::string subgroup_name, gf_mesh &m) {
h5::group gr = fg.open_group(subgroup_name);
typename gf_mesh::domain_t dom;
long L;
int s = 1;
h5_read(gr, "domain", dom);
h5_read(gr, "size", L);
// backward compatibility : older file do not have this flags, default is true.
if (gr.has_key("positive_freq_only")) h5_read(gr, "positive_freq_only", s);
if (gr.has_key("start_at_0")) h5_read(gr, "start_at_0", s);
m = gf_mesh{std::move(dom), L, (s==1)};
}
/// HDF5 interface
friend void h5_read (h5::group g, std::string const & name, mc_generic & mc){
auto gr = g.open_group(name);
h5_read(gr,"moves", mc.AllMoves);
h5_read(gr,"measures", mc.AllMeasures);
h5_read(gr,"length_monte_carlo_cycle", mc.Length_MC_Cycle);
h5_read(gr,"number_cycle_requested", mc.NCycles);
h5_read(gr,"number_warming_cycle_requested", mc.NWarmIterations);
h5_read(gr,"number_cycle_done", mc.NC);
h5_read(gr,"number_measure_done", mc.nmeasures);
h5_read(gr,"sign", mc.sign);
h5_read(gr,"sum_sign", mc.sum_sign);
}
TEST(Gf, ArrayOf) {
triqs::clef::placeholder<0> w_;
auto agf = array<gf<imfreq>, 2>{2, 3};
auto bgf = array<gf<imfreq>, 2>{2, 3};
agf() = gf<imfreq>{{10.0, Fermion}, {1, 1}};
agf(0, 0)(w_) << 1 / (w_ + 2);
// test H5
{
h5::file file("ess_array_gf.h5", H5F_ACC_TRUNC);
h5_write(file, "Agf", agf);
}
{
h5::file file("ess_array_gf.h5", H5F_ACC_RDONLY);
h5_read(file, "Agf", bgf);
}
{
h5::file file("ess_array_gf2.h5", H5F_ACC_TRUNC);
h5_write(file, "Agf", bgf);
}
for (int i = 0; i < 2; ++i)
for (int j = 0; j < 2; ++j) EXPECT_GF_NEAR(agf(i, j), bgf(i, j));
}
template <bool IsView> static void read(h5::group gr, gf_impl<block_index, Target, nothing, void, IsView, false> &g) {
auto block_names = h5::h5_read<std::vector<std::string>> (gr, "block_names");
g._mesh = gf_mesh<block_index>(block_names);
//auto check_names = gr.get_all_subgroup_names();
// sort both and check ?
g._data.resize(g._mesh.size());
for (size_t i = 0; i < g.mesh().size(); ++i) h5_read(gr, g.mesh().domain().names()[i], g._data[i]);
}
template <bool IsView> static void read(h5::group gr, gf_impl<block_index, Target, Opt, IsView, false> &g) {
// does not work : need to read the block name and remake the mesh...
g._mesh = gf_mesh<block_index, Opt>(gr.get_all_subgroup_names());
g._data.resize(g._mesh.size());
// if (g._data.size() != g._mesh.size()) TRIQS_RUNTIME_ERROR << "h5 read block gf : number of block mismatch";
for (size_t i = 0; i < g.mesh().size(); ++i) h5_read(gr, g.mesh().domain().names()[i], g._data[i]);
// h5_read(gr,"symmetry",g._symmetry);
}
std::c14::enable_if_t<is_amv_value_or_view_class<ArrayType>::value && !has_scalar_or_string_value_type<ArrayType>::value>
h5_read(h5::group gr, std::string name, ArrayType& a) {
static_assert(!std::is_const<ArrayType>::value, "Cannot read in const object");
auto gr2 = gr.open_group(name);
// TODO checking scheme...
// load the shape
auto sha2 = a.shape();
array<int, 1> sha;
h5_read(gr2, "shape", sha);
if (first_dim(sha) != sha2.size())
TRIQS_RUNTIME_ERROR << " array<array<...>> load : rank mismatch. Expected " << sha2.size()<< " Got " << first_dim(sha);
for (int u = 0; u < sha2.size(); ++u) sha2[u] = sha(u);
if (a.shape() != sha2) a.resize(sha2);
#ifndef __cpp_generic_lambdas
foreach(a, h5_impl::_load_lambda<ArrayType>{a, gr2});
#else
foreach(a, [&](auto... is) { h5_read(gr2, h5_impl::_h5_name(is...), a(is...)); });
#endif
}
template <typename G > static void read(h5::group gr, G&g) {
auto block_names = h5::h5_read<std::vector<std::string>>(gr, "block_names");
auto m = gf_mesh<block_index>(block_names);
g._mesh = gf_mesh<block_index2>({m, m});
int s = m.size();
g._data.resize(s);
for (int i = 0; i < s; ++i) {
g._data[i].resize(s);
for (int j = 0; j < s; ++j) h5_read(gr, m.domain().names()[i] + "_" + m.domain().names()[j], g._data[i][j]);
}
}
TEST(GfCartesian, H5_RW_EvaluatorM) {
double beta = 1;
auto g = gf<cartesian_product<imfreq, imfreq>, matrix_valued>{{{beta, Fermion, 5}, {beta, Boson, 5}}, {1, 1}};
g() = 2;
h5::file file("g_nu_nuph5", H5F_ACC_TRUNC);
h5_write(file, "g", g);
gf<cartesian_product<imfreq, imfreq>, matrix_valued> g2{};
h5_read(file, "g", g2);
EXPECT_ARRAY_NEAR(g.data(), g2.data());
//EXPECT_GF_NEAR(g, g2);
auto w0 = matsubara_freq(0, beta, Fermion);
auto W10 = matsubara_freq(10, beta, Boson);
auto W0 = matsubara_freq(0, beta, Boson);
EXPECT_ARRAY_NEAR(g(w0, W0), g2(w0, W0));
EXPECT_ARRAY_NEAR(g(w0, W10), g2(w0, W10));
}
int
main(int argc, char **argv)
{
double starttime, endtime;
int ntasks, myrank;
char name[128];
int namelen;
MPI_Status status;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
MPI_Comm_size(MPI_COMM_WORLD, &ntasks);
MPI_Get_processor_name(name,&namelen);
/* Get a few OpenMP parameters. */
int O_P = omp_get_num_procs(); /* get number of OpenMP processors */
int O_T = omp_get_num_threads(); /* get number of OpenMP threads */
int O_ID = omp_get_thread_num(); /* get OpenMP thread ID */
//printf("name:%s M_ID:%d O_ID:%d O_P:%d O_T:%d\n", name,myrank,O_ID,O_P,O_T);
FILE *f;
char line[LINE_SIZE];
int numlines = 0;
exprinfo *exprannot = NULL;
char **glines = NULL;
f = fopen("gene_list.txt", "r");
while(fgets(line, LINE_SIZE, f)) {
glines = (char**)realloc(glines, sizeof(char*)*(numlines+1));
glines[numlines] = strdup(line);
char *pch = strtok (line,",");
char * gene = pch;
pch = strtok (NULL, ",");
int chr = atoi(trimwhitespace(pch));
exprannot = (exprinfo*)realloc(exprannot,sizeof(exprinfo)*(numlines+1));
exprannot[numlines].gene = strdup(gene);
exprannot[numlines].chr = chr;
if (!exprannot) printf("not allcoated\n");
numlines++;
}
fclose(f);
f = fopen("probe_id_mapping.txt", "r");
numlines = 0;
free(glines[0]);
free(glines);
probeinfo *records = NULL;
char **lines = NULL;
while(fgets(line, LINE_SIZE, f)) {
lines = (char**)realloc(lines, sizeof(char*)*(numlines+1));
lines[numlines] = strdup(line);
char *pch = strtok (line,",");
int probeid = atoi(pch);
pch = strtok (NULL, ",");
char * gene = pch;
pch = strtok (NULL, ",");
int chr = atoi(trimwhitespace(pch));
records = (probeinfo*)realloc(records,sizeof(probeinfo)*(numlines+1));
records[numlines].probeid = probeid;
records[numlines].chr = chr;
records[numlines].gene = strdup(gene);
if (!records) printf("not allcoated\n");
numlines++;
}
free(lines[0]);
free(lines);
fclose(f);
int NUM_GENES = numlines;
unsigned long x_nr, x_nc, y_nr, y_nc;
double **X = h5_read("x.h5", 1, "/X", &x_nr, &x_nc);
double **Y = h5_read("filtered_probes.h5", 1, "/FilteredProbes", &y_nr, &y_nc);
//printf("loaded X, num rows = %d, num cols = %d\n", x_nr, x_nc);
//printf("loaded Y, num rows = %d, num cols = %d\n", y_nr, y_nc);
unsigned long total_mem = (x_nr * y_nc);
double **RHO = create2dArray(x_nr, y_nc);
int gene, probe, tid, work_completed;
work_completed = 0;
int BLOCK_SIZE = NUM_ROWS/ntasks;
int offset = myrank*BLOCK_SIZE;
int STOP_IDX = offset+BLOCK_SIZE;
if (NUM_ROWS - STOP_IDX < BLOCK_SIZE)
STOP_IDX = NUM_ROWS;
// printf("offset = %d, for rank %d, with ntasks = %d, and BLOCK_SIZE = %d, STOP_IDX = %d\n",
// offset, myrank, ntasks, BLOCK_SIZE, STOP_IDX);
int num_sig_found = 0;
starttime = MPI_Wtime();
#pragma omp parallel \
for shared(X, Y, RHO, BLOCK_SIZE, offset, work_completed) \
private(probe,gene, tid)
for (gene = offset; gene < STOP_IDX; gene++) {
for (probe = 0; probe < NUM_COLS; probe++) {
double *x = getrowvec(X, gene, BUFFER_SIZE);
double *y = getcolvec(Y, probe, BUFFER_SIZE);
double avgx = mean(x, BUFFER_SIZE);
double * xcentered = sub(x, avgx, BUFFER_SIZE);
double avgy = mean(y, BUFFER_SIZE);
double * ycentered = sub(y, avgy, BUFFER_SIZE);
double * prod_result = prod(xcentered, ycentered, BUFFER_SIZE);
double sum_prod = sum(prod_result, BUFFER_SIZE);
double stdX = stddev(x, avgx, BUFFER_SIZE);
double stdY = stddev(y, avgy, BUFFER_SIZE);
double rho = sum_prod/((BUFFER_SIZE-1)*(stdX*stdY));
RHO[gene][probe] = rho;
if (work_completed % 10000 == 0) {
tid = omp_get_thread_num();
printf("rank = %d, work = %d, result[%d,%d] from %d = %f\n", myrank, work_completed,
gene, probe, tid, rho);
}
work_completed++;
free(x);
free(y);
free(xcentered);
free(ycentered);
free(prod_result);
}
}
//printf("********* %d FINISHED **********\n", myrank);
//f = fopen("significant.txt", "a");
#pragma omp parallel for shared(RHO,exprannot, records)
for (int i = 0; i < NUM_ROWS; i++) {
for (int j = 0; j < NUM_COLS; j++) {
double zscore = ztest(RHO[i][j],BUFFER_SIZE);
/*
if (zscore > 5.0) {
fprintf(f, "%d,%d,%f,%f,%d,%s,%d,%s\n",
i, j, zscore, RHO[i][j], records[j].chr, records[j].gene, exprannot[i].chr,exprannot[i].gene);
if (i*j%1000 == 0)
printf("%d,%d,%f,%f,%d,%s,%d,%s\n",
i, j, zscore, RHO[i][j], records[j].chr, records[j].gene, exprannot[i].chr,exprannot[i].gene);
}
*/
}
}
//fclose(f);
endtime = MPI_Wtime();
free(X[0]);
free(X);
free(Y[0]);
free(Y);
printf("rank %d - elapse time - %f\n",myrank, endtime-starttime);
//for (int i = 0; i < NUM_ROWS; i++) {
//printf("%s,%d\n", exprannot[i].gene, exprannot[i].chr);
//}
//printf("rank %d FINISHED\n",myrank);
//h5_write(RHO, NUM_ROWS, NUM_COLS, "rho_omp.h5", "/rho");
free(RHO[0]);
free(exprannot);
free(records);
free(RHO);
MPI_Finalize();
return 0;
}
friend void h5_read (h5::group g, std::string const & name, measure_set & ms) {
auto gr = g.open_group(name);
for (auto & p : ms.m_map) h5_read(gr,p.first, p.second);
}
/// Read from HDF5
friend void h5_read (tqa::h5::group_or_file fg, std::string subgroup_name, discrete_mesh & m){
tqa::h5::group_or_file gr = fg.open_group(subgroup_name);
typename discrete_mesh::domain_t dom;
h5_read(gr,"domain",m._dom);
m = discrete_mesh(std::move(dom));
}
/// Read from HDF5
friend void h5_read(h5::group fg, std::string subgroup_name, gf_mesh &m) {
h5::group gr = fg.open_group(subgroup_name);
auto l = [gr](int N, auto &m) { h5_read(gr, "MeshComponent" + std::to_string(N), m); };
triqs::tuple::for_each_enumerate(m.components(), l);
}
// A generic read
template <typename T> T h5_read(group gr, std::string const &name) {
if constexpr (std::is_default_constructible_v<T>) {
T x;
h5_read(gr, name, x);
return x;
} else {
void read_array(h5::group g, std::string const& name, arrays::vector<std::string>& V) {
std::vector<std::string> tmp;
h5_read(g, name, tmp);
V.resize(tmp.size());
std::copy(begin(tmp), end(tmp), begin(V));
}
friend void h5_read(h5::group fg, std::string subgroup_name, tail_impl &t) {
auto gr = fg.open_group(subgroup_name);
h5_read(gr, "omin", t.omin);
h5_read(gr, "mask", t._mask);
h5_read(gr, "data", t._data);
}
template<typename T> h5_rw_lambda_t make_h5_read_impl(T * p, std::true_type) {
return [p](h5::group F, std::string const &Name) {h5_read(F,Name, *p);};
}
template <typename... Is> void operator()(Is const&... is) { h5_read(g, _h5_name(is...), a(is...)); }
/// Read from HDF5
friend void h5_read(h5::group fg, std::string subgroup_name, discrete_mesh &m) {
h5::group gr = fg.open_group(subgroup_name);
typename discrete_mesh::domain_t dom;
h5_read(gr, "domain", dom);
m = discrete_mesh(std::move(dom));
}
