void run()
{
// Read filenames
vector<std::string> filenames;
{
std::string folder = Path::dirname (argument[0]);
std::ifstream ifs (argument[0].c_str());
std::string temp;
while (getline (ifs, temp)) {
std::string filename (Path::join (folder, temp));
size_t p = filename.find_last_not_of(" \t");
if (std::string::npos != p)
filename.erase(p+1);
if (filename.size()) {
if (!MR::Path::exists (filename))
throw Exception ("Input connectome file not found: \"" + filename + "\"");
filenames.push_back (filename);
}
}
}
const MR::Connectome::matrix_type example_connectome = load_matrix (filenames.front());
if (example_connectome.rows() != example_connectome.cols())
throw Exception ("Connectome of first subject is not square (" + str(example_connectome.rows()) + " x " + str(example_connectome.cols()) + ")");
const MR::Connectome::node_t num_nodes = example_connectome.rows();
// Initialise enhancement algorithm
std::shared_ptr<Stats::EnhancerBase> enhancer;
switch (int(argument[1])) {
case 0: {
auto opt = get_options ("threshold");
if (!opt.size())
throw Exception ("For NBS algorithm, -threshold option must be provided");
enhancer.reset (new MR::Connectome::Enhance::NBS (num_nodes, opt[0][0]));
}
break;
case 1: {
std::shared_ptr<Stats::TFCE::EnhancerBase> base (new MR::Connectome::Enhance::NBS (num_nodes));
enhancer.reset (new Stats::TFCE::Wrapper (base));
load_tfce_parameters (*(dynamic_cast<Stats::TFCE::Wrapper*>(enhancer.get())));
if (get_options ("threshold").size())
WARN (std::string (argument[1]) + " is a threshold-free algorithm; -threshold option ignored");
}
break;
case 2: {
enhancer.reset (new MR::Connectome::Enhance::PassThrough());
if (get_options ("threshold").size())
WARN ("No enhancement algorithm being used; -threshold option ignored");
}
break;
default:
throw Exception ("Unknown enhancement algorithm");
}
size_t num_perms = get_option_value ("nperms", DEFAULT_NUMBER_PERMUTATIONS);
const bool do_nonstationary_adjustment = get_options ("nonstationary").size();
size_t nperms_nonstationary = get_option_value ("nperms_nonstationarity", DEFAULT_NUMBER_PERMUTATIONS_NONSTATIONARITY);
// Load design matrix
const matrix_type design = load_matrix (argument[2]);
if (size_t(design.rows()) != filenames.size())
throw Exception ("number of subjects does not match number of rows in design matrix");
// Load permutations file if supplied
auto opt = get_options("permutations");
vector<vector<size_t> > permutations;
if (opt.size()) {
permutations = Math::Stats::Permutation::load_permutations_file (opt[0][0]);
num_perms = permutations.size();
if (permutations[0].size() != (size_t)design.rows())
throw Exception ("number of rows in the permutations file (" + str(opt[0][0]) + ") does not match number of rows in design matrix");
}
// Load non-stationary correction permutations file if supplied
opt = get_options("permutations_nonstationary");
vector<vector<size_t> > permutations_nonstationary;
if (opt.size()) {
permutations_nonstationary = Math::Stats::Permutation::load_permutations_file (opt[0][0]);
nperms_nonstationary = permutations.size();
if (permutations_nonstationary[0].size() != (size_t)design.rows())
throw Exception ("number of rows in the nonstationary permutations file (" + str(opt[0][0]) + ") does not match number of rows in design matrix");
}
// Load contrast matrix
matrix_type contrast = load_matrix (argument[3]);
if (contrast.cols() > design.cols())
throw Exception ("too many contrasts for design matrix");
contrast.conservativeResize (contrast.rows(), design.cols());
const std::string output_prefix = argument[4];
// Load input data
// For compatibility with existing statistics code, symmetric matrix data is adjusted
// into vector form - one row per edge in the symmetric connectome. The Mat2Vec class
// deals with the re-ordering of matrix data into this form.
MR::Connectome::Mat2Vec mat2vec (num_nodes);
const size_t num_edges = mat2vec.vec_size();
matrix_type data (num_edges, filenames.size());
{
ProgressBar progress ("Loading input connectome data", filenames.size());
for (size_t subject = 0; subject < filenames.size(); subject++) {
const std::string& path (filenames[subject]);
MR::Connectome::matrix_type subject_data;
try {
subject_data = load_matrix (path);
} catch (Exception& e) {
throw Exception (e, "Error loading connectome data for subject #" + str(subject) + " (file \"" + path + "\"");
}
try {
MR::Connectome::to_upper (subject_data);
if (size_t(subject_data.rows()) != num_nodes)
throw Exception ("Connectome matrix is not the correct size (" + str(subject_data.rows()) + ", should be " + str(num_nodes) + ")");
} catch (Exception& e) {
throw Exception (e, "Connectome for subject #" + str(subject) + " (file \"" + path + "\") invalid");
}
for (size_t i = 0; i != num_edges; ++i)
data(i, subject) = subject_data (mat2vec(i).first, mat2vec(i).second);
++progress;
}
}
{
ProgressBar progress ("outputting beta coefficients, effect size and standard deviation...", contrast.cols() + 3);
const matrix_type betas = Math::Stats::GLM::solve_betas (data, design);
for (size_t i = 0; i < size_t(contrast.cols()); ++i) {
save_matrix (mat2vec.V2M (betas.col(i)), output_prefix + "_beta_" + str(i) + ".csv");
++progress;
}
const matrix_type abs_effects = Math::Stats::GLM::abs_effect_size (data, design, contrast);
save_matrix (mat2vec.V2M (abs_effects.col(0)), output_prefix + "_abs_effect.csv");
++progress;
const matrix_type std_effects = Math::Stats::GLM::std_effect_size (data, design, contrast);
matrix_type first_std_effect = mat2vec.V2M (std_effects.col (0));
for (MR::Connectome::node_t i = 0; i != num_nodes; ++i) {
for (MR::Connectome::node_t j = 0; j != num_nodes; ++j) {
if (!std::isfinite (first_std_effect (i, j)))
first_std_effect (i, j) = 0.0;
}
}
save_matrix (first_std_effect, output_prefix + "_std_effect.csv");
++progress;
const matrix_type stdevs = Math::Stats::GLM::stdev (data, design);
save_vector (stdevs.col(0), output_prefix + "_std_dev.csv");
}
Math::Stats::GLMTTest glm_ttest (data, design, contrast);
// If performing non-stationarity adjustment we need to pre-compute the empirical statistic
vector_type empirical_statistic;
if (do_nonstationary_adjustment) {
empirical_statistic = vector_type::Zero (num_edges);
if (permutations_nonstationary.size()) {
Stats::PermTest::PermutationStack perm_stack (permutations_nonstationary, "precomputing empirical statistic for non-stationarity adjustment...");
Stats::PermTest::precompute_empirical_stat (glm_ttest, enhancer, perm_stack, empirical_statistic);
} else {
Stats::PermTest::PermutationStack perm_stack (nperms_nonstationary, design.rows(), "precomputing empirical statistic for non-stationarity adjustment...", true);
Stats::PermTest::precompute_empirical_stat (glm_ttest, enhancer, perm_stack, empirical_statistic);
}
save_matrix (mat2vec.V2M (empirical_statistic), output_prefix + "_empirical.csv");
}
// Precompute default statistic and enhanced statistic
vector_type tvalue_output (num_edges);
vector_type enhanced_output (num_edges);
Stats::PermTest::precompute_default_permutation (glm_ttest, enhancer, empirical_statistic, enhanced_output, std::shared_ptr<vector_type>(), tvalue_output);
save_matrix (mat2vec.V2M (tvalue_output), output_prefix + "_tvalue.csv");
save_matrix (mat2vec.V2M (enhanced_output), output_prefix + "_enhanced.csv");
// Perform permutation testing
if (!get_options ("notest").size()) {
// FIXME Getting NANs in the null distribution
// Check: was result of pre-nulled subject data
vector_type null_distribution (num_perms);
vector_type uncorrected_pvalues (num_edges);
if (permutations.size()) {
Stats::PermTest::run_permutations (permutations, glm_ttest, enhancer, empirical_statistic,
enhanced_output, std::shared_ptr<vector_type>(),
null_distribution, std::shared_ptr<vector_type>(),
uncorrected_pvalues, std::shared_ptr<vector_type>());
} else {
Stats::PermTest::run_permutations (num_perms, glm_ttest, enhancer, empirical_statistic,
enhanced_output, std::shared_ptr<vector_type>(),
null_distribution, std::shared_ptr<vector_type>(),
uncorrected_pvalues, std::shared_ptr<vector_type>());
}
save_vector (null_distribution, output_prefix + "_null_dist.txt");
vector_type pvalue_output (num_edges);
Math::Stats::Permutation::statistic2pvalue (null_distribution, enhanced_output, pvalue_output);
save_matrix (mat2vec.V2M (pvalue_output), output_prefix + "_fwe_pvalue.csv");
save_matrix (mat2vec.V2M (uncorrected_pvalues), output_prefix + "_uncorrected_pvalue.csv");
}
}
void run() {
const value_type cluster_forming_threshold = get_option_value ("threshold", NaN);
const value_type tfce_dh = get_option_value ("tfce_dh", DEFAULT_TFCE_DH);
const value_type tfce_H = get_option_value ("tfce_h", DEFAULT_TFCE_H);
const value_type tfce_E = get_option_value ("tfce_e", DEFAULT_TFCE_E);
const bool use_tfce = !std::isfinite (cluster_forming_threshold);
int num_perms = get_option_value ("nperms", DEFAULT_NUMBER_PERMUTATIONS);
int nperms_nonstationary = get_option_value ("nperms_nonstationary", DEFAULT_NUMBER_PERMUTATIONS_NONSTATIONARITY);
const bool do_26_connectivity = get_options("connectivity").size();
const bool do_nonstationary_adjustment = get_options ("nonstationary").size();
// Read filenames
vector<std::string> subjects;
{
std::string folder = Path::dirname (argument[0]);
std::ifstream ifs (argument[0].c_str());
std::string temp;
while (getline (ifs, temp))
subjects.push_back (Path::join (folder, temp));
}
// Load design matrix
const matrix_type design = load_matrix<value_type> (argument[1]);
if (design.rows() != (ssize_t)subjects.size())
throw Exception ("number of input files does not match number of rows in design matrix");
// Load permutations file if supplied
auto opt = get_options("permutations");
vector<vector<size_t> > permutations;
if (opt.size()) {
permutations = Math::Stats::Permutation::load_permutations_file (opt[0][0]);
num_perms = permutations.size();
if (permutations[0].size() != (size_t)design.rows())
throw Exception ("number of rows in the permutations file (" + str(opt[0][0]) + ") does not match number of rows in design matrix");
}
// Load non-stationary correction permutations file if supplied
opt = get_options("permutations_nonstationary");
vector<vector<size_t> > permutations_nonstationary;
if (opt.size()) {
permutations_nonstationary = Math::Stats::Permutation::load_permutations_file (opt[0][0]);
nperms_nonstationary = permutations.size();
if (permutations_nonstationary[0].size() != (size_t)design.rows())
throw Exception ("number of rows in the nonstationary permutations file (" + str(opt[0][0]) + ") does not match number of rows in design matrix");
}
// Load contrast matrix
const matrix_type contrast = load_matrix<value_type> (argument[2]);
if (contrast.cols() != design.cols())
throw Exception ("the number of contrasts does not equal the number of columns in the design matrix");
auto mask_header = Header::open (argument[3]);
// Load Mask and compute adjacency
auto mask_image = mask_header.get_image<value_type>();
Filter::Connector connector (do_26_connectivity);
vector<vector<int> > mask_indices = connector.precompute_adjacency (mask_image);
const size_t num_vox = mask_indices.size();
matrix_type data (num_vox, subjects.size());
{
// Load images
ProgressBar progress("loading images", subjects.size());
for (size_t subject = 0; subject < subjects.size(); subject++) {
LogLevelLatch log_level (0);
auto input_image = Image<float>::open (subjects[subject]); //.with_direct_io (3); <- Should be inputting 3D images?
check_dimensions (input_image, mask_image, 0, 3);
int index = 0;
vector<vector<int> >::iterator it;
for (it = mask_indices.begin(); it != mask_indices.end(); ++it) {
input_image.index(0) = (*it)[0];
input_image.index(1) = (*it)[1];
input_image.index(2) = (*it)[2];
data (index++, subject) = input_image.value();
}
progress++;
}
}
if (!data.allFinite())
WARN ("input data contains non-finite value(s)");
Header output_header (mask_header);
output_header.datatype() = DataType::Float32;
output_header.keyval()["num permutations"] = str(num_perms);
output_header.keyval()["26 connectivity"] = str(do_26_connectivity);
output_header.keyval()["nonstationary adjustment"] = str(do_nonstationary_adjustment);
if (use_tfce) {
output_header.keyval()["tfce_dh"] = str(tfce_dh);
output_header.keyval()["tfce_e"] = str(tfce_E);
output_header.keyval()["tfce_h"] = str(tfce_H);
} else {
output_header.keyval()["threshold"] = str(cluster_forming_threshold);
}
const std::string prefix (argument[4]);
bool compute_negative_contrast = get_options("negative").size();
vector_type default_cluster_output (num_vox);
std::shared_ptr<vector_type> default_cluster_output_neg;
vector_type tvalue_output (num_vox);
vector_type empirical_enhanced_statistic;
if (compute_negative_contrast)
default_cluster_output_neg.reset (new vector_type (num_vox));
Math::Stats::GLMTTest glm (data, design, contrast);
std::shared_ptr<Stats::EnhancerBase> enhancer;
if (use_tfce) {
std::shared_ptr<Stats::TFCE::EnhancerBase> base (new Stats::Cluster::ClusterSize (connector, cluster_forming_threshold));
enhancer.reset (new Stats::TFCE::Wrapper (base, tfce_dh, tfce_E, tfce_H));
} else {
enhancer.reset (new Stats::Cluster::ClusterSize (connector, cluster_forming_threshold));
}
if (do_nonstationary_adjustment) {
if (!use_tfce)
throw Exception ("nonstationary adjustment is not currently implemented for threshold-based cluster analysis");
empirical_enhanced_statistic = vector_type::Zero (num_vox);
if (permutations_nonstationary.size()) {
Stats::PermTest::PermutationStack permutations (permutations_nonstationary, "precomputing empirical statistic for non-stationarity adjustment...");
Stats::PermTest::precompute_empirical_stat (glm, enhancer, permutations, empirical_enhanced_statistic);
} else {
Stats::PermTest::PermutationStack permutations (nperms_nonstationary, design.rows(), "precomputing empirical statistic for non-stationarity adjustment...", false);
Stats::PermTest::precompute_empirical_stat (glm, enhancer, permutations, empirical_enhanced_statistic);
}
save_matrix (empirical_enhanced_statistic, prefix + "empirical.txt");
}
Stats::PermTest::precompute_default_permutation (glm, enhancer, empirical_enhanced_statistic,
default_cluster_output, default_cluster_output_neg, tvalue_output);
{
ProgressBar progress ("generating pre-permutation output", (compute_negative_contrast ? 3 : 2) + contrast.cols() + 3);
{
auto tvalue_image = Image<float>::create (prefix + "tvalue.mif", output_header);
write_output (tvalue_output, mask_indices, tvalue_image);
}
++progress;
{
auto cluster_image = Image<float>::create (prefix + (use_tfce ? "tfce.mif" : "cluster_sizes.mif"), output_header);
write_output (default_cluster_output, mask_indices, cluster_image);
}
++progress;
if (compute_negative_contrast) {
assert (default_cluster_output_neg);
auto cluster_image_neg = Image<float>::create (prefix + (use_tfce ? "tfce_neg.mif" : "cluster_sizes_neg.mif"), output_header);
write_output (*default_cluster_output_neg, mask_indices, cluster_image_neg);
++progress;
}
auto temp = Math::Stats::GLM::solve_betas (data, design);
for (ssize_t i = 0; i < contrast.cols(); ++i) {
auto beta_image = Image<float>::create (prefix + "beta" + str(i) + ".mif", output_header);
write_output (temp.row(i), mask_indices, beta_image);
++progress;
}
{
const auto temp = Math::Stats::GLM::abs_effect_size (data, design, contrast);
auto abs_effect_image = Image<float>::create (prefix + "abs_effect.mif", output_header);
write_output (temp.row(0), mask_indices, abs_effect_image);
}
++progress;
{
const auto temp = Math::Stats::GLM::std_effect_size (data, design, contrast);
auto std_effect_image = Image<float>::create (prefix + "std_effect.mif", output_header);
write_output (temp.row(0), mask_indices, std_effect_image);
}
++progress;
{
const auto temp = Math::Stats::GLM::stdev (data, design);
auto std_dev_image = Image<float>::create (prefix + "std_dev.mif", output_header);
write_output (temp.row(0), mask_indices, std_dev_image);
}
}
if (!get_options ("notest").size()) {
vector_type perm_distribution (num_perms);
std::shared_ptr<vector_type> perm_distribution_neg;
vector_type uncorrected_pvalue (num_vox);
std::shared_ptr<vector_type> uncorrected_pvalue_neg;
if (compute_negative_contrast) {
perm_distribution_neg.reset (new vector_type (num_perms));
uncorrected_pvalue_neg.reset (new vector_type (num_vox));
}
if (permutations.size()) {
Stats::PermTest::run_permutations (permutations, glm, enhancer, empirical_enhanced_statistic,
default_cluster_output, default_cluster_output_neg,
perm_distribution, perm_distribution_neg,
uncorrected_pvalue, uncorrected_pvalue_neg);
} else {
Stats::PermTest::run_permutations (num_perms, glm, enhancer, empirical_enhanced_statistic,
default_cluster_output, default_cluster_output_neg,
perm_distribution, perm_distribution_neg,
uncorrected_pvalue, uncorrected_pvalue_neg);
}
save_matrix (perm_distribution, prefix + "perm_dist.txt");
if (compute_negative_contrast) {
assert (perm_distribution_neg);
save_matrix (*perm_distribution_neg, prefix + "perm_dist_neg.txt");
}
ProgressBar progress ("generating output", compute_negative_contrast ? 4 : 2);
{
auto uncorrected_pvalue_image = Image<float>::create (prefix + "uncorrected_pvalue.mif", output_header);
write_output (uncorrected_pvalue, mask_indices, uncorrected_pvalue_image);
}
++progress;
{
vector_type fwe_pvalue_output (num_vox);
Math::Stats::Permutation::statistic2pvalue (perm_distribution, default_cluster_output, fwe_pvalue_output);
auto fwe_pvalue_image = Image<float>::create (prefix + "fwe_pvalue.mif", output_header);
write_output (fwe_pvalue_output, mask_indices, fwe_pvalue_image);
}
++progress;
if (compute_negative_contrast) {
assert (uncorrected_pvalue_neg);
assert (perm_distribution_neg);
auto uncorrected_pvalue_image_neg = Image<float>::create (prefix + "uncorrected_pvalue_neg.mif", output_header);
write_output (*uncorrected_pvalue_neg, mask_indices, uncorrected_pvalue_image_neg);
++progress;
vector_type fwe_pvalue_output_neg (num_vox);
Math::Stats::Permutation::statistic2pvalue (*perm_distribution_neg, *default_cluster_output_neg, fwe_pvalue_output_neg);
auto fwe_pvalue_image_neg = Image<float>::create (prefix + "fwe_pvalue_neg.mif", output_header);
write_output (fwe_pvalue_output_neg, mask_indices, fwe_pvalue_image_neg);
}
}
}