py::tuple compute_frequencies(Vector &ea, Vector &eix, Vector &eiy, Vector &gj, Vector &rhoJ, int &n) {
int ndof;
ndof = n*6;
// int nnode;
// nnode = BldR.size();
// ndof = 6 * nnode;
// Vector freqs(ndof);
// Matrix eig_vec(ndof, ndof);
Vector freqs(ndof);
Matrix eig_vec(0, 0);
mycurve->frequencies(ea, eix, eiy, gj, rhoJ, freqs, eig_vec);
return py::make_tuple(freqs, eig_vec);
}
void PowershellEventSubscriber::addScriptResult(Row& results) {
Row r;
r["time"] = results["time"];
r["datetime"] = results["datetime"];
r["script_block_id"] = results["ScriptBlockId"];
r["script_block_count"] = INTEGER(results["MessageTotal"]);
r["script_name"] = results["Name"];
r["script_path"] = results["Path"];
r["script_text"] = results["ScriptBlockText"];
// Grab the feature vectors from the configuration
auto parser = Config::getParser(kFeatureVectorsRootKey);
if (parser == nullptr) {
VLOG(1) << "Failed to get configured feature vectors";
add(r);
return;
}
// Get the reassembled powershell scripts character frequency vector
std::vector<double> freqs(kCharFreqVectorLen, 0.0);
for (const auto chr : r["script_text"]) {
if (chr < kCharFreqVectorLen) {
freqs[chr] += 1.0 / r["script_text"].length();
}
}
const auto& cf =
parser->getData().doc()["feature_vectors"]["character_frequencies"];
if (cf.Empty() || cf.Size() != kCharFreqVectorLen) {
VLOG(1) << "Invalid character frequency map found, skipping computation";
add(r);
return;
}
std::vector<double> cfg_freqs(kCharFreqVectorLen, 0.0);
for (unsigned int i = 0; i < cf.Size(); i++) {
cfg_freqs[i] = cf[i].GetDouble();
}
r["cosine_similarity"] = DOUBLE(cosineSimilarity(freqs, cfg_freqs));
add(r);
}
void decompressStaticly(std::istream& in, std::ostream& out) {
// read frequencies that we need to for static data model
std::vector<unsigned> freqs(NUM_SYMBOLS);
for (size_t i = 0; i < freqs.size(); ++i) {
in.read(reinterpret_cast<char*>(&freqs[i]), sizeof(unsigned));
}
StaticDataModel dataModel(freqs);
ArithmeticDecoder decoder(std::make_shared<BitStreamReader>(&in));
for (;;) {
auto symbol = decoder.decode(&dataModel);
if (symbol >= NUM_SYMBOLS)
throw std::runtime_error("Read bad symbol value. Symbols are expected to be 1 byte long.");
// on ending symbol exit loop
if (symbol == NUM_SYMBOLS - 1)
break;
out << static_cast<char>(symbol);
}
}
int KNN_prune_superfluous
(
PatternList p, // source
Categories c, // source
long y, // source instance index
long k, // k(!)
long skipper // Skipping instance skipper
)
{
if (y > p -> ny) y = p -> ny; // safety belt
if (k > p -> ny) k = p -> ny;
autoFeatureWeights fws = FeatureWeights_create (p -> nx);
autoNUMvector <long> indices (0L, k - 1);
autoNUMvector <long> freqindices (0L, k - 1);
autoNUMvector <double> distances (0L, k - 1);
autoNUMvector <double> freqs (0L, k - 1);
if (! KNN_kNeighboursSkip (p, p, fws.get(), y, k, indices.peek(), skipper)) return 0;
long ncategories = KNN_kIndicesToFrequenciesAndDistances (c, k, indices.peek(), distances.peek(), freqs.peek(), freqindices.peek());
int result = FeatureWeights_areFriends (c->at [y], c->at [freqindices [KNN_max (freqs.peek(), ncategories)]]);
if (result)
return 1;
return 0;
}
char MSAConsensusAlgorithmDefault::getConsensusCharAndScore(const MAlignment& msa, int pos, int& cnt, const QVector<qint64> &seqIdx) const {
//TODO: use var-length array!
QVector<QPair<int, char> > freqs(32);
int ch = MAlignment_GapChar;
int nSeq = seqIdx.isEmpty() ? msa.getNumRows() : seqIdx.size();
for (int seq = 0; seq < nSeq; seq++) {
uchar c = (uchar)msa.charAt( seqIdx.isEmpty() ? seq : seqIdx[ seq ],
pos);
if (c >= 'A' && c <= 'Z') {
int idx = c - 'A';
assert(idx >=0 && idx <= freqs.size());
freqs[idx].first++;
freqs[idx].second = c;
}
}
qSort(freqs);
int p1 = freqs[freqs.size()-1].first;
int p2 = freqs[freqs.size()-2].first;
if (p1 == 0 || (p1 == 1 && nSeq > 1)) {
ch = MAlignment_GapChar;
cnt = 0;
} else {
int c1 = freqs[freqs.size()-1].second;
ch = p2 == p1 ? '+' : c1;
cnt = p1;
}
//lowercase alpha chars with < threshold% content
int currentThreshold = getThreshold();
int cntToUseLowerCase = int(currentThreshold / 100.0 * nSeq);
if (cnt < cntToUseLowerCase && (ch >='A' && ch <='Z')) {
ch = ch + ('a'-'A');
}
return ch;
}
void LRedMarginLinearLogLike(double *Cube, int &ndim, int &npars, double &lnew, void *context)
{
int numfit=((MNStruct *)context)->numFitTiming + ((MNStruct *)context)->numFitJumps+1;
double Fitparams[numfit];
double *EFAC;
double EQUAD;
int pcount=0;
// printf("here1\n");
for(int p=0;p<ndim;p++){
// printf("param %i %g %g\n",p,((MNStruct *)context)->Dpriors[p][0],((MNStruct *)context)->Dpriors[p][1]);
Cube[p]=(((MNStruct *)context)->Dpriors[p][1]-((MNStruct *)context)->Dpriors[p][0])*Cube[p]+((MNStruct *)context)->Dpriors[p][0];
}
// printf("here1.5\n");
for(int p=0;p < numfit; p++){
Fitparams[p]=Cube[p];
pcount++;
// printf("param: %i %g \n",p,Fitparams[p]);
}
if(((MNStruct *)context)->numFitEFAC == 0){
EFAC=new double[1];
EFAC[0]=1;
//
}
else if(((MNStruct *)context)->numFitEFAC == 1){
EFAC=new double[1];
EFAC[0]=Cube[pcount];
pcount++;
}
else if(((MNStruct *)context)->numFitEFAC > 1){
EFAC=new double[((MNStruct *)context)->numFitEFAC];
for(int p=0;p< ((MNStruct *)context)->numFitEFAC; p++){
EFAC[p]=Cube[pcount];
pcount++;
}
}
if(((MNStruct *)context)->numFitEQUAD == 0){
EQUAD=0;
// printf("EQUAD: %g \n",EQUAD);
}
else{
EQUAD=pow(10.0,2*Cube[pcount]);
pcount++;
// printf("EQUAD: %g %g %g %i \n",EQUAD,EQUADPrior[0],EQUADPrior[1],((MNStruct *)context)->numFitTiming + ((MNStruct *)context)->numFitJumps + ((MNStruct *)context)->numFitEFAC);
}
double *Fitvec=new double[((MNStruct *)context)->pulse->nobs];
double *Diffvec=new double[((MNStruct *)context)->pulse->nobs];
dgemv(((MNStruct *)context)->DMatrix,Fitparams,Fitvec,((MNStruct *)context)->pulse->nobs,numfit,'N');
for(int o=0;o<((MNStruct *)context)->pulse->nobs; o++){
Diffvec[o]=((MNStruct *)context)->pulse->obsn[o].residual-Fitvec[o];
}
int FitCoeff=2*(((MNStruct *)context)->numFitRedCoeff);
double *powercoeff=new double[FitCoeff];
double *Noise=new double[((MNStruct *)context)->pulse->nobs];
double *GDiffvec=new double[((MNStruct *)context)->Gsize];
for(int o=0;o<((MNStruct *)context)->pulse->nobs; o++){
Noise[o]=pow(((((MNStruct *)context)->pulse->obsn[o].toaErr)*pow(10.0,-6))*EFAC[((MNStruct *)context)->sysFlags[o]],2) + EQUAD;
}
double **NG = new double*[((MNStruct *)context)->pulse->nobs]; for (int k=0; k<((MNStruct *)context)->pulse->nobs; k++) NG[k] = new double[((MNStruct *)context)->Gsize];
for(int i=0;i<((MNStruct *)context)->pulse->nobs;i++){
for(int j=0;j<((MNStruct *)context)->Gsize; j++){
NG[i][j]=((MNStruct *)context)->GMatrix[i][j]*Noise[i];
}
}
double** GG = new double*[((MNStruct *)context)->Gsize]; for (int k=0; k<((MNStruct *)context)->Gsize; k++) GG[k] = new double[((MNStruct *)context)->Gsize];
dgemm(((MNStruct *)context)->GMatrix, NG,GG,((MNStruct *)context)->pulse->nobs, ((MNStruct *)context)->Gsize,((MNStruct *)context)->pulse->nobs, ((MNStruct *)context)->Gsize, 'T','N');
double tdet=0;
dpotrf(GG, ((MNStruct *)context)->Gsize, tdet);
dpotri(GG,((MNStruct *)context)->Gsize);
dgemm(((MNStruct *)context)->GMatrix, GG,NG,((MNStruct *)context)->pulse->nobs, ((MNStruct *)context)->Gsize, ((MNStruct *)context)->Gsize, ((MNStruct *)context)->Gsize, 'N','N');
double **GNG = new double*[((MNStruct *)context)->pulse->nobs]; for (int k=0; k<((MNStruct *)context)->pulse->nobs; k++) GNG[k] = new double[((MNStruct *)context)->pulse->nobs];
dgemm(NG, ((MNStruct *)context)->GMatrix, GNG,((MNStruct *)context)->pulse->nobs, ((MNStruct *)context)->Gsize, ((MNStruct *)context)->pulse->nobs, ((MNStruct *)context)->Gsize, 'N','T');
double timelike=0;
for(int o1=0; o1<((MNStruct *)context)->pulse->nobs; o1++){
for(int o2=0;o2<((MNStruct *)context)->pulse->nobs; o2++){
timelike=timelike+Diffvec[o1]*GNG[o1][o2]*Diffvec[o2];
}
}
double *NFd = new double[FitCoeff];
double **FMatrix=new double*[((MNStruct *)context)->pulse->nobs];
for(int i=0;i<((MNStruct *)context)->pulse->nobs;i++){
FMatrix[i]=new double[FitCoeff];
}
double **NF=new double*[((MNStruct *)context)->pulse->nobs];
for(int i=0;i<((MNStruct *)context)->pulse->nobs;i++){
NF[i]=new double[FitCoeff];
}
double **FNF=new double*[FitCoeff];
for(int i=0;i<FitCoeff;i++){
FNF[i]=new double[FitCoeff];
}
double start,end;
int go=0;
for (int i=0;i<((MNStruct *)context)->pulse->nobs;i++)
{
if (((MNStruct *)context)->pulse->obsn[i].deleted==0)
{
if (go==0)
{
go = 1;
start = (double)((MNStruct *)context)->pulse->obsn[i].bat;
end = start;
}
else
{
if (start > (double)((MNStruct *)context)->pulse->obsn[i].bat)
start = (double)((MNStruct *)context)->pulse->obsn[i].bat;
if (end < (double)((MNStruct *)context)->pulse->obsn[i].bat)
end = (double)((MNStruct *)context)->pulse->obsn[i].bat;
}
}
}
// printf("Total time span = %.6f days = %.6f years\n",end-start,(end-start)/365.25);
double maxtspan=end-start;
double freqdet=0;
for (int i=0; i<FitCoeff/2; i++){
int pnum=pcount;
double pc=Cube[pcount];
powercoeff[i]=pow(10.0,pc)/(maxtspan*24*60*60);///(365.25*24*60*60)/4;
powercoeff[i+FitCoeff/2]=powercoeff[i];
freqdet=freqdet+2*log(powercoeff[i]);
pcount++;
}
int coeffsize=FitCoeff/2;
std::vector<double>freqs(FitCoeff/2);
for(int i=0;i<FitCoeff/2;i++){
freqs[i]=double(i+1)/maxtspan;
}
for(int i=0;i<FitCoeff/2;i++){
for(int k=0;k<((MNStruct *)context)->pulse->nobs;k++){
double time=(double)((MNStruct *)context)->pulse->obsn[k].bat; //- (double)((MNStruct *)context)->pulse->param[param_pepoch].val[0] - maxtspan/2;
FMatrix[k][i]=cos(2*M_PI*freqs[i]*time);
// printf("cos %i %i %g \n",i,k,time);
}
}
for(int i=0;i<FitCoeff/2;i++){
for(int k=0;k<((MNStruct *)context)->pulse->nobs;k++){
double time=(double)((MNStruct *)context)->pulse->obsn[k].bat; //- (double)((MNStruct *)context)->pulse->param[param_pepoch].val[0] - maxtspan/2;
FMatrix[k][i+FitCoeff/2]=sin(2*M_PI*freqs[i]*time);
// printf("sin %i %i %g \n",i+FitCoeff/2,k,time);
}
}
dgemm(GNG, FMatrix , NF, ((MNStruct *)context)->pulse->nobs,((MNStruct *)context)->pulse->nobs, ((MNStruct *)context)->pulse->nobs, FitCoeff, 'N', 'N');
dgemm(FMatrix, NF , FNF, ((MNStruct *)context)->pulse->nobs, FitCoeff, ((MNStruct *)context)->pulse->nobs, FitCoeff, 'T', 'N');
dgemv(NF,Diffvec,NFd,((MNStruct *)context)->pulse->nobs,FitCoeff,'T');
double **PPFM=new double*[FitCoeff];
for(int i=0;i<FitCoeff;i++){
PPFM[i]=new double[FitCoeff];
for(int j=0;j<FitCoeff;j++){
PPFM[i][j]=0;
}
}
for(int c1=0; c1<FitCoeff; c1++){
PPFM[c1][c1]=1.0/powercoeff[c1];
}
for(int j=0;j<FitCoeff;j++){
for(int k=0;k<FitCoeff;k++){
PPFM[j][k]=PPFM[j][k]+FNF[j][k];
}
}
double jointdet=0;
dpotrf(PPFM, FitCoeff, jointdet);
dpotri(PPFM,FitCoeff);
double freqlike=0;
for(int i=0;i<FitCoeff;i++){
for(int j=0;j<FitCoeff;j++){
// printf("%i %i %g %g\n",i,j,NFd[i],PPFM[i][j]);
freqlike=freqlike+NFd[i]*PPFM[i][j]*NFd[j];
}
}
lnew=-0.5*(tdet+jointdet+freqdet+timelike-freqlike);
if(isnan(lnew) || isinf(lnew)){
lnew=-pow(10.0,200);
// printf("red amp and alpha %g %g\n",redamp,redalpha);
// printf("Like: %g %g %g \n",lnew,Chisq,covdet);
}
delete[] EFAC;
delete[] powercoeff;
delete[] NFd;
for (int j = 0; j < FitCoeff; j++){
delete[]PPFM[j];
}
delete[]PPFM;
for (int j = 0; j < ((MNStruct *)context)->pulse->nobs; j++){
delete[]NF[j];
}
delete[]NF;
for (int j = 0; j < FitCoeff; j++){
delete[]FNF[j];
}
delete[]FNF;
for (int j = 0; j < ((MNStruct *)context)->pulse->nobs; j++){
delete[]FMatrix[j];
}
delete[]FMatrix;
delete[] Noise;
delete[] Diffvec;
delete[] Fitvec;
for (int j = 0; j < ((MNStruct *)context)->pulse->nobs; j++){
delete[] NG[j];
}
delete[] NG;
for (int j = 0; j < ((MNStruct *)context)->Gsize; j++){
delete[]GG[j];
}
delete[] GG;
for (int j = 0; j < ((MNStruct *)context)->pulse->nobs; j++){
delete[] GNG[j];
}
delete[] GNG;
// if(isinf(lnew) || isinf(jointdet) || isinf(tdet) || isinf(freqdet) || isinf(timelike) || isinf(freqlike)){
// printf("Chisq: %g %g %g %g %g %g \n",lnew,jointdet,tdet,freqdet,timelike,freqlike);
// }
}
void acoder::_init_laplace_distribution(const float &factor,
const float &lambda,
arcoder_base *const coder_base,
const sz_t &i, const bool purge)
{
// Проверим входные параметры
assert(0 != coder_base);
assert(0 != lambda);
assert(0 != factor);
// Установка текущей модели
coder_base->model(i);
// Подсчёт часто используемых значений
const float lambda_2 = float(lambda / 2);
// Ссылка на описание используемой модели
const model_t &model = _models[i];
#ifdef LIBWIC_DEBUG
std::vector<unsigned int> freqs(model._symbols);
#endif
#ifdef LIBWIC_ACODER_PRINT_STATISTIC
std::cout << std::endl
<< "acoder::init::model::" << i << "::" << model._symbols
<< "::" << model._delta << std::endl;
#endif
// Вычисление вероятности для каждого символа в модели
for (value_type x = model.min; model.max >= x; ++x)
{
// Вероятность умноженая на масштабный коэффициент
const float p = factor*lambda_2*exp(-lambda*abs(x));
// Символ из модели аркодера (не отритцательный)
const value_type smb = x + model._delta;
// Проверка не отритцательности симвлоа
assert(0 <= smb);
// Целочисленное значение полученной вероятности
const unsigned int ip = (unsigned int)floor(p + 0.5);
#ifdef LIBWIC_DEBUG
freqs[smb] = ip;
#endif
#ifdef LIBWIC_ACODER_PRINT_STATISTIC
std::cout << "acoder::init::value::" << i << "::" << smb
<< "::" << ip << std::endl;
#endif
if (purge) continue;
// Инициализация модели аркодера
for (unsigned int k = ip; 0 < k; --k) coder_base->update_model(smb);
}
if (!purge) return;
#ifdef LIBWIC_DEBUG
_dbg_out_stream << "#" << std::setw(2) << i;
_dbg_out_stream << "[" << std::setw(5) << model._symbols << "]";
_dbg_out_stream << " | abs_avg = " << std::setw(7) << model._abs_average;
_dbg_out_stream << " | lambda = " << std::setw(7) << lambda;
_dbg_out_stream << std::endl;
_dbg_out_stream << "[_init_normal_distribution]" << std::endl;
_dbg_freqs_out(_dbg_out_stream, freqs, model, i, coder_base);
_dbg_out_stream << "[/_init_normal_distribution]" << std::endl;
#endif
}
/**
*Compress a file using Huffman code
*The first argument: infile name
*The second argument: outfile name
**/
int main(int argc, char *argv[])
{
HCTree hTree;
//variable for number of characters
int numChars;
if(argc < 2)
{
std::cout << "Error: Too Few Arguments" << std::endl;
return 1;
}
//store the contents of the first file into inFile
std::string inFile = argv[1];
//store the contents of the second file into outFile
std::string outFile = argv[2];
//std::cout << "Section1" << std::endl;
//create instream
std::ifstream in;
//create outstream
std::ofstream out;
//open infile
in.open(inFile, ios::binary);
long sizeOfFile;
in.seekg(0, ios::end);
sizeOfFile = in.tellg();
in.seekg(0, ios::beg);
if(sizeOfFile == 0)
{
std::cout<<"blank file...try again!!!"<<endl;
std::ofstream empty;
empty.open(outFile, ios::binary);
empty.close();
return 0;
}
//open outfile
out.open(outFile, ios::binary);
//put the instream into bitinputstream
BitInputStream bitInStream(in);
BitOutputStream bitOutStream(out);
//std::cout << "Section2" << std::endl;
int total, count, index;
//variable to hold frequency
std::vector<int> freqs(256,0);
int symb = bitInStream.readByte();
while(symb != -1)
{
//std::cout << (char) symb << std::endl;
freqs[symb]++;
symb = bitInStream.readByte();
}
//std::cout << "Section3" << std::endl;
/**for(int i = 0; i < freqs.size(); i++)
{
if(freqs[i])
std::cout << freqs[i] <<std::endl;
}**/
hTree.build(freqs);
//std::cout << "Writing Header" << std::endl;
write_header(freqs, bitOutStream);
//std::cout << "Files closed" << std::endl;
in.close();
in.open(inFile, ios::binary);
//std::cout << "Section4" << std::endl;
/**HCNode node1(1, 'r', nullptr, nullptr, nullptr);
HCNode node2(1, 'r', nullptr, nullptr, nullptr);
HCNode node3(2, 'a', nullptr, nullptr, nullptr);
HCNode node4(3, 'a', nullptr, nullptr, nullptr);**/
//std::cout << "node1 == node2: " << (comp(&node1,&node2)) << std::endl;
//std::cout << "node1 == node1: " << (comp(&node1,&node1)) << std::endl;
//std::cout << "node2 == node3: " << (comp(&node2,&node3)) << std::endl;
//std::cout << "node3 == node4: " << (comp(&node3,&node4)) << std::endl;
//hTree.print();
//char cake = 'a';
//hTree.encode(cake, bitOutStream);
symb = bitInStream.readByte();
while(symb != -1)
{
//std::cout << (char) symb << std::endl;
hTree.encode(symb, bitOutStream);
symb = bitInStream.readByte();
}
bitOutStream.flush();
in.close();
out.close();
/**std::cout << "Encoding has Finished" << std::endl;
std::ifstream in2;
//create outstream
std::ofstream out2;
//open infile
in2.open(outFile, ios::binary);
//open outfile
out2.open("FinalOutput.txt", ios::binary);
//put the instream into bitinputstream
BitInputStream bitInStream2(in2);
BitOutputStream bitOutStream2(out2);
char symbol = hTree.decode(bitInStream2);
while(symbol != -1)
{
//std::cout << (char) symbol << std::endl;
symbol = hTree.decode(bitInStream2);
}
in2.close();
out2.close();**/
return 0;
}
/*! \param[in] lambda Параметр lambda распределения лапласа
\param[in] power Степень в котору будет возведён аргумент x в
распределении лапласа.
\param[in] factor Множитель, на который будет умножено значение
вероятности по формуле распределения Лапласа. Полученное значение будет
использоваться как количество вызовов функции update_model()
арифметического кодера.
\param[in] part Часть диапазона модели, которая будет инициализирована.
\param[in,out] coder_base Указатель на объект используемого
арифметического кодера
\param[in] i Номер используемой модели
\param[in] purge Если <i>true</i> инициализация модели производится не
будет
*/
void acoder::_init_laplace_distribution(const float &lambda, const float &power,
const float &factor, const float &part,
arcoder_base *const coder_base,
const sz_t &i, const bool purge)
{
// Проверим входные параметры
assert(0 != coder_base);
assert(0 != lambda);
assert(0 != factor);
assert(1 <= part);
// Установка текущей модели
coder_base->model(i);
// Ссылка на описание используемой модели
const model_t &model = _models[i];
#ifdef LIBWIC_DEBUG
std::vector<unsigned int> freqs(model._symbols);
#endif
#ifdef LIBWIC_ACODER_PRINT_STATISTIC
std::cout << std::endl
<< "acoder::init::model::" << i << "::" << model._symbols
<< "::" << model._delta << "::" << model.abs_avg
<< std::endl;
#endif
// вычисление предела инициализации
value_type limit;
if (0 <= model.min && 0 <= model.max)
{
const float min_d = float(model.max - model.min);
limit = (value_type)floor(min_d / part + 0.5f) - model._delta;
}
else
{
const float min_d = float(std::min(abs(model.min), abs(model.max)));
limit = (value_type)floor(min_d / part + 0.5f);
}
// Вычисление вероятности для каждого символа в модели
for (value_type x = model.min; model.max >= x; ++x)
{
// Вероятность появления символаи
const float p = _laplace_func(lambda, x, power);
// Символ из модели аркодера (не отритцательный)
const value_type smb = x + model._delta;
// Проверка не отритцательности симвлоа
assert(0 <= smb);
// Целочисленное значение полученной вероятности
const unsigned int ip = (abs(x) <= limit)
? (unsigned int)floor(factor*p + 0.5f) : 0;
#ifdef LIBWIC_DEBUG
freqs[smb] = ip;
#endif
#ifdef LIBWIC_ACODER_PRINT_STATISTIC
std::cout << "acoder::init::value::" << i << "::" << smb
<< "::" << ip << std::endl;
#endif
if (purge) continue;
// Инициализация модели аркодера
for (unsigned int k = ip; 0 < k; --k) coder_base->update_model(smb);
}
if (!purge) return;
#ifdef LIBWIC_DEBUG
_dbg_out_stream << "#" << std::setw(2) << i;
_dbg_out_stream << "[" << std::setw(5) << model._symbols << "]";
_dbg_out_stream << " | abs_avg = " << std::setw(7) << model._abs_average;
_dbg_out_stream << " | lambda = " << std::setw(7) << lambda;
_dbg_out_stream << std::endl;
_dbg_out_stream << "[_init_laplace_distribution]" << std::endl;
_dbg_freqs_out(_dbg_out_stream, freqs, model, i, coder_base);
_dbg_out_stream << "[/_init_laplace_distribution]" << std::endl;
#endif
}
void runBeagle(int resource,
int stateCount,
int ntaxa,
int nsites,
bool manualScaling,
bool autoScaling,
bool dynamicScaling,
int rateCategoryCount,
int nreps,
bool fullTiming,
bool requireDoublePrecision,
bool requireSSE,
int compactTipCount,
int randomSeed,
int rescaleFrequency,
bool unrooted,
bool calcderivs,
bool logscalers,
int eigenCount,
bool eigencomplex,
bool ievectrans,
bool setmatrix)
{
int edgeCount = ntaxa*2-2;
int internalCount = ntaxa-1;
int partialCount = ((ntaxa+internalCount)-compactTipCount)*eigenCount;
int scaleCount = ((manualScaling || dynamicScaling) ? ntaxa : 0);
BeagleInstanceDetails instDetails;
// create an instance of the BEAGLE library
int instance = beagleCreateInstance(
ntaxa, /**< Number of tip data elements (input) */
partialCount, /**< Number of partials buffers to create (input) */
compactTipCount, /**< Number of compact state representation buffers to create (input) */
stateCount, /**< Number of states in the continuous-time Markov chain (input) */
nsites, /**< Number of site patterns to be handled by the instance (input) */
eigenCount, /**< Number of rate matrix eigen-decomposition buffers to allocate (input) */
(calcderivs ? (3*edgeCount*eigenCount) : edgeCount*eigenCount),/**< Number of rate matrix buffers (input) */
rateCategoryCount,/**< Number of rate categories */
scaleCount*eigenCount, /**< scaling buffers */
&resource, /**< List of potential resource on which this instance is allowed (input, NULL implies no restriction */
1, /**< Length of resourceList list (input) */
0, /**< Bit-flags indicating preferred implementation charactertistics, see BeagleFlags (input) */
(ievectrans ? BEAGLE_FLAG_INVEVEC_TRANSPOSED : BEAGLE_FLAG_INVEVEC_STANDARD) |
(logscalers ? BEAGLE_FLAG_SCALERS_LOG : BEAGLE_FLAG_SCALERS_RAW) |
(eigencomplex ? BEAGLE_FLAG_EIGEN_COMPLEX : BEAGLE_FLAG_EIGEN_REAL) |
(dynamicScaling ? BEAGLE_FLAG_SCALING_DYNAMIC : 0) |
(autoScaling ? BEAGLE_FLAG_SCALING_AUTO : 0) |
(requireDoublePrecision ? BEAGLE_FLAG_PRECISION_DOUBLE : BEAGLE_FLAG_PRECISION_SINGLE) |
(requireSSE ? BEAGLE_FLAG_VECTOR_SSE : BEAGLE_FLAG_VECTOR_NONE), /**< Bit-flags indicating required implementation characteristics, see BeagleFlags (input) */
&instDetails);
if (instance < 0) {
fprintf(stderr, "Failed to obtain BEAGLE instance\n\n");
return;
}
int rNumber = instDetails.resourceNumber;
fprintf(stdout, "Using resource %i:\n", rNumber);
fprintf(stdout, "\tRsrc Name : %s\n",instDetails.resourceName);
fprintf(stdout, "\tImpl Name : %s\n", instDetails.implName);
if (!(instDetails.flags & BEAGLE_FLAG_SCALING_AUTO))
autoScaling = false;
// set the sequences for each tip using partial likelihood arrays
gt_srand(randomSeed); // fix the random seed...
for(int i=0; i<ntaxa; i++)
{
if (i >= compactTipCount) {
double* tmpPartials = getRandomTipPartials(nsites, stateCount);
beagleSetTipPartials(instance, i, tmpPartials);
free(tmpPartials);
} else {
int* tmpStates = getRandomTipStates(nsites, stateCount);
beagleSetTipStates(instance, i, tmpStates);
free(tmpStates);
}
}
#ifdef _WIN32
std::vector<double> rates(rateCategoryCount);
#else
double rates[rateCategoryCount];
#endif
for (int i = 0; i < rateCategoryCount; i++) {
rates[i] = gt_rand() / (double) GT_RAND_MAX;
}
beagleSetCategoryRates(instance, &rates[0]);
double* patternWeights = (double*) malloc(sizeof(double) * nsites);
for (int i = 0; i < nsites; i++) {
patternWeights[i] = gt_rand() / (double) GT_RAND_MAX;
}
beagleSetPatternWeights(instance, patternWeights);
free(patternWeights);
// create base frequency array
#ifdef _WIN32
std::vector<double> freqs(stateCount);
#else
double freqs[stateCount];
#endif
// create an array containing site category weights
#ifdef _WIN32
std::vector<double> weights(rateCategoryCount);
#else
double weights[rateCategoryCount];
#endif
for (int eigenIndex=0; eigenIndex < eigenCount; eigenIndex++) {
for (int i = 0; i < rateCategoryCount; i++) {
weights[i] = gt_rand() / (double) GT_RAND_MAX;
}
beagleSetCategoryWeights(instance, eigenIndex, &weights[0]);
}
double* eval;
if (!eigencomplex)
eval = (double*)malloc(sizeof(double)*stateCount);
else
eval = (double*)malloc(sizeof(double)*stateCount*2);
double* evec = (double*)malloc(sizeof(double)*stateCount*stateCount);
double* ivec = (double*)malloc(sizeof(double)*stateCount*stateCount);
for (int eigenIndex=0; eigenIndex < eigenCount; eigenIndex++) {
if (!eigencomplex && ((stateCount & (stateCount-1)) == 0)) {
for (int i=0; i<stateCount; i++) {
freqs[i] = 1.0 / stateCount;
}
// an eigen decomposition for the general state-space JC69 model
// If stateCount = 2^n is a power-of-two, then Sylvester matrix H_n describes
// the eigendecomposition of the infinitesimal rate matrix
double* Hn = evec;
Hn[0*stateCount+0] = 1.0; Hn[0*stateCount+1] = 1.0;
Hn[1*stateCount+0] = 1.0; Hn[1*stateCount+1] = -1.0; // H_1
for (int k=2; k < stateCount; k <<= 1) {
// H_n = H_1 (Kronecker product) H_{n-1}
for (int i=0; i<k; i++) {
for (int j=i; j<k; j++) {
double Hijold = Hn[i*stateCount + j];
Hn[i *stateCount + j + k] = Hijold;
Hn[(i+k)*stateCount + j ] = Hijold;
Hn[(i+k)*stateCount + j + k] = -Hijold;
Hn[j *stateCount + i + k] = Hn[i *stateCount + j + k];
Hn[(j+k)*stateCount + i ] = Hn[(i+k)*stateCount + j ];
Hn[(j+k)*stateCount + i + k] = Hn[(i+k)*stateCount + j + k];
}
}
}
// Since evec is Hadamard, ivec = (evec)^t / stateCount;
for (int i=0; i<stateCount; i++) {
for (int j=i; j<stateCount; j++) {
ivec[i*stateCount+j] = evec[j*stateCount+i] / stateCount;
ivec[j*stateCount+i] = ivec[i*stateCount+j]; // Symmetric
}
}
eval[0] = 0.0;
for (int i=1; i<stateCount; i++) {
eval[i] = -stateCount / (stateCount - 1.0);
}
} else if (!eigencomplex) {
for (int i=0; i<stateCount; i++) {
freqs[i] = gt_rand() / (double) GT_RAND_MAX;
}
double** qmat=New2DArray<double>(stateCount, stateCount);
double* relNucRates = new double[(stateCount * stateCount - stateCount) / 2];
int rnum=0;
for(int i=0;i<stateCount;i++){
for(int j=i+1;j<stateCount;j++){
relNucRates[rnum] = gt_rand() / (double) GT_RAND_MAX;
qmat[i][j]=relNucRates[rnum] * freqs[j];
qmat[j][i]=relNucRates[rnum] * freqs[i];
rnum++;
}
}
//set diags to sum rows to 0
double sum;
for(int x=0;x<stateCount;x++){
sum=0.0;
for(int y=0;y<stateCount;y++){
if(x!=y) sum+=qmat[x][y];
}
qmat[x][x]=-sum;
}
double* eigvalsimag=new double[stateCount];
double** eigvecs=New2DArray<double>(stateCount, stateCount);//eigenvecs
double** teigvecs=New2DArray<double>(stateCount, stateCount);//temp eigenvecs
double** inveigvecs=New2DArray<double>(stateCount, stateCount);//inv eigenvecs
int* iwork=new int[stateCount];
double* work=new double[stateCount];
EigenRealGeneral(stateCount, qmat, eval, eigvalsimag, eigvecs, iwork, work);
memcpy(*teigvecs, *eigvecs, stateCount*stateCount*sizeof(double));
InvertMatrix(teigvecs, stateCount, work, iwork, inveigvecs);
for(int x=0;x<stateCount;x++){
for(int y=0;y<stateCount;y++){
evec[x * stateCount + y] = eigvecs[x][y];
if (ievectrans)
ivec[x * stateCount + y] = inveigvecs[y][x];
else
ivec[x * stateCount + y] = inveigvecs[x][y];
}
}
Delete2DArray(qmat);
delete relNucRates;
delete eigvalsimag;
Delete2DArray(eigvecs);
Delete2DArray(teigvecs);
Delete2DArray(inveigvecs);
delete iwork;
delete work;
} else if (eigencomplex && stateCount==4 && eigenCount==1) {
// create base frequency array
double temp_freqs[4] = { 0.25, 0.25, 0.25, 0.25 };
// an eigen decomposition for the 4-state 1-step circulant infinitesimal generator
double temp_evec[4 * 4] = {
-0.5, 0.6906786606674509, 0.15153543380548623, 0.5,
0.5, -0.15153543380548576, 0.6906786606674498, 0.5,
-0.5, -0.6906786606674498, -0.15153543380548617, 0.5,
0.5, 0.15153543380548554, -0.6906786606674503, 0.5
};
double temp_ivec[4 * 4] = {
-0.5, 0.5, -0.5, 0.5,
0.6906786606674505, -0.15153543380548617, -0.6906786606674507, 0.15153543380548645,
0.15153543380548568, 0.6906786606674509, -0.15153543380548584, -0.6906786606674509,
0.5, 0.5, 0.5, 0.5
};
double temp_eval[8] = { -2.0, -1.0, -1.0, 0, 0, 1, -1, 0 };
for(int x=0;x<stateCount;x++){
freqs[x] = temp_freqs[x];
eval[x] = temp_eval[x];
eval[x+stateCount] = temp_eval[x+stateCount];
for(int y=0;y<stateCount;y++){
evec[x * stateCount + y] = temp_evec[x * stateCount + y];
if (ievectrans)
ivec[x * stateCount + y] = temp_ivec[x + y * stateCount];
else
ivec[x * stateCount + y] = temp_ivec[x * stateCount + y];
}
}
} else {
abort("should not be here");
}
beagleSetStateFrequencies(instance, eigenIndex, &freqs[0]);
if (!setmatrix) {
// set the Eigen decomposition
beagleSetEigenDecomposition(instance, eigenIndex, &evec[0], &ivec[0], &eval[0]);
}
}
free(eval);
free(evec);
free(ivec);
// a list of indices and edge lengths
int* edgeIndices = new int[edgeCount*eigenCount];
int* edgeIndicesD1 = new int[edgeCount*eigenCount];
int* edgeIndicesD2 = new int[edgeCount*eigenCount];
for(int i=0; i<edgeCount*eigenCount; i++) {
edgeIndices[i]=i;
edgeIndicesD1[i]=(edgeCount*eigenCount)+i;
edgeIndicesD2[i]=2*(edgeCount*eigenCount)+i;
}
double* edgeLengths = new double[edgeCount];
for(int i=0; i<edgeCount; i++) {
edgeLengths[i]=gt_rand() / (double) GT_RAND_MAX;
}
// create a list of partial likelihood update operations
// the order is [dest, destScaling, source1, matrix1, source2, matrix2]
int* operations = new int[(internalCount)*BEAGLE_OP_COUNT*eigenCount];
int* scalingFactorsIndices = new int[(internalCount)*eigenCount]; // internal nodes
for(int i=0; i<internalCount*eigenCount; i++){
operations[BEAGLE_OP_COUNT*i+0] = ntaxa+i;
operations[BEAGLE_OP_COUNT*i+1] = (dynamicScaling ? i : BEAGLE_OP_NONE);
operations[BEAGLE_OP_COUNT*i+2] = (dynamicScaling ? i : BEAGLE_OP_NONE);
int child1Index;
if (((i % internalCount)*2) < ntaxa)
child1Index = (i % internalCount)*2;
else
child1Index = i*2 - internalCount * (int)(i / internalCount);
operations[BEAGLE_OP_COUNT*i+3] = child1Index;
operations[BEAGLE_OP_COUNT*i+4] = child1Index;
int child2Index;
if (((i % internalCount)*2+1) < ntaxa)
child2Index = (i % internalCount)*2+1;
else
child2Index = i*2+1 - internalCount * (int)(i / internalCount);
operations[BEAGLE_OP_COUNT*i+5] = child2Index;
operations[BEAGLE_OP_COUNT*i+6] = child2Index;
scalingFactorsIndices[i] = i;
// printf("i %d dest %d c1 %d c2 %d\n", i, ntaxa+i, child1Index, child2Index);
if (autoScaling)
scalingFactorsIndices[i] += ntaxa;
}
int* rootIndices = new int[eigenCount];
int* lastTipIndices = new int[eigenCount];
int* categoryWeightsIndices = new int[eigenCount];
int* stateFrequencyIndices = new int[eigenCount];
int* cumulativeScalingFactorIndices = new int[eigenCount];
for (int eigenIndex=0; eigenIndex < eigenCount; eigenIndex++) {
rootIndices[eigenIndex] = ntaxa+(internalCount*(eigenIndex+1))-1;//ntaxa*2-2;
lastTipIndices[eigenIndex] = ntaxa-1;
categoryWeightsIndices[eigenIndex] = eigenIndex;
stateFrequencyIndices[eigenIndex] = 0;
cumulativeScalingFactorIndices[eigenIndex] = ((manualScaling || dynamicScaling) ? (scaleCount*eigenCount-1)-eigenCount+eigenIndex+1 : BEAGLE_OP_NONE);
if (dynamicScaling)
beagleResetScaleFactors(instance, cumulativeScalingFactorIndices[eigenIndex]);
}
// start timing!
struct timeval time1, time2, time3, time4, time5;
double bestTimeUpdateTransitionMatrices, bestTimeUpdatePartials, bestTimeAccumulateScaleFactors, bestTimeCalculateRootLogLikelihoods, bestTimeTotal;
double logL = 0.0;
double deriv1 = 0.0;
double deriv2 = 0.0;
double previousLogL = 0.0;
double previousDeriv1 = 0.0;
double previousDeriv2 = 0.0;
for (int i=0; i<nreps; i++){
if (manualScaling && (!(i % rescaleFrequency) || !((i-1) % rescaleFrequency))) {
for(int j=0; j<internalCount*eigenCount; j++){
operations[BEAGLE_OP_COUNT*j+1] = (((manualScaling && !(i % rescaleFrequency))) ? j : BEAGLE_OP_NONE);
operations[BEAGLE_OP_COUNT*j+2] = (((manualScaling && (i % rescaleFrequency))) ? j : BEAGLE_OP_NONE);
}
}
gettimeofday(&time1,NULL);
for (int eigenIndex=0; eigenIndex < eigenCount; eigenIndex++) {
if (!setmatrix) {
// tell BEAGLE to populate the transition matrices for the above edge lengths
beagleUpdateTransitionMatrices(instance, // instance
eigenIndex, // eigenIndex
&edgeIndices[eigenIndex*edgeCount], // probabilityIndices
(calcderivs ? &edgeIndicesD1[eigenIndex*edgeCount] : NULL), // firstDerivativeIndices
(calcderivs ? &edgeIndicesD2[eigenIndex*edgeCount] : NULL), // secondDerivativeIndices
edgeLengths, // edgeLengths
edgeCount); // count
} else {
double* inMatrix = new double[stateCount*stateCount*rateCategoryCount];
for (int matrixIndex=0; matrixIndex < edgeCount; matrixIndex++) {
for(int z=0;z<rateCategoryCount;z++){
for(int x=0;x<stateCount;x++){
for(int y=0;y<stateCount;y++){
inMatrix[z*stateCount*stateCount + x*stateCount + y] = gt_rand() / (double) GT_RAND_MAX;
}
}
}
beagleSetTransitionMatrix(instance, edgeIndices[eigenIndex*edgeCount + matrixIndex], inMatrix, 1);
if (calcderivs) {
beagleSetTransitionMatrix(instance, edgeIndicesD1[eigenIndex*edgeCount + matrixIndex], inMatrix, 0);
beagleSetTransitionMatrix(instance, edgeIndicesD2[eigenIndex*edgeCount + matrixIndex], inMatrix, 0);
}
}
}
}
gettimeofday(&time2, NULL);
// update the partials
beagleUpdatePartials( instance, // instance
(BeagleOperation*)operations, // eigenIndex
internalCount*eigenCount, // operationCount
(dynamicScaling ? internalCount : BEAGLE_OP_NONE)); // cumulative scaling index
gettimeofday(&time3, NULL);
int scalingFactorsCount = internalCount;
for (int eigenIndex=0; eigenIndex < eigenCount; eigenIndex++) {
if (manualScaling && !(i % rescaleFrequency)) {
beagleResetScaleFactors(instance,
cumulativeScalingFactorIndices[eigenIndex]);
beagleAccumulateScaleFactors(instance,
&scalingFactorsIndices[eigenIndex*internalCount],
scalingFactorsCount,
cumulativeScalingFactorIndices[eigenIndex]);
} else if (autoScaling) {
beagleAccumulateScaleFactors(instance, &scalingFactorsIndices[eigenIndex*internalCount], scalingFactorsCount, BEAGLE_OP_NONE);
}
}
gettimeofday(&time4, NULL);
// calculate the site likelihoods at the root node
if (!unrooted) {
beagleCalculateRootLogLikelihoods(instance, // instance
rootIndices,// bufferIndices
categoryWeightsIndices, // weights
stateFrequencyIndices, // stateFrequencies
cumulativeScalingFactorIndices,
eigenCount, // count
&logL); // outLogLikelihoods
} else {
// calculate the site likelihoods at the root node
beagleCalculateEdgeLogLikelihoods(instance, // instance
rootIndices,// bufferIndices
lastTipIndices,
lastTipIndices,
(calcderivs ? edgeIndicesD1 : NULL),
(calcderivs ? edgeIndicesD2 : NULL),
categoryWeightsIndices, // weights
stateFrequencyIndices, // stateFrequencies
cumulativeScalingFactorIndices,
eigenCount, // count
&logL, // outLogLikelihood
(calcderivs ? &deriv1 : NULL),
(calcderivs ? &deriv2 : NULL));
}
// end timing!
gettimeofday(&time5,NULL);
if (i == 0 || getTimeDiff(time1, time2) < bestTimeUpdateTransitionMatrices)
bestTimeUpdateTransitionMatrices = getTimeDiff(time1, time2);
if (i == 0 || getTimeDiff(time2, time3) < bestTimeUpdatePartials)
bestTimeUpdatePartials = getTimeDiff(time2, time3);
if (i == 0 || getTimeDiff(time3, time4) < bestTimeAccumulateScaleFactors)
bestTimeAccumulateScaleFactors = getTimeDiff(time3, time4);
if (i == 0 || getTimeDiff(time4, time5) < bestTimeUpdateTransitionMatrices)
bestTimeCalculateRootLogLikelihoods = getTimeDiff(time4, time5);
if (i == 0 || getTimeDiff(time1, time5) < bestTimeTotal)
bestTimeTotal = getTimeDiff(time1, time5);
if (!(logL - logL == 0.0))
abort("error: invalid lnL");
if (i > 0 && abs(logL - previousLogL) > MAX_DIFF)
abort("error: large lnL difference between reps");
if (calcderivs) {
if (!(deriv1 - deriv1 == 0.0) || !(deriv2 - deriv2 == 0.0))
abort("error: invalid deriv");
if (i > 0 && ((abs(deriv1 - previousDeriv1) > MAX_DIFF) || (abs(deriv2 - previousDeriv2) > MAX_DIFF)) )
abort("error: large deriv difference between reps");
}
previousLogL = logL;
previousDeriv1 = deriv1;
previousDeriv2 = deriv2;
}
if (resource == 0) {
cpuTimeUpdateTransitionMatrices = bestTimeUpdateTransitionMatrices;
cpuTimeUpdatePartials = bestTimeUpdatePartials;
cpuTimeAccumulateScaleFactors = bestTimeAccumulateScaleFactors;
cpuTimeCalculateRootLogLikelihoods = bestTimeCalculateRootLogLikelihoods;
cpuTimeTotal = bestTimeTotal;
}
if (!calcderivs)
fprintf(stdout, "logL = %.5f \n", logL);
else
fprintf(stdout, "logL = %.5f d1 = %.5f d2 = %.5f\n", logL, deriv1, deriv2);
std::cout.setf(std::ios::showpoint);
std::cout.setf(std::ios::floatfield, std::ios::fixed);
int timePrecision = 6;
int speedupPrecision = 2;
int percentPrecision = 2;
std::cout << "best run: ";
printTiming(bestTimeTotal, timePrecision, resource, cpuTimeTotal, speedupPrecision, 0, 0, 0);
if (fullTiming) {
std::cout << " transMats: ";
printTiming(bestTimeUpdateTransitionMatrices, timePrecision, resource, cpuTimeUpdateTransitionMatrices, speedupPrecision, 1, bestTimeTotal, percentPrecision);
std::cout << " partials: ";
printTiming(bestTimeUpdatePartials, timePrecision, resource, cpuTimeUpdatePartials, speedupPrecision, 1, bestTimeTotal, percentPrecision);
if (manualScaling || autoScaling) {
std::cout << " accScalers: ";
printTiming(bestTimeAccumulateScaleFactors, timePrecision, resource, cpuTimeAccumulateScaleFactors, speedupPrecision, 1, bestTimeTotal, percentPrecision);
}
std::cout << " rootLnL: ";
printTiming(bestTimeCalculateRootLogLikelihoods, timePrecision, resource, cpuTimeCalculateRootLogLikelihoods, speedupPrecision, 1, bestTimeTotal, percentPrecision);
}
std::cout << "\n";
beagleFinalizeInstance(instance);
}
