inline void
LU( DistMatrix<F>& A, DistMatrix<Int,VC,STAR>& p, DistMatrix<Int,VC,STAR>& q )
{
#ifndef RELEASE
CallStackEntry entry("LU");
#endif
p.ResizeTo( Min(A.Height(),A.Width()), 1 );
q.ResizeTo( Min(A.Height(),A.Width()), 1 );
lu::Full( A, p, q );
}
inline void
Wilkinson( DistMatrix<T,U,V>& A, int k )
{
#ifndef RELEASE
CallStackEntry entry("Wilkinson");
#endif
const int n = 2*k+1;
A.ResizeTo( n, n );
MakeZeros( A );
const int localHeight = A.LocalHeight();
const int localWidth = A.LocalWidth();
const int colShift = A.ColShift();
const int rowShift = A.RowShift();
const int colStride = A.ColStride();
const int rowStride = A.RowStride();
for( int jLocal=0; jLocal<localWidth; ++jLocal )
{
const int j = rowShift + jLocal*rowStride;
for( int iLocal=0; iLocal<localHeight; ++iLocal )
{
const int i = colShift + iLocal*colStride;
if( i == j )
{
if( j <= k )
A.SetLocal( iLocal, jLocal, T(k-j) );
else
A.SetLocal( iLocal, jLocal, T(j-k) );
}
else if( i == j-1 || i == j+1 )
A.SetLocal( iLocal, jLocal, T(1) );
}
}
}
inline void
Redheffer( DistMatrix<T,U,V>& R, Int n )
{
#ifndef RELEASE
CallStackEntry entry("Redheffer");
#endif
R.ResizeTo( n, n );
const Int localHeight = R.LocalHeight();
const Int localWidth = R.LocalWidth();
const Int colShift = R.ColShift();
const Int rowShift = R.RowShift();
const Int colStride = R.ColStride();
const Int rowStride = R.RowStride();
for( Int jLoc=0; jLoc<localWidth; ++jLoc )
{
const Int j = rowShift + jLoc*rowStride;
for( Int iLoc=0; iLoc<localHeight; ++iLoc )
{
const Int i = colShift + iLoc*colStride;
if( j==0 || ((j+1)%(i+1))==0 )
R.SetLocal( iLoc, jLoc, T(1) );
else
R.SetLocal( iLoc, jLoc, T(0) );
}
}
}
inline void
Hanowa( DistMatrix<T,U,V>& A, int n, T mu )
{
#ifndef RELEASE
CallStackEntry entry("Hanowa");
#endif
if( n % 2 != 0 )
throw std::logic_error("n must be an even integer");
A.ResizeTo( n, n );
const int m = n/2;
std::vector<T> d(m);
DistMatrix<T,U,V> ABlock( A.Grid() );
for( int j=0; j<m; ++j )
d[j] = mu;
View( ABlock, A, 0, 0, m, m );
Diagonal( ABlock, d );
View( ABlock, A, m, m, m, m );
Diagonal( ABlock, d );
for( int j=0; j<m; ++j )
d[j] = -(j+1);
View( ABlock, A, 0, m, m, m );
Diagonal( ABlock, d );
for( int j=0; j<m; ++j )
d[j] = j+1;
View( ABlock, A, m, 0, m, m );
Diagonal( ABlock, d );
}
inline void
Diagonal( const std::vector<T>& d, DistMatrix<T,U,V>& D )
{
#ifndef RELEASE
PushCallStack("Diagonal");
#endif
const int n = d.size();
D.ResizeTo( n, n );
MakeZeros( D );
const int localWidth = D.LocalWidth();
const int colShift = D.ColShift();
const int rowShift = D.RowShift();
const int colStride = D.ColStride();
const int rowStride = D.RowStride();
for( int jLocal=0; jLocal<localWidth; ++jLocal )
{
const int j = rowShift + jLocal*rowStride;
if( (j-colShift+colStride) % colStride == 0 )
{
const int iLocal = (j-colShift) / colStride;
D.SetLocal( iLocal, jLocal, d[j] );
}
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
Riemann( DistMatrix<T,U,V>& R, int n )
{
#ifndef RELEASE
CallStackEntry entry("Riemann");
#endif
R.ResizeTo( n, n );
const int localHeight = R.LocalHeight();
const int localWidth = R.LocalWidth();
const int colShift = R.ColShift();
const int rowShift = R.RowShift();
const int colStride = R.ColStride();
const int rowStride = R.RowStride();
for( int jLocal=0; jLocal<localWidth; ++jLocal )
{
const int j = rowShift + jLocal*rowStride;
for( int iLocal=0; iLocal<localHeight; ++iLocal )
{
const int i = colShift + iLocal*colStride;
if( ((j+2)%(i+2))==0 )
R.SetLocal( iLocal, jLocal, T(i+1) );
else
R.SetLocal( iLocal, jLocal, T(-1) );
}
}
}
inline void
Hankel( int m, int n, const std::vector<T>& a, DistMatrix<T,U,V>& A )
{
#ifndef RELEASE
PushCallStack("Hankel");
#endif
const int length = m+n-1;
if( a.size() != (unsigned)length )
throw std::logic_error("a was the wrong size");
A.ResizeTo( m, n );
const int localHeight = A.LocalHeight();
const int localWidth = A.LocalWidth();
const int colShift = A.ColShift();
const int rowShift = A.RowShift();
const int colStride = A.ColStride();
const int rowStride = A.RowStride();
for( int jLocal=0; jLocal<localWidth; ++jLocal )
{
const int j = rowShift + jLocal*rowStride;
for( int iLocal=0; iLocal<localHeight; ++iLocal )
{
const int i = colShift + iLocal*colStride;
A.SetLocal( iLocal, jLocal, a[i+j] );
}
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
Pei( DistMatrix<T,U,V>& P, Int n, T alpha )
{
#ifndef RELEASE
CallStackEntry entry("MakeIdentity");
#endif
P.ResizeTo( n, n );
const Int localHeight = P.LocalHeight();
const Int localWidth = P.LocalWidth();
const Int colShift = P.ColShift();
const Int rowShift = P.RowShift();
const Int colStride = P.ColStride();
const Int rowStride = P.RowStride();
for( Int jLoc=0; jLoc<localWidth; ++jLoc )
{
const Int j = rowShift + jLoc*rowStride;
for( Int iLoc=0; iLoc<localHeight; ++iLoc )
{
const Int i = colShift + iLoc*colStride;
P.SetLocal( iLoc, jLoc, T(1) );
if( i == j )
P.UpdateLocal( iLoc, jLoc, alpha );
}
}
}
inline void
Hankel( DistMatrix<T,U,V>& A, Int m, Int n, const std::vector<T>& a )
{
#ifndef RELEASE
CallStackEntry entry("Hankel");
#endif
const Int length = m+n-1;
if( a.size() != (Unsigned)length )
LogicError("a was the wrong size");
A.ResizeTo( m, n );
const Int localHeight = A.LocalHeight();
const Int localWidth = A.LocalWidth();
const Int colShift = A.ColShift();
const Int rowShift = A.RowShift();
const Int colStride = A.ColStride();
const Int rowStride = A.RowStride();
for( Int jLoc=0; jLoc<localWidth; ++jLoc )
{
const Int j = rowShift + jLoc*rowStride;
for( Int iLoc=0; iLoc<localHeight; ++iLoc )
{
const Int i = colShift + iLoc*colStride;
A.SetLocal( iLoc, jLoc, a[i+j] );
}
}
}
// Broadcast a matrix from the root grid to the others
void DepthBroadcast
( const mpi::Comm& depthComm,
const DistMatrix<double,MC,MR>& A,
DistMatrix<double,MC,MR>& B )
{
const int rank = mpi::CommRank(mpi::COMM_WORLD);
const Grid& meshGrid = A.Grid();
const int meshSize = meshGrid.Size();
const int depthRank = rank / meshSize;
const int localSize = A.LocalHeight()*A.LocalWidth();
if( A.LocalHeight() != A.LocalLDim() )
throw std::logic_error("Leading dimension did not match local height");
B.Empty();
B.AlignWith( A );
B.ResizeTo( A.Height(), A.Width() );
// Have the root pack the broadcast data
if( depthRank == 0 )
MemCopy( B.LocalBuffer(), A.LockedLocalBuffer(), localSize );
// Broadcast from the root
mpi::Broadcast( B.LocalBuffer(), localSize, 0, depthComm );
}
inline void
Ones( DistMatrix<T,U,V>& A, int m, int n )
{
#ifndef RELEASE
CallStackEntry entry("Ones");
#endif
A.ResizeTo( m, n );
MakeOnes( A );
}
inline void
Hilbert( DistMatrix<F,U,V>& A, Int n )
{
#ifndef RELEASE
CallStackEntry entry("Hilbert");
#endif
A.ResizeTo( n, n );
MakeHilbert( A );
}
inline void
GKS( DistMatrix<F,U,V>& A, Int n )
{
#ifndef RELEASE
CallStackEntry entry("GKS");
#endif
A.ResizeTo( n, n );
MakeGKS( A );
}
inline void
Legendre( DistMatrix<F,U,V>& A, Int n )
{
#ifndef RELEASE
CallStackEntry entry("Legendre");
#endif
A.ResizeTo( n, n );
MakeLegendre( A );
}
inline void
DiscreteFourier( DistMatrix<Complex<R>,U,V>& A, int n )
{
#ifndef RELEASE
CallStackEntry entry("DiscreteFourier");
#endif
A.ResizeTo( n, n );
MakeDiscreteFourier( A );
}
inline void
Forsythe( DistMatrix<T,U,V>& J, Int n, T alpha, T lambda )
{
#ifndef RELEASE
CallStackEntry entry("Forsythe");
#endif
J.ResizeTo( n, n );
MakeForsythe( J, alpha, lambda );
}
inline void
GCDMatrix( DistMatrix<T,U,V>& G, Int m, Int n )
{
#ifndef RELEASE
CallStackEntry entry("GCDMatrix");
#endif
G.ResizeTo( m, n );
MakeGCDMatrix( G );
}
inline void
NormalUniformSpectrum
( DistMatrix<Complex<R>,U,V>& A, Int n, Complex<R> center=0, R radius=1 )
{
#ifndef RELEASE
CallStackEntry entry("NormalUniformSpectrum");
#endif
A.ResizeTo( n, n );
MakeNormalUniformSpectrum( A, center, radius );
}
inline void
Ones( int m, int n, DistMatrix<T,U,V>& A )
{
#ifndef RELEASE
PushCallStack("Ones");
#endif
A.ResizeTo( m, n );
MakeOnes( A );
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
OneTwoOne( int n, DistMatrix<T,U,V>& A )
{
#ifndef RELEASE
PushCallStack("OneTwoOne");
#endif
A.ResizeTo( n, n );
MakeOneTwoOne( A );
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
Kahan( F phi, int n, DistMatrix<F,U,V>& A )
{
#ifndef RELEASE
PushCallStack("Kahan");
#endif
A.ResizeTo( n, n );
MakeKahan( phi, A );
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
Hilbert( int n, DistMatrix<F,U,V>& A )
{
#ifndef RELEASE
PushCallStack("Hilbert");
#endif
A.ResizeTo( n, n );
MakeHilbert( A );
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
Walsh( int k, DistMatrix<T,U,V>& A, bool binary )
{
#ifndef RELEASE
PushCallStack("Walsh");
#endif
if( k < 1 )
throw std::logic_error("Walsh matrices are only defined for k>=1");
const unsigned n = 1u<<k;
A.ResizeTo( n, n );
// Run an O(n^2 log n / p) algorithm based upon successive sign flips
const T onValue = 1;
const T offValue = ( binary ? 0 : -1 );
const unsigned localHeight = A.LocalHeight();
const unsigned localWidth = A.LocalWidth();
const unsigned colShift = A.ColShift();
const unsigned rowShift = A.RowShift();
const unsigned colStride = A.ColStride();
const unsigned rowStride = A.RowStride();
for( unsigned jLocal=0; jLocal<localWidth; ++jLocal )
{
const unsigned j = rowShift + jLocal*rowStride;
for( unsigned iLocal=0; iLocal<localHeight; ++iLocal )
{
const unsigned i = colShift + iLocal*colStride;
// Recurse on the quadtree, flipping the sign of the entry each
// time we are in the bottom-right quadrant
unsigned r = i;
unsigned s = j;
unsigned t = n;
bool on = true;
while( t != 1u )
{
t >>= 1;
if( r >= t && s >= t )
on = !on;
r %= t;
s %= t;
}
if( on )
A.SetLocal( iLocal, jLocal, onValue );
else
A.SetLocal( iLocal, jLocal, offValue );
}
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
Uniform
( int m, int n, DistMatrix<T,U,V>& A, T center, typename Base<T>::type radius )
{
#ifndef RELEASE
PushCallStack("Uniform");
#endif
A.ResizeTo( m, n );
MakeUniform( A, center, radius );
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
NormalUniformSpectrum
( int n, DistMatrix<Complex<R>,U,V>& A, Complex<R> center, R radius )
{
#ifndef RELEASE
PushCallStack("NormalUniformSpectrum");
#endif
A.ResizeTo( n, n );
MakeNormalUniformSpectrum( A, center, radius );
#ifndef RELEASE
PopCallStack();
#endif
}
inline void HermitianSVD
( UpperOrLower uplo, DistMatrix<F>& A,
DistMatrix<BASE(F),VR,STAR>& s, DistMatrix<F>& U, DistMatrix<F>& V )
{
#ifndef RELEASE
CallStackEntry entry("HermitianSVD");
#endif
#ifdef HAVE_PMRRR
typedef BASE(F) R;
// Grab an eigenvalue decomposition of A
HermitianEig( uplo, A, s, V );
// Redistribute the singular values into an [MR,* ] distribution
const Grid& grid = A.Grid();
DistMatrix<R,MR,STAR> s_MR_STAR( grid );
s_MR_STAR.AlignWith( V.DistData() );
s_MR_STAR = s;
// Set the singular values to the absolute value of the eigenvalues
const Int numLocalVals = s.LocalHeight();
for( Int iLoc=0; iLoc<numLocalVals; ++iLoc )
{
const R sigma = s.GetLocal(iLoc,0);
s.SetLocal(iLoc,0,Abs(sigma));
}
// Copy V into U (flipping the sign as necessary)
U.AlignWith( V );
U.ResizeTo( V.Height(), V.Width() );
const Int localHeight = V.LocalHeight();
const Int localWidth = V.LocalWidth();
for( Int jLoc=0; jLoc<localWidth; ++jLoc )
{
const R sigma = s_MR_STAR.GetLocal( jLoc, 0 );
F* UCol = U.Buffer( 0, jLoc );
const F* VCol = V.LockedBuffer( 0, jLoc );
if( sigma >= 0 )
for( Int iLoc=0; iLoc<localHeight; ++iLoc )
UCol[iLoc] = VCol[iLoc];
else
for( Int iLoc=0; iLoc<localHeight; ++iLoc )
UCol[iLoc] = -VCol[iLoc];
}
#else
U = A;
MakeHermitian( uplo, U );
SVD( U, s, V );
#endif // ifdef HAVE_PMRRR
}
inline void HermitianSVD
( UpperOrLower uplo,
DistMatrix<F>& A, DistMatrix<typename Base<F>::type,VR,STAR>& s,
DistMatrix<F>& U, DistMatrix<F>& V )
{
#ifndef RELEASE
PushCallStack("HermitianSVD");
#endif
typedef typename Base<F>::type R;
// Grab an eigenvalue decomposition of A
HermitianEig( uplo, A, s, V );
// Redistribute the singular values into an [MR,* ] distribution
const Grid& grid = A.Grid();
DistMatrix<R,MR,STAR> s_MR_STAR( grid );
s_MR_STAR.AlignWith( V );
s_MR_STAR = s;
// Set the singular values to the absolute value of the eigenvalues
const int numLocalVals = s.LocalHeight();
for( int iLocal=0; iLocal<numLocalVals; ++iLocal )
{
const R sigma = s.GetLocal(iLocal,0);
s.SetLocal(iLocal,0,Abs(sigma));
}
// Copy V into U (flipping the sign as necessary)
U.AlignWith( V );
U.ResizeTo( V.Height(), V.Width() );
const int localHeight = V.LocalHeight();
const int localWidth = V.LocalWidth();
for( int jLocal=0; jLocal<localWidth; ++jLocal )
{
const R sigma = s_MR_STAR.GetLocal( jLocal, 0 );
F* UCol = U.LocalBuffer( 0, jLocal );
const F* VCol = V.LockedLocalBuffer( 0, jLocal );
if( sigma >= 0 )
for( int iLocal=0; iLocal<localHeight; ++iLocal )
UCol[iLocal] = VCol[iLocal];
else
for( int iLocal=0; iLocal<localHeight; ++iLocal )
UCol[iLocal] = -VCol[iLocal];
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
CauchyLike
( const std::vector<F>& r, const std::vector<F>& s,
const std::vector<F>& x, const std::vector<F>& y,
DistMatrix<F,U,V>& A )
{
#ifndef RELEASE
PushCallStack("CauchyLike");
#endif
const int m = r.size();
const int n = s.size();
if( x.size() != (unsigned)m )
throw std::logic_error("x vector was the wrong length");
if( y.size() != (unsigned)n )
throw std::logic_error("y vector was the wrong length");
A.ResizeTo( m, n );
const int localHeight = A.LocalHeight();
const int localWidth = A.LocalWidth();
const int colShift = A.ColShift();
const int rowShift = A.RowShift();
const int colStride = A.ColStride();
const int rowStride = A.RowStride();
for( int jLocal=0; jLocal<localWidth; ++jLocal )
{
const int j = rowShift + jLocal*rowStride;
for( int iLocal=0; iLocal<localHeight; ++iLocal )
{
const int i = colShift + iLocal*colStride;
#ifndef RELEASE
// TODO: Use tolerance instead?
if( x[i] == y[j] )
{
std::ostringstream msg;
msg << "x[" << i << "] = y[" << j << "] (" << x[i]
<< ") is not allowed for Cauchy-like matrices";
throw std::logic_error( msg.str().c_str() );
}
#endif
A.SetLocal( iLocal, jLocal, r[i]*s[j]/(x[i]-y[j]) );
}
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
Cauchy
( const std::vector<F>& x, const std::vector<F>& y, DistMatrix<F,U,V>& A )
{
#ifndef RELEASE
PushCallStack("Cauchy");
#endif
const int m = x.size();
const int n = y.size();
A.ResizeTo( m, n );
const F one = F(1);
const int localHeight = A.LocalHeight();
const int localWidth = A.LocalWidth();
const int colShift = A.ColShift();
const int rowShift = A.RowShift();
const int colStride = A.ColStride();
const int rowStride = A.RowStride();
for( int jLocal=0; jLocal<localWidth; ++jLocal )
{
const int j = rowShift + jLocal*rowStride;
for( int iLocal=0; iLocal<localHeight; ++iLocal )
{
const int i = colShift + iLocal*colStride;
#ifndef RELEASE
// TODO: Use tolerance instead?
if( x[i] == y[j] )
{
std::ostringstream msg;
msg << "x[" << i << "] = y[" << j << "] (" << x[i]
<< ") is not allowed for Cauchy matrices";
throw std::logic_error( msg.str().c_str() );
}
#endif
A.SetLocal( iLocal, jLocal, one/(x[i]-y[j]) );
}
}
#ifndef RELEASE
PopCallStack();
#endif
}
// Reduce across depth to get end result C
void SumContributions
( mpi::Comm& depthComm,
const DistMatrix<double,MC,MR>& APartial,
DistMatrix<double,MC,MR>& A )
{
const int rank = mpi::CommRank( mpi::COMM_WORLD );
const Grid& meshGrid = APartial.Grid();
A.Empty();
A.AlignWith( APartial );
A.ResizeTo( APartial.Height(), APartial.Width() );
if( APartial.LocalHeight() != APartial.LocalLDim() )
throw std::logic_error
("APartial did not have matching local height/ldim");
if( A.LocalHeight() != A.LocalLDim() )
throw std::logic_error("A did not have matching local height/ldim");
const int dataSize = APartial.LocalHeight()*APartial.LocalWidth();
mpi::AllReduce
( APartial.LockedLocalBuffer(), A.LocalBuffer(), dataSize,
mpi::SUM, depthComm );
}