void EntrywiseMap
( const ElementalMatrix<S>& A,
ElementalMatrix<T>& B,
function<T(S)> func )
{
if( A.DistData().colDist == B.DistData().colDist &&
A.DistData().rowDist == B.DistData().rowDist )
{
B.AlignWith( A.DistData() );
B.Resize( A.Height(), A.Width() );
EntrywiseMap( A.LockedMatrix(), B.Matrix(), func );
}
else
{
B.Resize( A.Height(), A.Width() );
#define GUARD(CDIST,RDIST) \
B.DistData().colDist == CDIST && B.DistData().rowDist == RDIST
#define PAYLOAD(CDIST,RDIST) \
DistMatrix<S,CDIST,RDIST> AProx(B.Grid()); \
AProx.AlignWith( B.DistData() ); \
Copy( A, AProx ); \
EntrywiseMap( AProx.Matrix(), B.Matrix(), func );
#include <El/macros/GuardAndPayload.h>
#undef GUARD
#undef PAYLOAD
}
}
void TransposeContract
( const ElementalMatrix<T>& A,
ElementalMatrix<T>& B, bool conjugate )
{
EL_DEBUG_CSE
const Dist U = B.ColDist();
const Dist V = B.RowDist();
if( A.ColDist() == V && A.RowDist() == Partial(U) )
{
Transpose( A, B, conjugate );
}
else
{
unique_ptr<ElementalMatrix<T>>
ASumFilt( B.ConstructTranspose(B.Grid(),B.Root()) );
if( B.ColConstrained() )
ASumFilt->AlignRowsWith( B, true );
if( B.RowConstrained() )
ASumFilt->AlignColsWith( B, true );
Contract( A, *ASumFilt );
if( !B.ColConstrained() )
B.AlignColsWith( *ASumFilt, false );
if( !B.RowConstrained() )
B.AlignRowsWith( *ASumFilt, false );
// We should have ensured that the alignments match
B.Resize( A.Width(), A.Height() );
Transpose( ASumFilt->LockedMatrix(), B.Matrix(), conjugate );
}
}
void ExtendedKahan( ElementalMatrix<F>& A, Int k, Base<F> phi, Base<F> mu )
{
EL_DEBUG_CSE
const Int n = 3*(1u<<k);
A.Resize( n, n );
MakeExtendedKahan( A, phi, mu );
}
void UniformHelmholtzGreens
( ElementalMatrix<Complex<Real>>& A, Int n, Real lambda )
{
EL_DEBUG_CSE
typedef Complex<Real> C;
const Real pi = 4*Atan( Real(1) );
const Real k0 = 2*pi/lambda;
const Grid& g = A.Grid();
// Generate a list of n uniform samples from the 3D unit ball
DistMatrix<Real,STAR,VR> X_STAR_VR(3,n,g);
for( Int jLoc=0; jLoc<X_STAR_VR.LocalWidth(); ++jLoc )
{
Real x0, x1, x2;
// Sample uniformly from [-1,+1]^3 until a point is drawn from the ball
while( true )
{
x0 = SampleUniform( Real(-1), Real(1) );
x1 = SampleUniform( Real(-1), Real(1) );
x2 = SampleUniform( Real(-1), Real(1) );
const Real radiusSq = x0*x0 + x1*x1 + x2*x2;
if( radiusSq > 0 && radiusSq <= 1 )
break;
}
X_STAR_VR.SetLocal( 0, jLoc, x0 );
X_STAR_VR.SetLocal( 1, jLoc, x1 );
X_STAR_VR.SetLocal( 2, jLoc, x2 );
}
DistMatrix<Real,STAR,STAR> X_STAR_STAR( X_STAR_VR );
A.Resize( n, n );
for( Int jLoc=0; jLoc<A.LocalWidth(); ++jLoc )
{
const Int j = A.GlobalCol(jLoc);
const Real xj0 = X_STAR_STAR.GetLocal(0,j);
const Real xj1 = X_STAR_STAR.GetLocal(1,j);
const Real xj2 = X_STAR_STAR.GetLocal(2,j);
for( Int iLoc=0; iLoc<A.LocalHeight(); ++iLoc )
{
const Int i = A.GlobalRow(iLoc);
if( i == j )
{
A.SetLocal( iLoc, jLoc, 0 );
}
else
{
const Real d0 = X_STAR_STAR.GetLocal(0,i)-xj0;
const Real d1 = X_STAR_STAR.GetLocal(1,i)-xj1;
const Real d2 = X_STAR_STAR.GetLocal(2,i)-xj2;
const Real gamma = k0*Sqrt(d0*d0+d1*d1+d2*d2);
const Real realPart = Cos(gamma)/gamma;
const Real imagPart = Sin(gamma)/gamma;
A.SetLocal( iLoc, jLoc, C(realPart,imagPart) );
}
}
}
}
void SDC
( UpperOrLower uplo,
ElementalMatrix<F>& APre,
ElementalMatrix<Base<F>>& wPre,
const HermitianSDCCtrl<Base<F>> ctrl )
{
DEBUG_CSE
typedef Base<F> Real;
const Int n = APre.Height();
wPre.Resize( n, 1 );
if( APre.Grid().Size() == 1 )
{
HermitianEig( uplo, APre.Matrix(), wPre.Matrix() );
return;
}
if( n <= ctrl.cutoff )
{
HermitianEig( uplo, APre, wPre );
return;
}
DistMatrixReadWriteProxy<F,F,MC,MR> AProx( APre );
DistMatrixWriteProxy<Real,Real,VR,STAR> wProx( wPre );
auto& A = AProx.Get();
auto& w = wProx.Get();
// Perform this level's split
const auto part = SpectralDivide( uplo, A, ctrl );
auto ind1 = IR(0,part.index);
auto ind2 = IR(part.index,n);
auto ATL = A( ind1, ind1 );
auto ATR = A( ind1, ind2 );
auto ABL = A( ind2, ind1 );
auto ABR = A( ind2, ind2 );
auto wT = w( ind1, ALL );
auto wB = w( ind2, ALL );
if( uplo == LOWER )
Zero( ABL );
else
Zero( ATR );
// Recurse on the two subproblems
DistMatrix<F> ATLSub, ABRSub;
DistMatrix<Real,VR,STAR> wTSub, wBSub;
PushSubproblems
( ATL, ABR, ATLSub, ABRSub, wT, wB, wTSub, wBSub, ctrl.progress );
if( ATL.Participating() )
SDC( uplo, ATLSub, wTSub, ctrl );
if( ABR.Participating() )
SDC( uplo, ABRSub, wBSub, ctrl );
PullSubproblems( ATL, ABR, ATLSub, ABRSub, wT, wB, wTSub, wBSub );
}
void ExplicitTriang( ElementalMatrix<F>& A )
{
DEBUG_CSE
const Grid& g = A.Grid();
DistMatrix<F,MD,STAR> householderScalars(g);
DistMatrix<Base<F>,MD,STAR> signature(g);
LQ( A, householderScalars, signature );
const Int m = A.Height();
const Int n = A.Width();
const Int minDim = Min(m,n);
A.Resize( m, minDim );
MakeTrapezoidal( LOWER, A );
}
void ExplicitTriang( ElementalMatrix<F>& A, const QRCtrl<Base<F>>& ctrl )
{
DEBUG_CSE
DistMatrix<F,MD,STAR> householderScalars(A.Grid());
DistMatrix<Base<F>,MD,STAR> signature(A.Grid());
if( ctrl.colPiv )
{
DistPermutation Omega(A.Grid());
BusingerGolub( A, householderScalars, signature, Omega, ctrl );
}
else
Householder( A, householderScalars, signature );
A.Resize( householderScalars.Height(), A.Width() );
MakeTrapezoidal( UPPER, A );
}
void ExplicitTriang( ElementalMatrix<F>& A, const QRCtrl<Base<F>>& ctrl )
{
DEBUG_ONLY(CSE cse("qr::ExplicitTriang"))
DistMatrix<F,MD,STAR> t(A.Grid());
DistMatrix<Base<F>,MD,STAR> d(A.Grid());
if( ctrl.colPiv )
{
DistPermutation Omega(A.Grid());
BusingerGolub( A, t, d, Omega, ctrl );
}
else
Householder( A, t, d );
A.Resize( t.Height(), A.Width() );
MakeTrapezoidal( UPPER, A );
}
void ReshapeIntoGrid
( Int realSize, Int imagSize,
const ElementalMatrix<T>& x, ElementalMatrix<T>& X )
{
X.SetGrid( x.Grid() );
X.Resize( imagSize, realSize );
auto xSub = unique_ptr<ElementalMatrix<T>>
( x.Construct(x.Grid(),x.Root()) );
auto XSub = unique_ptr<ElementalMatrix<T>>
( X.Construct(X.Grid(),X.Root()) );
for( Int j=0; j<realSize; ++j )
{
View( *XSub, X, IR(0,imagSize), IR(j) );
LockedView( *xSub, x, IR(j*imagSize,(j+1)*imagSize), ALL );
Copy( *xSub, *XSub );
}
}
void IndexDependentMap
( const ElementalMatrix<S>& A,
ElementalMatrix<T>& B,
function<T(Int,Int,S)> func )
{
DEBUG_CSE
const Int mLoc = A.LocalHeight();
const Int nLoc = A.LocalWidth();
B.AlignWith( A.DistData() );
B.Resize( A.Height(), A.Width() );
auto& ALoc = A.LockedMatrix();
auto& BLoc = B.Matrix();
for( Int jLoc=0; jLoc<nLoc; ++jLoc )
{
const Int j = A.GlobalCol(jLoc);
for( Int iLoc=0; iLoc<mLoc; ++iLoc )
{
const Int i = A.GlobalRow(iLoc);
BLoc(iLoc,jLoc) = func(i,j,ALoc(iLoc,jLoc));
}
}
}
void SDC
( UpperOrLower uplo,
ElementalMatrix<F>& APre,
ElementalMatrix<Base<F>>& wPre,
ElementalMatrix<F>& QPre,
const HermitianSDCCtrl<Base<F>> ctrl )
{
DEBUG_CSE
typedef Base<F> Real;
const Grid& g = APre.Grid();
const Int n = APre.Height();
wPre.Resize( n, 1 );
QPre.Resize( n, n );
if( APre.Grid().Size() == 1 )
{
HermitianEig( uplo, APre.Matrix(), wPre.Matrix(), QPre.Matrix() );
return;
}
if( n <= ctrl.cutoff )
{
HermitianEig( uplo, APre, wPre, QPre );
return;
}
DistMatrixReadWriteProxy<F,F,MC,MR> AProx( APre );
DistMatrixWriteProxy<F,F,MC,MR> QProx( QPre );
DistMatrixWriteProxy<Real,Real,VR,STAR> wProx( wPre );
auto& A = AProx.Get();
auto& Q = QProx.Get();
auto& w = wProx.Get();
// Perform this level's split
const auto part = SpectralDivide( uplo, A, Q, ctrl );
auto ind1 = IR(0,part.index);
auto ind2 = IR(part.index,n);
auto ATL = A( ind1, ind1 );
auto ATR = A( ind1, ind2 );
auto ABL = A( ind2, ind1 );
auto ABR = A( ind2, ind2 );
auto wT = w( ind1, ALL );
auto wB = w( ind2, ALL );
auto QL = Q( ALL, ind1 );
auto QR = Q( ALL, ind2 );
if( uplo == LOWER )
Zero( ABL );
else
Zero( ATR );
// Recurse on the two subproblems
DistMatrix<F> ATLSub, ABRSub, ZTSub, ZBSub;
DistMatrix<Real,VR,STAR> wTSub, wBSub;
PushSubproblems
( ATL, ABR, ATLSub, ABRSub, wT, wB, wTSub, wBSub, ZTSub, ZBSub,
ctrl.progress );
if( ATLSub.Participating() )
SDC( uplo, ATLSub, wTSub, ZTSub, ctrl );
if( ABRSub.Participating() )
SDC( uplo, ABRSub, wBSub, ZBSub, ctrl );
// Pull the results back to this grid
DistMatrix<F> ZT(g), ZB(g);
PullSubproblems
( ATL, ABR, ATLSub, ABRSub, wT, wB, wTSub, wBSub, ZT, ZB, ZTSub, ZBSub );
// Update the eigen vectors
auto G( QL );
Gemm( NORMAL, NORMAL, F(1), G, ZT, QL );
G = QR;
Gemm( NORMAL, NORMAL, F(1), G, ZB, QR );
}
void Scatter
( const DistMatrix<T,CIRC,CIRC>& A,
ElementalMatrix<T>& B )
{
DEBUG_CSE
AssertSameGrids( A, B );
const Int m = A.Height();
const Int n = A.Width();
const Int colStride = B.ColStride();
const Int rowStride = B.RowStride();
B.Resize( m, n );
if( B.CrossSize() != 1 || B.RedundantSize() != 1 )
{
// TODO:
// Broadcast over the redundant communicator and use mpi::Translate
// rank to determine whether a process is the root of the broadcast.
GeneralPurpose( A, B );
return;
}
const Int pkgSize = mpi::Pad(MaxLength(m,colStride)*MaxLength(n,rowStride));
const Int recvSize = pkgSize;
const Int sendSize = B.DistSize()*pkgSize;
// Translate the root of A into the DistComm of B (if possible)
const Int root = A.Root();
const Int target = mpi::Translate( A.CrossComm(), root, B.DistComm() );
if( target == mpi::UNDEFINED )
return;
if( B.DistSize() == 1 )
{
Copy( A.LockedMatrix(), B.Matrix() );
return;
}
vector<T> buffer;
T* recvBuf=0; // some compilers (falsely) warn otherwise
if( A.CrossRank() == root )
{
FastResize( buffer, sendSize+recvSize );
T* sendBuf = &buffer[0];
recvBuf = &buffer[sendSize];
// Pack the send buffer
copy::util::StridedPack
( m, n,
B.ColAlign(), colStride,
B.RowAlign(), rowStride,
A.LockedBuffer(), A.LDim(),
sendBuf, pkgSize );
// Scatter from the root
mpi::Scatter
( sendBuf, pkgSize, recvBuf, pkgSize, target, B.DistComm() );
}
else
{
FastResize( buffer, recvSize );
recvBuf = &buffer[0];
// Perform the receiving portion of the scatter from the non-root
mpi::Scatter
( static_cast<T*>(0), pkgSize,
recvBuf, pkgSize, target, B.DistComm() );
}
// Unpack
copy::util::InterleaveMatrix
( B.LocalHeight(), B.LocalWidth(),
recvBuf, 1, B.LocalHeight(),
B.Buffer(), 1, B.LDim() );
}