inline void
TrsvUN( UnitOrNonUnit diag, const DistMatrix<F>& U, DistMatrix<F>& x )
{
#ifndef RELEASE
PushCallStack("internal::TrsvUN");
if( U.Grid() != x.Grid() )
throw std::logic_error("{U,x} must be distributed over the same grid");
if( U.Height() != U.Width() )
throw std::logic_error("U must be square");
if( x.Width() != 1 && x.Height() != 1 )
throw std::logic_error("x must be a vector");
const int xLength = ( x.Width() == 1 ? x.Height() : x.Width() );
if( U.Width() != xLength )
throw std::logic_error("Nonconformal TrsvUN");
#endif
const Grid& g = U.Grid();
if( x.Width() == 1 )
{
// Matrix views
DistMatrix<F> U01(g),
U11(g);
DistMatrix<F>
xT(g), x0(g),
xB(g), x1(g),
x2(g);
// Temporary distributions
DistMatrix<F,STAR,STAR> U11_STAR_STAR(g);
DistMatrix<F,STAR,STAR> x1_STAR_STAR(g);
DistMatrix<F,MR, STAR> x1_MR_STAR(g);
DistMatrix<F,MC, STAR> z_MC_STAR(g);
// Views of z[MC,* ], which will store updates to x
DistMatrix<F,MC,STAR> z0_MC_STAR(g),
z1_MC_STAR(g);
z_MC_STAR.AlignWith( U );
Zeros( x.Height(), 1, z_MC_STAR );
// Start the algorithm
PartitionUp
( x, xT,
xB, 0 );
while( xT.Height() > 0 )
{
RepartitionUp
( xT, x0,
x1,
/**/ /**/
xB, x2 );
const int n0 = x0.Height();
const int n1 = x1.Height();
LockedView( U01, U, 0, n0, n0, n1 );
LockedView( U11, U, n0, n0, n1, n1 );
View( z0_MC_STAR, z_MC_STAR, 0, 0, n0, 1 );
View( z1_MC_STAR, z_MC_STAR, n0, 0, n1, 1 );
x1_MR_STAR.AlignWith( U01 );
//----------------------------------------------------------------//
if( x2.Height() != 0 )
x1.SumScatterUpdate( F(1), z1_MC_STAR );
x1_STAR_STAR = x1;
U11_STAR_STAR = U11;
Trsv
( UPPER, NORMAL, diag,
U11_STAR_STAR.LockedLocalMatrix(),
x1_STAR_STAR.LocalMatrix() );
x1 = x1_STAR_STAR;
x1_MR_STAR = x1_STAR_STAR;
Gemv
( NORMAL, F(-1),
U01.LockedLocalMatrix(),
x1_MR_STAR.LockedLocalMatrix(),
F(1), z0_MC_STAR.LocalMatrix() );
//----------------------------------------------------------------//
x1_MR_STAR.FreeAlignments();
SlidePartitionUp
( xT, x0,
/**/ /**/
x1,
xB, x2 );
}
}
else
{
// Matrix views
DistMatrix<F> U01(g),
U11(g);
DistMatrix<F>
xL(g), xR(g),
x0(g), x1(g), x2(g);
// Temporary distributions
DistMatrix<F,STAR,STAR> U11_STAR_STAR(g);
DistMatrix<F,STAR,STAR> x1_STAR_STAR(g);
DistMatrix<F,STAR,MR > x1_STAR_MR(g);
DistMatrix<F,MC, MR > z1(g);
DistMatrix<F,MR, MC > z1_MR_MC(g);
DistMatrix<F,STAR,MC > z_STAR_MC(g);
// Views of z[* ,MC]
DistMatrix<F,STAR,MC> z0_STAR_MC(g),
z1_STAR_MC(g);
z_STAR_MC.AlignWith( U );
Zeros( 1, x.Width(), z_STAR_MC );
// Start the algorithm
PartitionLeft( x, xL, xR, 0 );
while( xL.Width() > 0 )
{
RepartitionLeft
( xL, /**/ xR,
x0, x1, /**/ x2 );
const int n0 = x0.Width();
const int n1 = x1.Width();
LockedView( U01, U, 0, n0, n0, n1 );
LockedView( U11, U, n0, n0, n1, n1 );
View( z0_STAR_MC, z_STAR_MC, 0, 0, 1, n0 );
View( z1_STAR_MC, z_STAR_MC, 0, n0, 1, n1 );
x1_STAR_MR.AlignWith( U01 );
z1.AlignWith( x1 );
//----------------------------------------------------------------//
if( x2.Width() != 0 )
{
z1_MR_MC.SumScatterFrom( z1_STAR_MC );
z1 = z1_MR_MC;
Axpy( F(1), z1, x1 );
}
x1_STAR_STAR = x1;
U11_STAR_STAR = U11;
Trsv
( UPPER, NORMAL, diag,
U11_STAR_STAR.LockedLocalMatrix(),
x1_STAR_STAR.LocalMatrix() );
x1 = x1_STAR_STAR;
x1_STAR_MR = x1_STAR_STAR;
Gemv
( NORMAL, F(-1),
U01.LockedLocalMatrix(),
x1_STAR_MR.LockedLocalMatrix(),
F(1), z0_STAR_MC.LocalMatrix() );
//----------------------------------------------------------------//
x1_STAR_MR.FreeAlignments();
z1.FreeAlignments();
SlidePartitionLeft
( xL, /**/ xR,
x0, /**/ x1, x2 );
}
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
Symv
( UpperOrLower uplo,
T alpha, const DistMatrix<T>& A,
const DistMatrix<T>& x,
T beta, DistMatrix<T>& y,
bool conjugate=false )
{
#ifndef RELEASE
CallStackEntry entry("Symv");
if( A.Grid() != x.Grid() || x.Grid() != y.Grid() )
throw std::logic_error
("{A,x,y} must be distributed over the same grid");
if( A.Height() != A.Width() )
throw std::logic_error("A must be square");
if( ( x.Width() != 1 && x.Height() != 1 ) ||
( y.Width() != 1 && y.Height() != 1 ) )
throw std::logic_error("x and y are assumed to be vectors");
const int xLength = ( x.Width()==1 ? x.Height() : x.Width() );
const int yLength = ( y.Width()==1 ? y.Height() : y.Width() );
if( A.Height() != xLength || A.Height() != yLength )
{
std::ostringstream msg;
msg << "Nonconformal Symv: \n"
<< " A ~ " << A.Height() << " x " << A.Width() << "\n"
<< " x ~ " << x.Height() << " x " << x.Width() << "\n"
<< " y ~ " << y.Height() << " x " << y.Width() << "\n";
throw std::logic_error( msg.str() );
}
#endif
const Grid& g = A.Grid();
if( x.Width() == 1 && y.Width() == 1 )
{
// Temporary distributions
DistMatrix<T,MC,STAR> x_MC_STAR(g), z_MC_STAR(g);
DistMatrix<T,MR,STAR> x_MR_STAR(g), z_MR_STAR(g);
DistMatrix<T,MR,MC > z_MR_MC(g);
DistMatrix<T> z(g);
// Begin the algoritm
Scale( beta, y );
x_MC_STAR.AlignWith( A );
x_MR_STAR.AlignWith( A );
z_MC_STAR.AlignWith( A );
z_MR_STAR.AlignWith( A );
z.AlignWith( y );
Zeros( z_MC_STAR, y.Height(), 1 );
Zeros( z_MR_STAR, y.Height(), 1 );
//--------------------------------------------------------------------//
x_MC_STAR = x;
x_MR_STAR = x_MC_STAR;
if( uplo == LOWER )
{
internal::LocalSymvColAccumulateL
( alpha, A, x_MC_STAR, x_MR_STAR, z_MC_STAR, z_MR_STAR, conjugate );
}
else
{
internal::LocalSymvColAccumulateU
( alpha, A, x_MC_STAR, x_MR_STAR, z_MC_STAR, z_MR_STAR, conjugate );
}
z_MR_MC.SumScatterFrom( z_MR_STAR );
z = z_MR_MC;
z.SumScatterUpdate( T(1), z_MC_STAR );
Axpy( T(1), z, y );
//--------------------------------------------------------------------//
x_MC_STAR.FreeAlignments();
x_MR_STAR.FreeAlignments();
z_MC_STAR.FreeAlignments();
z_MR_STAR.FreeAlignments();
z.FreeAlignments();
}
else if( x.Width() == 1 )
{
// Temporary distributions
DistMatrix<T,MC,STAR> x_MC_STAR(g), z_MC_STAR(g);
DistMatrix<T,MR,STAR> x_MR_STAR(g), z_MR_STAR(g);
DistMatrix<T,MR,MC > z_MR_MC(g);
DistMatrix<T> z(g), zTrans(g);
// Begin the algoritm
Scale( beta, y );
x_MC_STAR.AlignWith( A );
x_MR_STAR.AlignWith( A );
z_MC_STAR.AlignWith( A );
z_MR_STAR.AlignWith( A );
z.AlignWith( y );
z_MR_MC.AlignWith( y );
Zeros( z_MC_STAR, y.Width(), 1 );
Zeros( z_MR_STAR, y.Width(), 1 );
//--------------------------------------------------------------------//
x_MC_STAR = x;
x_MR_STAR = x_MC_STAR;
if( uplo == LOWER )
{
internal::LocalSymvColAccumulateL
( alpha, A, x_MC_STAR, x_MR_STAR, z_MC_STAR, z_MR_STAR, conjugate );
}
else
{
internal::LocalSymvColAccumulateU
( alpha, A, x_MC_STAR, x_MR_STAR, z_MC_STAR, z_MR_STAR, conjugate );
}
z.SumScatterFrom( z_MC_STAR );
z_MR_MC = z;
z_MR_MC.SumScatterUpdate( T(1), z_MR_STAR );
Transpose( z_MR_MC, zTrans );
Axpy( T(1), zTrans, y );
//--------------------------------------------------------------------//
x_MC_STAR.FreeAlignments();
x_MR_STAR.FreeAlignments();
z_MC_STAR.FreeAlignments();
z_MR_STAR.FreeAlignments();
z.FreeAlignments();
z_MR_MC.FreeAlignments();
}
else if( y.Width() == 1 )
{
// Temporary distributions
DistMatrix<T,STAR,MC> x_STAR_MC(g), z_STAR_MC(g);
DistMatrix<T,STAR,MR> x_STAR_MR(g), z_STAR_MR(g);
DistMatrix<T,MR, MC> z_MR_MC(g);
DistMatrix<T> z(g), zTrans(g);
// Begin the algoritm
Scale( beta, y );
x_STAR_MC.AlignWith( A );
x_STAR_MR.AlignWith( A );
z_STAR_MC.AlignWith( A );
z_STAR_MR.AlignWith( A );
z.AlignWith( y );
z_MR_MC.AlignWith( y );
Zeros( z_STAR_MC, 1, y.Height() );
Zeros( z_STAR_MR, 1, y.Height() );
//--------------------------------------------------------------------//
x_STAR_MR = x;
x_STAR_MC = x_STAR_MR;
if( uplo == LOWER )
{
internal::LocalSymvRowAccumulateL
( alpha, A, x_STAR_MC, x_STAR_MR, z_STAR_MC, z_STAR_MR, conjugate );
}
else
{
internal::LocalSymvRowAccumulateU
( alpha, A, x_STAR_MC, x_STAR_MR, z_STAR_MC, z_STAR_MR, conjugate );
}
z.SumScatterFrom( z_STAR_MR );
z_MR_MC = z;
z_MR_MC.SumScatterUpdate( T(1), z_STAR_MC );
Transpose( z_MR_MC, zTrans );
Axpy( T(1), zTrans, y );
//--------------------------------------------------------------------//
x_STAR_MC.FreeAlignments();
x_STAR_MR.FreeAlignments();
z_STAR_MC.FreeAlignments();
z_STAR_MR.FreeAlignments();
z.FreeAlignments();
z_MR_MC.FreeAlignments();
}
else
{
// Temporary distributions
DistMatrix<T,STAR,MC> x_STAR_MC(g), z_STAR_MC(g);
DistMatrix<T,STAR,MR> x_STAR_MR(g), z_STAR_MR(g);
DistMatrix<T,MR, MC> z_MR_MC(g);
DistMatrix<T> z(g);
// Begin the algoritm
Scale( beta, y );
x_STAR_MC.AlignWith( A );
x_STAR_MR.AlignWith( A );
z_STAR_MC.AlignWith( A );
z_STAR_MR.AlignWith( A );
z.AlignWith( y );
z_MR_MC.AlignWith( y );
Zeros( z_STAR_MC, 1, y.Width() );
Zeros( z_STAR_MR, 1, y.Width() );
//--------------------------------------------------------------------//
x_STAR_MR = x;
x_STAR_MC = x_STAR_MR;
if( uplo == LOWER )
{
internal::LocalSymvRowAccumulateL
( alpha, A, x_STAR_MC, x_STAR_MR, z_STAR_MC, z_STAR_MR, conjugate );
}
else
{
internal::LocalSymvRowAccumulateU
( alpha, A, x_STAR_MC, x_STAR_MR, z_STAR_MC, z_STAR_MR, conjugate );
}
z_MR_MC.SumScatterFrom( z_STAR_MC );
z = z_MR_MC;
z.SumScatterUpdate( T(1), z_STAR_MR );
Axpy( T(1), z, y );
//--------------------------------------------------------------------//
x_STAR_MC.FreeAlignments();
x_STAR_MR.FreeAlignments();
z_STAR_MC.FreeAlignments();
z_STAR_MR.FreeAlignments();
z.FreeAlignments();
z_MR_MC.FreeAlignments();
}
}
inline void
PanelLQ( DistMatrix<Real>& A )
{
#ifndef RELEASE
PushCallStack("internal::PanelLQ");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<Real>
ATL(g), ATR(g), A00(g), a01(g), A02(g), aTopRow(g), ABottomPan(g),
ABL(g), ABR(g), a10(g), alpha11(g), a12(g),
A20(g), a21(g), A22(g);
// Temporary distributions
DistMatrix<Real,STAR,MR> aTopRow_STAR_MR(g);
DistMatrix<Real,MC,STAR> z_MC_STAR(g);
PushBlocksizeStack( 1 );
PartitionDownLeftDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
while( ATL.Height() < A.Height() && ATL.Width() < A.Width() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ a01, A02,
/*************/ /**********************/
/**/ a10, /**/ alpha11, a12,
ABL, /**/ ABR, A20, /**/ a21, A22 );
aTopRow.View1x2( alpha11, a12 );
ABottomPan.View1x2( a21, A22 );
aTopRow_STAR_MR.AlignWith( ABottomPan );
z_MC_STAR.AlignWith( ABottomPan );
Zeros( ABottomPan.Height(), 1, z_MC_STAR );
//--------------------------------------------------------------------//
const Real tau = Reflector( alpha11, a12 );
const bool myDiagonalEntry = ( g.Row() == alpha11.ColAlignment() &&
g.Col() == alpha11.RowAlignment() );
Real alpha = 0;
if( myDiagonalEntry )
{
alpha = alpha11.GetLocal(0,0);
alpha11.SetLocal(0,0,1);
}
aTopRow_STAR_MR = aTopRow;
Gemv
( NORMAL,
Real(1), ABottomPan.LockedLocalMatrix(),
aTopRow_STAR_MR.LockedLocalMatrix(),
Real(0), z_MC_STAR.LocalMatrix() );
z_MC_STAR.SumOverRow();
Ger
( -tau,
z_MC_STAR.LockedLocalMatrix(),
aTopRow_STAR_MR.LockedLocalMatrix(),
ABottomPan.LocalMatrix() );
if( myDiagonalEntry )
alpha11.SetLocal(0,0,alpha);
//--------------------------------------------------------------------//
aTopRow_STAR_MR.FreeAlignments();
z_MC_STAR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, a01, /**/ A02,
/**/ a10, alpha11, /**/ a12,
/*************/ /**********************/
ABL, /**/ ABR, A20, a21, /**/ A22 );
}
PopBlocksizeStack();
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
PanelHouseholder( DistMatrix<F>& A, DistMatrix<F,MD,STAR>& t )
{
#ifndef RELEASE
CallStackEntry entry("lq::PanelHouseholder");
if( A.Grid() != t.Grid() )
LogicError("{A,t} must be distributed over the same grid");
if( t.Height() != Min(A.Height(),A.Width()) || t.Width() != 1 )
LogicError
("t must be a vector of height equal to the minimum dimension of A");
if( !t.AlignedWithDiagonal( A, 0 ) )
LogicError("t must be aligned with A's main diagonal");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), a01(g), A02(g), aTopRow(g), ABottomPan(g),
ABL(g), ABR(g), a10(g), alpha11(g), a12(g),
A20(g), a21(g), A22(g);
DistMatrix<F,MD,STAR>
tT(g), t0(g),
tB(g), tau1(g),
t2(g);
// Temporary distributions
DistMatrix<F> aTopRowConj(g);
DistMatrix<F,STAR,MR > aTopRowConj_STAR_MR(g);
DistMatrix<F,MC, STAR> z_MC_STAR(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
PartitionDown
( t, tT,
tB, 0 );
while( ATL.Height() < A.Height() && ATL.Width() < A.Width() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ a01, A02,
/*************/ /**********************/
/**/ a10, /**/ alpha11, a12,
ABL, /**/ ABR, A20, /**/ a21, A22, 1 );
RepartitionDown
( tT, t0,
/**/ /****/
tau1,
tB, t2, 1 );
View1x2( aTopRow, alpha11, a12 );
View1x2( ABottomPan, a21, A22 );
aTopRowConj_STAR_MR.AlignWith( ABottomPan );
z_MC_STAR.AlignWith( ABottomPan );
//--------------------------------------------------------------------//
// Compute the Householder reflector
const F tau = Reflector( alpha11, a12 );
tau1.Set( 0, 0, tau );
// Apply the Householder reflector
const bool myDiagonalEntry = ( g.Row() == alpha11.ColAlignment() &&
g.Col() == alpha11.RowAlignment() );
F alpha = 0;
if( myDiagonalEntry )
{
alpha = alpha11.GetLocal(0,0);
alpha11.SetLocal(0,0,1);
}
Conjugate( aTopRow, aTopRowConj );
aTopRowConj_STAR_MR = aTopRowConj;
Zeros( z_MC_STAR, ABottomPan.Height(), 1 );
LocalGemv
( NORMAL, F(1), ABottomPan, aTopRowConj_STAR_MR, F(0), z_MC_STAR );
z_MC_STAR.SumOverRow();
Ger
( -Conj(tau),
z_MC_STAR.LockedMatrix(),
aTopRowConj_STAR_MR.LockedMatrix(),
ABottomPan.Matrix() );
if( myDiagonalEntry )
alpha11.SetLocal(0,0,alpha);
//--------------------------------------------------------------------//
SlidePartitionDown
( tT, t0,
tau1,
/**/ /****/
tB, t2 );
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, a01, /**/ A02,
/**/ a10, alpha11, /**/ a12,
/*************/ /**********************/
ABL, /**/ ABR, A20, a21, /**/ A22 );
}
}
inline void
PanelLQ
( DistMatrix<Complex<Real> >& A,
DistMatrix<Complex<Real>,MD,STAR>& t )
{
#ifndef RELEASE
PushCallStack("internal::PanelLQ");
if( A.Grid() != t.Grid() )
throw std::logic_error("{A,t} must be distributed over the same grid");
if( t.Height() != std::min(A.Height(),A.Width()) || t.Width() != 1 )
throw std::logic_error
("t must be a vector of height equal to the minimum dimension of A");
if( !t.AlignedWithDiagonal( A, 0 ) )
throw std::logic_error("t must be aligned with A's main diagonal");
#endif
typedef Complex<Real> C;
const Grid& g = A.Grid();
// Matrix views
DistMatrix<C>
ATL(g), ATR(g), A00(g), a01(g), A02(g), aTopRow(g), ABottomPan(g),
ABL(g), ABR(g), a10(g), alpha11(g), a12(g),
A20(g), a21(g), A22(g);
DistMatrix<C,MD,STAR>
tT(g), t0(g),
tB(g), tau1(g),
t2(g);
// Temporary distributions
DistMatrix<C> aTopRowConj(g);
DistMatrix<C,STAR,MR > aTopRowConj_STAR_MR(g);
DistMatrix<C,MC, STAR> z_MC_STAR(g);
PushBlocksizeStack( 1 );
PartitionDownLeftDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
PartitionDown
( t, tT,
tB, 0 );
while( ATL.Height() < A.Height() && ATL.Width() < A.Width() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ a01, A02,
/*************/ /**********************/
/**/ a10, /**/ alpha11, a12,
ABL, /**/ ABR, A20, /**/ a21, A22 );
RepartitionDown
( tT, t0,
/**/ /****/
tau1,
tB, t2 );
aTopRow.View1x2( alpha11, a12 );
ABottomPan.View1x2( a21, A22 );
aTopRowConj_STAR_MR.AlignWith( ABottomPan );
z_MC_STAR.AlignWith( ABottomPan );
Zeros( ABottomPan.Height(), 1, z_MC_STAR );
//--------------------------------------------------------------------//
const C tau = Reflector( alpha11, a12 );
tau1.Set( 0, 0, tau );
const bool myDiagonalEntry = ( g.Row() == alpha11.ColAlignment() &&
g.Col() == alpha11.RowAlignment() );
C alpha = 0;
if( myDiagonalEntry )
{
alpha = alpha11.GetLocal(0,0);
alpha11.SetLocal(0,0,1);
}
Conjugate( aTopRow, aTopRowConj );
aTopRowConj_STAR_MR = aTopRowConj;
Gemv
( NORMAL,
C(1), ABottomPan.LockedLocalMatrix(),
aTopRowConj_STAR_MR.LockedLocalMatrix(),
C(0), z_MC_STAR.LocalMatrix() );
z_MC_STAR.SumOverRow();
Ger
( -Conj(tau),
z_MC_STAR.LockedLocalMatrix(),
aTopRowConj_STAR_MR.LockedLocalMatrix(),
ABottomPan.LocalMatrix() );
if( myDiagonalEntry )
alpha11.SetLocal(0,0,alpha);
//--------------------------------------------------------------------//
aTopRowConj_STAR_MR.FreeAlignments();
z_MC_STAR.FreeAlignments();
SlidePartitionDown
( tT, t0,
tau1,
/**/ /****/
tB, t2 );
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, a01, /**/ A02,
/**/ a10, alpha11, /**/ a12,
/*************/ /**********************/
ABL, /**/ ABR, A20, a21, /**/ A22 );
}
PopBlocksizeStack();
#ifndef RELEASE
PopCallStack();
#endif
}
