inline void
internal::HegstRLVar2( DistMatrix<F,MC,MR>& A, const DistMatrix<F,MC,MR>& L )
{
#ifndef RELEASE
PushCallStack("internal::HegstRLVar2");
if( A.Height() != A.Width() )
throw std::logic_error("A must be square");
if( L.Height() != L.Width() )
throw std::logic_error("Triangular matrices must be square");
if( A.Height() != L.Height() )
throw std::logic_error("A and L must be the same size");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F,MC,MR>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F,MC,MR>
LTL(g), LTR(g), L00(g), L01(g), L02(g),
LBL(g), LBR(g), L10(g), L11(g), L12(g),
L20(g), L21(g), L22(g);
// Temporary distributions
DistMatrix<F,MR, STAR> A10Adj_MR_STAR(g);
DistMatrix<F,STAR,VR > A10_STAR_VR(g);
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,VC, STAR> A21_VC_STAR(g);
DistMatrix<F,MR, STAR> F10Adj_MR_STAR(g);
DistMatrix<F,MR, STAR> L10Adj_MR_STAR(g);
DistMatrix<F,VC, STAR> L10Adj_VC_STAR(g);
DistMatrix<F,STAR,MC > L10_STAR_MC(g);
DistMatrix<F,STAR,STAR> L11_STAR_STAR(g);
DistMatrix<F,MC, MR > X11(g);
DistMatrix<F,MC, STAR> X11_MC_STAR(g);
DistMatrix<F,MC, STAR> X21_MC_STAR(g);
DistMatrix<F,MC, MR > Y10Adj(g);
DistMatrix<F,MC, STAR> Y10Adj_MC_STAR(g);
DistMatrix<F,MR, MC > Y10Adj_MR_MC(g);
Matrix<F> Y10Local;
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( L, LTL, LTR,
LBL, LBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( LTL, /**/ LTR, L00, /**/ L01, L02,
/*************/ /******************/
/**/ L10, /**/ L11, L12,
LBL, /**/ LBR, L20, /**/ L21, L22 );
A10Adj_MR_STAR.AlignWith( L10 );
F10Adj_MR_STAR.AlignWith( A00 );
L10Adj_MR_STAR.AlignWith( A00 );
L10Adj_VC_STAR.AlignWith( A00 );
L10_STAR_MC.AlignWith( A00 );
X11.AlignWith( A11 );
X11_MC_STAR.AlignWith( L10 );
X21_MC_STAR.AlignWith( A20 );
Y10Adj_MC_STAR.AlignWith( A00 );
Y10Adj_MR_MC.AlignWith( A10 );
//--------------------------------------------------------------------//
// Y10 := L10 A00
L10Adj_MR_STAR.AdjointFrom( L10 );
L10Adj_VC_STAR = L10Adj_MR_STAR;
L10_STAR_MC.AdjointFrom( L10Adj_VC_STAR );
Y10Adj_MC_STAR.ResizeTo( A10.Width(), A10.Height() );
F10Adj_MR_STAR.ResizeTo( A10.Width(), A10.Height() );
Zero( Y10Adj_MC_STAR );
Zero( F10Adj_MR_STAR );
internal::LocalSymmetricAccumulateRL
( ADJOINT,
(F)1, A00, L10_STAR_MC, L10Adj_MR_STAR,
Y10Adj_MC_STAR, F10Adj_MR_STAR );
Y10Adj.SumScatterFrom( Y10Adj_MC_STAR );
Y10Adj_MR_MC = Y10Adj;
Y10Adj_MR_MC.SumScatterUpdate( (F)1, F10Adj_MR_STAR );
Adjoint( Y10Adj_MR_MC.LockedLocalMatrix(), Y10Local );
// X11 := A10 L10'
X11_MC_STAR.ResizeTo( A11.Height(), A11.Width() );
internal::LocalGemm
( NORMAL, NORMAL, (F)1, A10, L10Adj_MR_STAR, (F)0, X11_MC_STAR );
// A10 := A10 - Y10
Axpy( (F)-1, Y10Local, A10.LocalMatrix() );
A10Adj_MR_STAR.AdjointFrom( A10 );
// A11 := A11 - (X11 + L10 A10') = A11 - (A10 L10' + L10 A10')
internal::LocalGemm
( NORMAL, NORMAL,
(F)1, L10, A10Adj_MR_STAR, (F)1, X11_MC_STAR );
X11.SumScatterFrom( X11_MC_STAR );
MakeTrapezoidal( LEFT, LOWER, 0, X11 );
Axpy( (F)-1, X11, A11 );
// A10 := inv(L11) A10
L11_STAR_STAR = L11;
A10_STAR_VR.AdjointFrom( A10Adj_MR_STAR );
internal::LocalTrsm
( LEFT, LOWER, NORMAL, NON_UNIT, (F)1, L11_STAR_STAR, A10_STAR_VR );
A10 = A10_STAR_VR;
// A11 := inv(L11) A11 inv(L11)'
A11_STAR_STAR = A11;
internal::LocalHegst( RIGHT, LOWER, A11_STAR_STAR, L11_STAR_STAR );
A11 = A11_STAR_STAR;
// A21 := A21 - A20 L10'
X21_MC_STAR.ResizeTo( A21.Height(), A21.Width() );
internal::LocalGemm
( NORMAL, NORMAL,
(F)1, A20, L10Adj_MR_STAR, (F)0, X21_MC_STAR );
A21.SumScatterUpdate( (F)-1, X21_MC_STAR );
// A21 := A21 inv(L11)'
A21_VC_STAR = A21;
internal::LocalTrsm
( RIGHT, LOWER, ADJOINT, NON_UNIT, (F)1, L11_STAR_STAR, A21_VC_STAR );
A21 = A21_VC_STAR;
//--------------------------------------------------------------------//
A10Adj_MR_STAR.FreeAlignments();
F10Adj_MR_STAR.FreeAlignments();
L10Adj_MR_STAR.FreeAlignments();
L10Adj_VC_STAR.FreeAlignments();
L10_STAR_MC.FreeAlignments();
X11.FreeAlignments();
X11_MC_STAR.FreeAlignments();
X21_MC_STAR.FreeAlignments();
Y10Adj_MC_STAR.FreeAlignments();
Y10Adj_MR_MC.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( LTL, /**/ LTR, L00, L01, /**/ L02,
/**/ L10, L11, /**/ L12,
/**********************************/
LBL, /**/ LBR, L20, L21, /**/ L22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
LocalTrmmAccumulateRUN
( Orientation orientation, UnitOrNonUnit diag, T alpha,
const DistMatrix<T,MC, MR >& U,
const DistMatrix<T,STAR,MC >& X_STAR_MC,
DistMatrix<T,MR, STAR>& ZTrans_MR_STAR )
{
#ifndef RELEASE
CallStackEntry entry("internal::LocalTrmmAccumulateRUN");
if( U.Grid() != X_STAR_MC.Grid() ||
X_STAR_MC.Grid() != ZTrans_MR_STAR.Grid() )
throw std::logic_error
("{U,X,Z} must be distributed over the same grid");
if( U.Height() != U.Width() ||
U.Height() != X_STAR_MC.Width() ||
U.Height() != ZTrans_MR_STAR.Height() )
{
std::ostringstream msg;
msg << "Nonconformal LocalTrmmAccumulateRUN: \n"
<< " U ~ " << U.Height() << " x " << U.Width() << "\n"
<< " X[* ,MC] ~ " << X_STAR_MC.Height() << " x "
<< X_STAR_MC.Width() << "\n"
<< " Z^H/T[MR,* ] ~ " << ZTrans_MR_STAR.Height() << " x "
<< ZTrans_MR_STAR.Width() << "\n";
throw std::logic_error( msg.str().c_str() );
}
if( X_STAR_MC.RowAlignment() != U.ColAlignment() ||
ZTrans_MR_STAR.ColAlignment() != U.RowAlignment() )
throw std::logic_error("Partial matrix distributions are misaligned");
#endif
const Grid& g = U.Grid();
// Matrix views
DistMatrix<T>
UTL(g), UTR(g), U00(g), U01(g), U02(g),
UBL(g), UBR(g), U10(g), U11(g), U12(g),
U20(g), U21(g), U22(g);
DistMatrix<T> D11(g);
DistMatrix<T,STAR,MC>
XL_STAR_MC(g), XR_STAR_MC(g),
X0_STAR_MC(g), X1_STAR_MC(g), X2_STAR_MC(g);
DistMatrix<T,MR,STAR>
ZTTrans_MR_STAR(g), Z0Trans_MR_STAR(g),
ZBTrans_MR_STAR(g), Z1Trans_MR_STAR(g),
Z2Trans_MR_STAR(g);
const int ratio = std::max( g.Height(), g.Width() );
PushBlocksizeStack( ratio*Blocksize() );
LockedPartitionDownDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
LockedPartitionRight( X_STAR_MC, XL_STAR_MC, XR_STAR_MC, 0 );
PartitionDown
( ZTrans_MR_STAR, ZTTrans_MR_STAR,
ZBTrans_MR_STAR, 0 );
while( UTL.Height() < U.Height() )
{
LockedRepartitionDownDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
LockedRepartitionRight
( XL_STAR_MC, /**/ XR_STAR_MC,
X0_STAR_MC, /**/ X1_STAR_MC, X2_STAR_MC );
RepartitionDown
( ZTTrans_MR_STAR, Z0Trans_MR_STAR,
/***************/ /***************/
Z1Trans_MR_STAR,
ZBTrans_MR_STAR, Z2Trans_MR_STAR );
D11.AlignWith( U11 );
//--------------------------------------------------------------------//
D11 = U11;
MakeTriangular( UPPER, D11 );
if( diag == UNIT )
SetDiagonal( D11, T(1) );
LocalGemm
( orientation, orientation,
alpha, D11, X1_STAR_MC, T(1), Z1Trans_MR_STAR );
LocalGemm
( orientation, orientation,
alpha, U01, X0_STAR_MC, T(1), Z1Trans_MR_STAR );
//--------------------------------------------------------------------//
D11.FreeAlignments();
SlideLockedPartitionDownDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/*************/ /******************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
SlideLockedPartitionRight
( XL_STAR_MC, /**/ XR_STAR_MC,
X0_STAR_MC, X1_STAR_MC, /**/ X2_STAR_MC );
SlidePartitionDown
( ZTTrans_MR_STAR, Z0Trans_MR_STAR,
Z1Trans_MR_STAR,
/***************/ /***************/
ZBTrans_MR_STAR, Z2Trans_MR_STAR );
}
PopBlocksizeStack();
}
inline void
LocalTrmmAccumulateRLN
( Orientation orientation, UnitOrNonUnit diag, T alpha,
const DistMatrix<T,MC, MR >& L,
const DistMatrix<T,STAR,MC >& X_STAR_MC,
DistMatrix<T,MR, STAR>& ZAdjOrTrans_MR_STAR )
{
#ifndef RELEASE
PushCallStack("internal::LocalTrmmAccumulateRLN");
if( L.Grid() != X_STAR_MC.Grid() ||
X_STAR_MC.Grid() != ZAdjOrTrans_MR_STAR.Grid() )
throw std::logic_error
("{L,X,Z} must be distributed over the same grid");
if( L.Height() != L.Width() ||
L.Height() != X_STAR_MC.Width() ||
L.Height() != ZAdjOrTrans_MR_STAR.Height() )
{
std::ostringstream msg;
msg << "Nonconformal LocalTrmmAccumulateRLN: \n"
<< " L ~ " << L.Height() << " x " << L.Width() << "\n"
<< " X[* ,MC] ~ " << X_STAR_MC.Height() << " x "
<< X_STAR_MC.Width() << "\n"
<< " Z^H/T[MR,* ] ~ " << ZAdjOrTrans_MR_STAR.Height() << " x "
<< ZAdjOrTrans_MR_STAR.Width() << "\n";
throw std::logic_error( msg.str().c_str() );
}
if( X_STAR_MC.RowAlignment() != L.ColAlignment() ||
ZAdjOrTrans_MR_STAR.ColAlignment() != L.RowAlignment() )
throw std::logic_error("Partial matrix distributions are misaligned");
#endif
const Grid& g = L.Grid();
// Matrix views
DistMatrix<T>
LTL(g), LTR(g), L00(g), L01(g), L02(g),
LBL(g), LBR(g), L10(g), L11(g), L12(g),
L20(g), L21(g), L22(g);
DistMatrix<T> D11(g);
DistMatrix<T,STAR,MC>
XL_STAR_MC(g), XR_STAR_MC(g),
X0_STAR_MC(g), X1_STAR_MC(g), X2_STAR_MC(g);
DistMatrix<T,MR,STAR>
ZTAdjOrTrans_MR_STAR(g), Z0AdjOrTrans_MR_STAR(g),
ZBAdjOrTrans_MR_STAR(g), Z1AdjOrTrans_MR_STAR(g),
Z2AdjOrTrans_MR_STAR(g);
const int ratio = std::max( g.Height(), g.Width() );
PushBlocksizeStack( ratio*Blocksize() );
LockedPartitionDownDiagonal
( L, LTL, LTR,
LBL, LBR, 0 );
LockedPartitionRight( X_STAR_MC, XL_STAR_MC, XR_STAR_MC, 0 );
PartitionDown
( ZAdjOrTrans_MR_STAR, ZTAdjOrTrans_MR_STAR,
ZBAdjOrTrans_MR_STAR, 0 );
while( LTL.Height() < L.Height() )
{
LockedRepartitionDownDiagonal
( LTL, /**/ LTR, L00, /**/ L01, L02,
/*************/ /******************/
/**/ L10, /**/ L11, L12,
LBL, /**/ LBR, L20, /**/ L21, L22 );
LockedRepartitionRight
( XL_STAR_MC, /**/ XR_STAR_MC,
X0_STAR_MC, /**/ X1_STAR_MC, X2_STAR_MC );
RepartitionDown
( ZTAdjOrTrans_MR_STAR, Z0AdjOrTrans_MR_STAR,
/*********************/ /*********************/
Z1AdjOrTrans_MR_STAR,
ZBAdjOrTrans_MR_STAR, Z2AdjOrTrans_MR_STAR );
D11.AlignWith( L11 );
//--------------------------------------------------------------------//
D11 = L11;
MakeTrapezoidal( LEFT, LOWER, 0, D11 );
if( diag == UNIT )
SetDiagonalToOne( D11 );
LocalGemm
( orientation, orientation,
alpha, D11, X1_STAR_MC, T(1), Z1AdjOrTrans_MR_STAR );
LocalGemm
( orientation, orientation,
alpha, L21, X2_STAR_MC, T(1), Z1AdjOrTrans_MR_STAR );
//--------------------------------------------------------------------//
D11.FreeAlignments();
SlideLockedPartitionDownDiagonal
( LTL, /**/ LTR, L00, L01, /**/ L02,
/**/ L10, L11, /**/ L12,
/*************/ /******************/
LBL, /**/ LBR, L20, L21, /**/ L22 );
SlideLockedPartitionRight
( XL_STAR_MC, /**/ XR_STAR_MC,
X0_STAR_MC, X1_STAR_MC, /**/ X2_STAR_MC );
SlidePartitionDown
( ZTAdjOrTrans_MR_STAR, Z0AdjOrTrans_MR_STAR,
Z1AdjOrTrans_MR_STAR,
/********************/ /********************/
ZBAdjOrTrans_MR_STAR, Z2AdjOrTrans_MR_STAR );
}
PopBlocksizeStack();
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
TwoSidedTrmmUVar5( UnitOrNonUnit diag, Matrix<F>& A, const Matrix<F>& U )
{
#ifndef RELEASE
CallStackEntry entry("internal::TwoSidedTrmmUVar5");
if( A.Height() != A.Width() )
LogicError("A must be square");
if( U.Height() != U.Width() )
LogicError("Triangular matrices must be square");
if( A.Height() != U.Height() )
LogicError("A and U must be the same size");
#endif
// Matrix views
Matrix<F>
ATL, ATR, A00, A01, A02,
ABL, ABR, A10, A11, A12,
A20, A21, A22;
Matrix<F>
UTL, UTR, U00, U01, U02,
UBL, UBR, U10, U11, U12,
U20, U21, U22;
// Temporary products
Matrix<F> Y01;
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
//--------------------------------------------------------------------//
// Y01 := U01 A11
Zeros( Y01, A01.Height(), A01.Width() );
Hemm( RIGHT, UPPER, F(1), A11, U01, F(0), Y01 );
// A01 := U00 A01
Trmm( LEFT, UPPER, NORMAL, diag, F(1), U00, A01 );
// A01 := A01 + 1/2 Y01
Axpy( F(1)/F(2), Y01, A01 );
// A00 := A00 + (U01 A01' + A01 U01')
Her2k( UPPER, NORMAL, F(1), U01, A01, F(1), A00 );
// A01 := A01 + 1/2 Y01
Axpy( F(1)/F(2), Y01, A01 );
// A01 := A01 U11'
Trmm( RIGHT, UPPER, ADJOINT, diag, F(1), U11, A01 );
// A11 := U11 A11 U11'
TwoSidedTrmmUUnb( diag, A11, U11 );
//--------------------------------------------------------------------//
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/*************/ /******************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
}
}
inline void
TwoSidedTrsmLVar5
( UnitOrNonUnit diag, DistMatrix<F>& A, const DistMatrix<F>& L )
{
#ifndef RELEASE
CallStackEntry entry("internal::TwoSidedTrsmLVar5");
if( A.Height() != A.Width() )
LogicError("A must be square");
if( L.Height() != L.Width() )
LogicError("Triangular matrices must be square");
if( A.Height() != L.Height() )
LogicError("A and L must be the same size");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F>
LTL(g), LTR(g), L00(g), L01(g), L02(g),
LBL(g), LBR(g), L10(g), L11(g), L12(g),
L20(g), L21(g), L22(g);
// Temporary distributions
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,MC, STAR> A21_MC_STAR(g);
DistMatrix<F,VC, STAR> A21_VC_STAR(g);
DistMatrix<F,VR, STAR> A21_VR_STAR(g);
DistMatrix<F,STAR,MR > A21Adj_STAR_MR(g);
DistMatrix<F,STAR,STAR> L11_STAR_STAR(g);
DistMatrix<F,MC, STAR> L21_MC_STAR(g);
DistMatrix<F,VC, STAR> L21_VC_STAR(g);
DistMatrix<F,VR, STAR> L21_VR_STAR(g);
DistMatrix<F,STAR,MR > L21Adj_STAR_MR(g);
DistMatrix<F,VC, STAR> Y21_VC_STAR(g);
DistMatrix<F> Y21(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( L, LTL, LTR,
LBL, LBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( LTL, /**/ LTR, L00, /**/ L01, L02,
/*************/ /******************/
/**/ L10, /**/ L11, L12,
LBL, /**/ LBR, L20, /**/ L21, L22 );
A21_MC_STAR.AlignWith( A22 );
A21_VC_STAR.AlignWith( A22 );
A21_VR_STAR.AlignWith( A22 );
A21Adj_STAR_MR.AlignWith( A22 );
L21_MC_STAR.AlignWith( A22 );
L21_VC_STAR.AlignWith( A22 );
L21_VR_STAR.AlignWith( A22 );
L21Adj_STAR_MR.AlignWith( A22 );
Y21.AlignWith( A21 );
Y21_VC_STAR.AlignWith( A22 );
//--------------------------------------------------------------------//
// A11 := inv(L11) A11 inv(L11)'
L11_STAR_STAR = L11;
A11_STAR_STAR = A11;
LocalTwoSidedTrsm( LOWER, diag, A11_STAR_STAR, L11_STAR_STAR );
A11 = A11_STAR_STAR;
// Y21 := L21 A11
L21_VC_STAR = L21;
Zeros( Y21_VC_STAR, A21.Height(), A21.Width() );
Hemm
( RIGHT, LOWER,
F(1), A11_STAR_STAR.Matrix(), L21_VC_STAR.Matrix(),
F(0), Y21_VC_STAR.Matrix() );
Y21 = Y21_VC_STAR;
// A21 := A21 inv(L11)'
A21_VC_STAR = A21;
LocalTrsm
( RIGHT, LOWER, ADJOINT, diag, F(1), L11_STAR_STAR, A21_VC_STAR );
A21 = A21_VC_STAR;
// A21 := A21 - 1/2 Y21
Axpy( F(-1)/F(2), Y21, A21 );
// A22 := A22 - (L21 A21' + A21 L21')
A21_MC_STAR = A21;
L21_MC_STAR = L21;
A21_VC_STAR = A21_MC_STAR;
A21_VR_STAR = A21_VC_STAR;
L21_VR_STAR = L21_VC_STAR;
A21Adj_STAR_MR.AdjointFrom( A21_VR_STAR );
L21Adj_STAR_MR.AdjointFrom( L21_VR_STAR );
LocalTrr2k
( LOWER,
F(-1), L21_MC_STAR, A21Adj_STAR_MR,
A21_MC_STAR, L21Adj_STAR_MR,
F(1), A22 );
// A21 := A21 - 1/2 Y21
Axpy( F(-1)/F(2), Y21, A21 );
// A21 := inv(L22) A21
//
// This is the bottleneck because A21 only has blocksize columns
Trsm( LEFT, LOWER, NORMAL, diag, F(1), L22, A21 );
//--------------------------------------------------------------------//
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( LTL, /**/ LTR, L00, L01, /**/ L02,
/**/ L10, L11, /**/ L12,
/**********************************/
LBL, /**/ LBR, L20, L21, /**/ L22 );
}
}
inline void
internal::TrsvLN
( UnitOrNonUnit diag,
const DistMatrix<F,MC,MR>& L,
DistMatrix<F,MC,MR>& x )
{
#ifndef RELEASE
PushCallStack("internal::TrsvLN");
if( L.Grid() != x.Grid() )
throw std::logic_error("{L,x} must be distributed over the same grid");
if( L.Height() != L.Width() )
throw std::logic_error("L must be square");
if( x.Width() != 1 && x.Height() != 1 )
throw std::logic_error("x must be a vector");
const int xLength = ( x.Width() == 1 ? x.Height() : x.Width() );
if( L.Width() != xLength )
throw std::logic_error("Nonconformal TrsvLN");
#endif
const Grid& g = L.Grid();
if( x.Width() == 1 )
{
// Matrix views
DistMatrix<F,MC,MR>
LTL(g), LTR(g), L00(g), L01(g), L02(g),
LBL(g), LBR(g), L10(g), L11(g), L12(g),
L20(g), L21(g), L22(g);
DistMatrix<F,MC,MR>
xT(g), x0(g),
xB(g), x1(g),
x2(g);
// Temporary distributions
DistMatrix<F,STAR,STAR> L11_STAR_STAR(g);
DistMatrix<F,STAR,STAR> x1_STAR_STAR(g);
DistMatrix<F,MR, STAR> x1_MR_STAR(g);
DistMatrix<F,MC, STAR> z2_MC_STAR(g);
// Start the algorithm
LockedPartitionDownDiagonal
( L, LTL, LTR,
LBL, LBR, 0 );
PartitionDown
( x, xT,
xB, 0 );
while( xB.Height() > 0 )
{
LockedRepartitionDownDiagonal
( LTL, /**/ LTR, L00, /**/ L01, L02,
/*************/ /******************/
/**/ L10, /**/ L11, L12,
LBL, /**/ LBR, L20, /**/ L21, L22 );
RepartitionDown
( xT, x0,
/**/ /**/
x1,
xB, x2 );
x1_MR_STAR.AlignWith( L21 );
z2_MC_STAR.AlignWith( L21 );
z2_MC_STAR.ResizeTo( x2.Height(), 1 );
//----------------------------------------------------------------//
x1_STAR_STAR = x1;
L11_STAR_STAR = L11;
Trsv
( LOWER, NORMAL, diag,
L11_STAR_STAR.LockedLocalMatrix(),
x1_STAR_STAR.LocalMatrix() );
x1 = x1_STAR_STAR;
x1_MR_STAR = x1_STAR_STAR;
Gemv
( NORMAL, (F)-1,
L21.LockedLocalMatrix(),
x1_MR_STAR.LockedLocalMatrix(),
(F)0, z2_MC_STAR.LocalMatrix() );
x2.SumScatterUpdate( (F)1, z2_MC_STAR );
//----------------------------------------------------------------//
x1_MR_STAR.FreeAlignments();
z2_MC_STAR.FreeAlignments();
SlideLockedPartitionDownDiagonal
( LTL, /**/ LTR, L00, L01, /**/ L02,
/**/ L10, L11, /**/ L12,
/*************/ /******************/
LBL, /**/ LBR, L20, L21, /**/ L22 );
SlidePartitionDown
( xT, x0,
x1,
/**/ /**/
xB, x2 );
}
}
else
{
// Matrix views
DistMatrix<F,MC,MR>
LTL(g), LTR(g), L00(g), L01(g), L02(g),
LBL(g), LBR(g), L10(g), L11(g), L12(g),
L20(g), L21(g), L22(g);
DistMatrix<F,MC,MR>
xL(g), xR(g),
x0(g), x1(g), x2(g);
// Temporary distributions
DistMatrix<F,STAR,STAR> L11_STAR_STAR(g);
DistMatrix<F,STAR,STAR> x1_STAR_STAR(g);
DistMatrix<F,STAR,MR > x1_STAR_MR(g);
DistMatrix<F,STAR,MC > z2_STAR_MC(g);
DistMatrix<F,MR, MC > z2_MR_MC(g);
DistMatrix<F,MC, MR > z2(g);
// Start the algorithm
LockedPartitionDownDiagonal
( L, LTL, LTR,
LBL, LBR, 0 );
PartitionRight( x, xL, xR, 0 );
while( xR.Width() > 0 )
{
LockedRepartitionDownDiagonal
( LTL, /**/ LTR, L00, /**/ L01, L02,
/*************/ /******************/
/**/ L10, /**/ L11, L12,
LBL, /**/ LBR, L20, /**/ L21, L22 );
RepartitionRight
( xL, /**/ xR,
x0, /**/ x1, x2 );
x1_STAR_MR.AlignWith( L21 );
z2_STAR_MC.AlignWith( L21 );
z2.AlignWith( x2 );
z2_STAR_MC.ResizeTo( 1, x2.Width() );
//----------------------------------------------------------------//
x1_STAR_STAR = x1;
L11_STAR_STAR = L11;
Trsv
( LOWER, NORMAL, diag,
L11_STAR_STAR.LockedLocalMatrix(),
x1_STAR_STAR.LocalMatrix() );
x1 = x1_STAR_STAR;
x1_STAR_MR = x1_STAR_STAR;
Gemv
( NORMAL, (F)-1,
L21.LockedLocalMatrix(),
x1_STAR_MR.LockedLocalMatrix(),
(F)0, z2_STAR_MC.LocalMatrix() );
z2_MR_MC.SumScatterFrom( z2_STAR_MC );
z2 = z2_MR_MC;
Axpy( (F)1, z2, x2 );
//----------------------------------------------------------------//
x1_STAR_MR.FreeAlignments();
z2_STAR_MC.FreeAlignments();
z2.FreeAlignments();
SlideLockedPartitionDownDiagonal
( LTL, /**/ LTR, L00, L01, /**/ L02,
/**/ L10, L11, /**/ L12,
/*************/ /******************/
LBL, /**/ LBR, L20, L21, /**/ L22 );
SlidePartitionRight
( xL, /**/ xR,
x0, x1, /**/ x2 );
}
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
LocalSymmetricAccumulateRU
( Orientation orientation, T alpha,
const DistMatrix<T,MC, MR >& A,
const DistMatrix<T,STAR,MC >& B_STAR_MC,
const DistMatrix<T,MR, STAR>& BAdjOrTrans_MR_STAR,
DistMatrix<T,MC, STAR>& ZAdjOrTrans_MC_STAR,
DistMatrix<T,MR, STAR>& ZAdjOrTrans_MR_STAR )
{
#ifndef RELEASE
PushCallStack("internal::LocalSymmetricAccumulateRU");
if( A.Grid() != B_STAR_MC.Grid() ||
B_STAR_MC.Grid() != BAdjOrTrans_MR_STAR.Grid() ||
BAdjOrTrans_MR_STAR.Grid() != ZAdjOrTrans_MC_STAR.Grid() ||
ZAdjOrTrans_MC_STAR.Grid() != ZAdjOrTrans_MR_STAR.Grid() )
throw std::logic_error
("{A,B,C} must be distributed over the same grid");
if( A.Height() != A.Width() ||
A.Height() != B_STAR_MC.Width() ||
A.Height() != BAdjOrTrans_MR_STAR.Height() ||
A.Height() != ZAdjOrTrans_MC_STAR.Height() ||
A.Height() != ZAdjOrTrans_MR_STAR.Height() ||
B_STAR_MC.Height() != BAdjOrTrans_MR_STAR.Width() ||
BAdjOrTrans_MR_STAR.Width() != ZAdjOrTrans_MC_STAR.Width() ||
ZAdjOrTrans_MC_STAR.Width() != ZAdjOrTrans_MR_STAR.Width() )
{
std::ostringstream msg;
msg << "Nonconformal LocalSymmetricAccumulateRU: \n"
<< " A ~ " << A.Height() << " x " << A.Width() << "\n"
<< " B[* ,MC] ~ " << B_STAR_MC.Height() << " x "
<< B_STAR_MC.Width() << "\n"
<< " B^H/T[MR,* ] ~ " << BAdjOrTrans_MR_STAR.Height() << " x "
<< BAdjOrTrans_MR_STAR.Width() << "\n"
<< " Z^H/T[MC,* ] ~ " << ZAdjOrTrans_MC_STAR.Height() << " x "
<< ZAdjOrTrans_MC_STAR.Width() << "\n"
<< " Z^H/T[MR,* ] ~ " << ZAdjOrTrans_MR_STAR.Height() << " x "
<< ZAdjOrTrans_MR_STAR.Width() << "\n";
throw std::logic_error( msg.str().c_str() );
}
if( B_STAR_MC.RowAlignment() != A.ColAlignment() ||
BAdjOrTrans_MR_STAR.ColAlignment() != A.RowAlignment() ||
ZAdjOrTrans_MC_STAR.ColAlignment() != A.ColAlignment() ||
ZAdjOrTrans_MR_STAR.ColAlignment() != A.RowAlignment() )
throw std::logic_error("Partial matrix distributions are misaligned");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<T>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<T> D11(g);
DistMatrix<T,STAR,MC>
BL_STAR_MC(g), BR_STAR_MC(g),
B0_STAR_MC(g), B1_STAR_MC(g), B2_STAR_MC(g);
DistMatrix<T,MR,STAR>
BTAdjOrTrans_MR_STAR(g), B0AdjOrTrans_MR_STAR(g),
BBAdjOrTrans_MR_STAR(g), B1AdjOrTrans_MR_STAR(g),
B2AdjOrTrans_MR_STAR(g);
DistMatrix<T,MC,STAR>
ZTAdjOrTrans_MC_STAR(g), Z0AdjOrTrans_MC_STAR(g),
ZBAdjOrTrans_MC_STAR(g), Z1AdjOrTrans_MC_STAR(g),
Z2AdjOrTrans_MC_STAR(g);
DistMatrix<T,MR,STAR>
ZBAdjOrTrans_MR_STAR(g), Z0AdjOrTrans_MR_STAR(g),
ZTAdjOrTrans_MR_STAR(g), Z1AdjOrTrans_MR_STAR(g),
Z2AdjOrTrans_MR_STAR(g);
const int ratio = std::max( g.Height(), g.Width() );
PushBlocksizeStack( ratio*Blocksize() );
LockedPartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionRight( B_STAR_MC, BL_STAR_MC, BR_STAR_MC, 0 );
LockedPartitionDown
( BAdjOrTrans_MR_STAR, BTAdjOrTrans_MR_STAR,
BBAdjOrTrans_MR_STAR, 0 );
PartitionDown
( ZAdjOrTrans_MC_STAR, ZTAdjOrTrans_MC_STAR,
ZBAdjOrTrans_MC_STAR, 0 );
PartitionDown
( ZAdjOrTrans_MR_STAR, ZTAdjOrTrans_MR_STAR,
ZBAdjOrTrans_MR_STAR, 0 );
while( ATL.Height() < A.Height() )
{
LockedRepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionRight
( BL_STAR_MC, /**/ BR_STAR_MC,
B0_STAR_MC, /**/ B1_STAR_MC, B2_STAR_MC );
LockedRepartitionDown
( BTAdjOrTrans_MR_STAR, B0AdjOrTrans_MR_STAR,
/********************/ /********************/
B1AdjOrTrans_MR_STAR,
BBAdjOrTrans_MR_STAR, B2AdjOrTrans_MR_STAR );
RepartitionDown
( ZTAdjOrTrans_MC_STAR, Z0AdjOrTrans_MC_STAR,
/********************/ /********************/
Z1AdjOrTrans_MC_STAR,
ZBAdjOrTrans_MC_STAR, Z2AdjOrTrans_MC_STAR );
RepartitionDown
( ZTAdjOrTrans_MR_STAR, Z0AdjOrTrans_MR_STAR,
/********************/ /********************/
Z1AdjOrTrans_MR_STAR,
ZBAdjOrTrans_MR_STAR, Z2AdjOrTrans_MR_STAR );
D11.AlignWith( A11 );
//--------------------------------------------------------------------//
D11 = A11;
MakeTrapezoidal( LEFT, UPPER, 0, D11 );
LocalGemm
( orientation, orientation,
alpha, D11, B1_STAR_MC, T(1), Z1AdjOrTrans_MR_STAR );
MakeTrapezoidal( LEFT, UPPER, 1, D11 );
LocalGemm
( NORMAL, NORMAL, alpha, D11, B1AdjOrTrans_MR_STAR,
T(1), Z1AdjOrTrans_MC_STAR );
LocalGemm
( orientation, orientation,
alpha, A12, B1_STAR_MC, T(1), Z2AdjOrTrans_MR_STAR );
LocalGemm
( NORMAL, NORMAL, alpha, A12, B2AdjOrTrans_MR_STAR,
T(1), Z1AdjOrTrans_MC_STAR );
//--------------------------------------------------------------------//
D11.FreeAlignments();
SlideLockedPartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionRight
( BL_STAR_MC, /**/ BR_STAR_MC,
B0_STAR_MC, B1_STAR_MC, /**/ B2_STAR_MC );
SlideLockedPartitionDown
( BTAdjOrTrans_MR_STAR, B0AdjOrTrans_MR_STAR,
B1AdjOrTrans_MR_STAR,
/********************/ /********************/
BBAdjOrTrans_MR_STAR, B2AdjOrTrans_MR_STAR );
SlidePartitionDown
( ZTAdjOrTrans_MC_STAR, Z0AdjOrTrans_MC_STAR,
Z1AdjOrTrans_MC_STAR,
/********************/ /********************/
ZBAdjOrTrans_MC_STAR, Z2AdjOrTrans_MC_STAR );
SlidePartitionDown
( ZTAdjOrTrans_MR_STAR, Z0AdjOrTrans_MR_STAR,
Z1AdjOrTrans_MR_STAR,
/********************/ /********************/
ZBAdjOrTrans_MR_STAR, Z2AdjOrTrans_MR_STAR );
}
PopBlocksizeStack();
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
internal::TrmmLUNC
( UnitOrNonUnit diag,
T alpha, const DistMatrix<T,MC,MR>& U,
DistMatrix<T,MC,MR>& X )
{
#ifndef RELEASE
PushCallStack("internal::TrmmLUNC");
if( U.Grid() != X.Grid() )
throw std::logic_error
("U and X must be distributed over the same grid");
if( U.Height() != U.Width() || U.Width() != X.Height() )
{
std::ostringstream msg;
msg << "Nonconformal TrmmLUN: \n"
<< " U ~ " << U.Height() << " x " << U.Width() << "\n"
<< " X ~ " << X.Height() << " x " << X.Width() << "\n";
throw std::logic_error( msg.str().c_str() );
}
#endif
const Grid& g = U.Grid();
// Matrix views
DistMatrix<T,MC,MR>
UTL(g), UTR(g), U00(g), U01(g), U02(g),
UBL(g), UBR(g), U10(g), U11(g), U12(g),
U20(g), U21(g), U22(g);
DistMatrix<T,MC,MR> XT(g), X0(g),
XB(g), X1(g),
X2(g);
// Temporary distributions
DistMatrix<T,STAR,STAR> U11_STAR_STAR(g);
DistMatrix<T,STAR,MC > U12_STAR_MC(g);
DistMatrix<T,STAR,VR > X1_STAR_VR(g);
DistMatrix<T,MR, STAR> D1Trans_MR_STAR(g);
DistMatrix<T,MR, MC > D1Trans_MR_MC(g);
DistMatrix<T,MC, MR > D1(g);
// Start the algorithm
Scal( alpha, X );
LockedPartitionDownDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
PartitionDown
( X, XT,
XB, 0 );
while( XB.Height() > 0 )
{
LockedRepartitionDownDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
RepartitionDown
( XT, X0,
/**/ /**/
X1,
XB, X2 );
U12_STAR_MC.AlignWith( X2 );
D1Trans_MR_STAR.AlignWith( X1 );
D1Trans_MR_MC.AlignWith( X1 );
D1.AlignWith( X1 );
D1Trans_MR_STAR.ResizeTo( X1.Width(), X1.Height() );
D1.ResizeTo( X1.Height(), X1.Width() );
//--------------------------------------------------------------------//
X1_STAR_VR = X1;
U11_STAR_STAR = U11;
internal::LocalTrmm
( LEFT, UPPER, NORMAL, diag, (T)1, U11_STAR_STAR, X1_STAR_VR );
X1 = X1_STAR_VR;
U12_STAR_MC = U12;
internal::LocalGemm
( TRANSPOSE, TRANSPOSE, (T)1, X2, U12_STAR_MC, (T)0, D1Trans_MR_STAR );
D1Trans_MR_MC.SumScatterFrom( D1Trans_MR_STAR );
Transpose( D1Trans_MR_MC.LocalMatrix(), D1.LocalMatrix() );
Axpy( (T)1, D1, X1 );
//--------------------------------------------------------------------//
D1.FreeAlignments();
D1Trans_MR_MC.FreeAlignments();
D1Trans_MR_STAR.FreeAlignments();
U12_STAR_MC.FreeAlignments();
SlideLockedPartitionDownDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/*************/ /******************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
SlidePartitionDown
( XT, X0,
X1,
/**/ /**/
XB, X2 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
internal::LocalTrmmAccumulateLUN
( Orientation orientation, UnitOrNonUnit diag, T alpha,
const DistMatrix<T,MC, MR >& U,
const DistMatrix<T,STAR,MR >& XAdjOrTrans_STAR_MR,
DistMatrix<T,MC, STAR>& Z_MC_STAR )
{
#ifndef RELEASE
PushCallStack("internal::LocalTrmmAccumulateLUN");
if( U.Grid() != XAdjOrTrans_STAR_MR.Grid() ||
XAdjOrTrans_STAR_MR.Grid() != Z_MC_STAR.Grid() )
throw std::logic_error
("{U,X,Z} must be distributed over the same grid");
if( U.Height() != U.Width() ||
U.Height() != XAdjOrTrans_STAR_MR.Width() ||
U.Height() != Z_MC_STAR.Height() ||
XAdjOrTrans_STAR_MR.Height() != Z_MC_STAR.Width() )
{
std::ostringstream msg;
msg << "Nonconformal LocalTrmmAccumulateLUN: \n"
<< " U ~ " << U.Height() << " x " << U.Width() << "\n"
<< " X^H/T[* ,MR] ~ " << XAdjOrTrans_STAR_MR.Height() << " x "
<< XAdjOrTrans_STAR_MR.Width() << "\n"
<< " Z[MC,* ] ~ " << Z_MC_STAR.Height() << " x "
<< Z_MC_STAR.Width() << "\n";
throw std::logic_error( msg.str().c_str() );
}
if( XAdjOrTrans_STAR_MR.RowAlignment() != U.RowAlignment() ||
Z_MC_STAR.ColAlignment() != U.ColAlignment() )
throw std::logic_error("Partial matrix distributions are misaligned");
#endif
const Grid& g = U.Grid();
// Matrix views
DistMatrix<T,MC,MR>
UTL(g), UTR(g), U00(g), U01(g), U02(g),
UBL(g), UBR(g), U10(g), U11(g), U12(g),
U20(g), U21(g), U22(g);
DistMatrix<T,MC,MR> D11(g);
DistMatrix<T,STAR,MR>
XLAdjOrTrans_STAR_MR(g), XRAdjOrTrans_STAR_MR(g),
X0AdjOrTrans_STAR_MR(g), X1AdjOrTrans_STAR_MR(g),
X2AdjOrTrans_STAR_MR(g);
DistMatrix<T,MC,STAR>
ZT_MC_STAR(g), Z0_MC_STAR(g),
ZB_MC_STAR(g), Z1_MC_STAR(g),
Z2_MC_STAR(g);
const int ratio = std::max( g.Height(), g.Width() );
PushBlocksizeStack( ratio*Blocksize() );
LockedPartitionDownDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
LockedPartitionRight
( XAdjOrTrans_STAR_MR, XLAdjOrTrans_STAR_MR, XRAdjOrTrans_STAR_MR, 0 );
PartitionDown
( Z_MC_STAR, ZT_MC_STAR,
ZB_MC_STAR, 0 );
while( UTL.Height() < U.Height() )
{
LockedRepartitionDownDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
LockedRepartitionRight
( XLAdjOrTrans_STAR_MR, /**/ XRAdjOrTrans_STAR_MR,
X0AdjOrTrans_STAR_MR, /**/ X1AdjOrTrans_STAR_MR, X2AdjOrTrans_STAR_MR
);
RepartitionDown
( ZT_MC_STAR, Z0_MC_STAR,
/**********/ /**********/
Z1_MC_STAR,
ZB_MC_STAR, Z2_MC_STAR );
D11.AlignWith( U11 );
//--------------------------------------------------------------------//
D11 = U11;
MakeTrapezoidal( LEFT, UPPER, 0, D11 );
if( diag == UNIT )
SetDiagonalToOne( D11 );
internal::LocalGemm
( NORMAL, orientation, alpha, D11, X1AdjOrTrans_STAR_MR,
(T)1, Z1_MC_STAR );
internal::LocalGemm
( NORMAL, orientation, alpha, U01, X1AdjOrTrans_STAR_MR,
(T)1, Z0_MC_STAR );
//--------------------------------------------------------------------//
D11.FreeAlignments();
SlideLockedPartitionDownDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/*************/ /******************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
SlideLockedPartitionRight
( XLAdjOrTrans_STAR_MR, /**/ XRAdjOrTrans_STAR_MR,
X0AdjOrTrans_STAR_MR, X1AdjOrTrans_STAR_MR, /**/ X2AdjOrTrans_STAR_MR
);
SlidePartitionDown
( ZT_MC_STAR, Z0_MC_STAR,
Z1_MC_STAR,
/**********/ /**********/
ZB_MC_STAR, Z2_MC_STAR );
}
PopBlocksizeStack();
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
TwoSidedTrmmLVar2( UnitOrNonUnit diag, Matrix<F>& A, const Matrix<F>& L )
{
#ifndef RELEASE
PushCallStack("internal::TwoSidedTrmmLVar2");
if( A.Height() != A.Width() )
throw std::logic_error( "A must be square." );
if( L.Height() != L.Width() )
throw std::logic_error( "Triangular matrices must be square." );
if( A.Height() != L.Height() )
throw std::logic_error( "A and L must be the same size." );
#endif
// Matrix views
Matrix<F>
ATL, ATR, A00, A01, A02,
ABL, ABR, A10, A11, A12,
A20, A21, A22;
Matrix<F>
LTL, LTR, L00, L01, L02,
LBL, LBR, L10, L11, L12,
L20, L21, L22;
// Temporary products
Matrix<F> Y21;
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( L, LTL, LTR,
LBL, LBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( LTL, /**/ LTR, L00, /**/ L01, L02,
/*************/ /******************/
/**/ L10, /**/ L11, L12,
LBL, /**/ LBR, L20, /**/ L21, L22 );
//--------------------------------------------------------------------//
// A10 := L11' A10
Trmm( LEFT, LOWER, ADJOINT, diag, F(1), L11, A10 );
// A10 := A10 + L21' A20
Gemm( ADJOINT, NORMAL, F(1), L21, A20, F(1), A10 );
// Y21 := A22 L21
Zeros( A21.Height(), A21.Width(), Y21 );
Hemm( LEFT, LOWER, F(1), A22, L21, F(0), Y21 );
// A21 := A21 L11
Trmm( RIGHT, LOWER, NORMAL, diag, F(1), L11, A21 );
// A21 := A21 + 1/2 Y21
Axpy( F(1)/F(2), Y21, A21 );
// A11 := L11' A11 L11
TwoSidedTrmmLUnb( diag, A11, L11 );
// A11 := A11 + (A21' L21 + L21' A21)
Her2k( LOWER, ADJOINT, F(1), A21, L21, F(1), A11 );
// A21 := A21 + 1/2 Y21
Axpy( F(1)/F(2), Y21, A21 );
//--------------------------------------------------------------------//
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( LTL, /**/ LTR, L00, L01, /**/ L02,
/**/ L10, L11, /**/ L12,
/*************/ /******************/
LBL, /**/ LBR, L20, L21, /**/ L22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
TwoSidedTrsmUVar2
( UnitOrNonUnit diag, DistMatrix<F>& A, const DistMatrix<F>& U )
{
#ifndef RELEASE
CallStackEntry entry("internal::TwoSidedTrsmUVar2");
if( A.Height() != A.Width() )
LogicError("A must be square");
if( U.Height() != U.Width() )
LogicError("Triangular matrices must be square");
if( A.Height() != U.Height() )
LogicError("A and U must be the same size");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F>
UTL(g), UTR(g), U00(g), U01(g), U02(g),
UBL(g), UBR(g), U10(g), U11(g), U12(g),
U20(g), U21(g), U22(g);
// Temporary distributions
DistMatrix<F,MC, STAR> A01_MC_STAR(g);
DistMatrix<F,VC, STAR> A01_VC_STAR(g);
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,STAR,VR > A12_STAR_VR(g);
DistMatrix<F,MC, STAR> F01_MC_STAR(g);
DistMatrix<F,MC, STAR> U01_MC_STAR(g);
DistMatrix<F,VR, STAR> U01_VR_STAR(g);
DistMatrix<F,STAR,MR > U01Adj_STAR_MR(g);
DistMatrix<F,STAR,STAR> U11_STAR_STAR(g);
DistMatrix<F,STAR,MR > X11_STAR_MR(g);
DistMatrix<F,MR, STAR> X12Adj_MR_STAR(g);
DistMatrix<F,MR, MC > X12Adj_MR_MC(g);
DistMatrix<F,MR, MC > Y01_MR_MC(g);
DistMatrix<F,MR, STAR> Y01_MR_STAR(g);
DistMatrix<F> X11(g);
DistMatrix<F> Y01(g);
Matrix<F> X12Local;
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
A01_MC_STAR.AlignWith( U01 );
Y01.AlignWith( A01 );
Y01_MR_STAR.AlignWith( A00 );
U01_MC_STAR.AlignWith( A00 );
U01_VR_STAR.AlignWith( A00 );
U01Adj_STAR_MR.AlignWith( A00 );
X11_STAR_MR.AlignWith( U01 );
X11.AlignWith( A11 );
X12Adj_MR_STAR.AlignWith( A02 );
X12Adj_MR_MC.AlignWith( A12 );
F01_MC_STAR.AlignWith( A00 );
//--------------------------------------------------------------------//
// Y01 := A00 U01
U01_MC_STAR = U01;
U01_VR_STAR = U01_MC_STAR;
U01Adj_STAR_MR.AdjointFrom( U01_VR_STAR );
Zeros( Y01_MR_STAR, A01.Height(), A01.Width() );
Zeros( F01_MC_STAR, A01.Height(), A01.Width() );
LocalSymmetricAccumulateLU
( ADJOINT,
F(1), A00, U01_MC_STAR, U01Adj_STAR_MR, F01_MC_STAR, Y01_MR_STAR );
Y01_MR_MC.SumScatterFrom( Y01_MR_STAR );
Y01 = Y01_MR_MC;
Y01.SumScatterUpdate( F(1), F01_MC_STAR );
// X11 := U01' A01
LocalGemm( ADJOINT, NORMAL, F(1), U01_MC_STAR, A01, X11_STAR_MR );
// A01 := A01 - Y01
Axpy( F(-1), Y01, A01 );
A01_MC_STAR = A01;
// A11 := A11 - triu(X11 + A01' U01) = A11 - (U01 A01 + A01' U01)
LocalGemm( ADJOINT, NORMAL, F(1), A01_MC_STAR, U01, F(1), X11_STAR_MR );
X11.SumScatterFrom( X11_STAR_MR );
MakeTriangular( UPPER, X11 );
Axpy( F(-1), X11, A11 );
// A01 := A01 inv(U11)
U11_STAR_STAR = U11;
A01_VC_STAR = A01_MC_STAR;
LocalTrsm
( RIGHT, UPPER, NORMAL, diag, F(1), U11_STAR_STAR, A01_VC_STAR );
A01 = A01_VC_STAR;
// A11 := inv(U11)' A11 inv(U11)
A11_STAR_STAR = A11;
LocalTwoSidedTrsm( UPPER, diag, A11_STAR_STAR, U11_STAR_STAR );
A11 = A11_STAR_STAR;
// A12 := A12 - A02' U01
LocalGemm( ADJOINT, NORMAL, F(1), A02, U01_MC_STAR, X12Adj_MR_STAR );
X12Adj_MR_MC.SumScatterFrom( X12Adj_MR_STAR );
Adjoint( X12Adj_MR_MC.LockedMatrix(), X12Local );
Axpy( F(-1), X12Local, A12.Matrix() );
// A12 := inv(U11)' A12
A12_STAR_VR = A12;
LocalTrsm
( LEFT, UPPER, ADJOINT, diag, F(1), U11_STAR_STAR, A12_STAR_VR );
A12 = A12_STAR_VR;
//--------------------------------------------------------------------//
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/*************/ /******************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
}
}
inline void
TwoSidedTrmmLVar2
( UnitOrNonUnit diag, DistMatrix<F>& A, const DistMatrix<F>& L )
{
#ifndef RELEASE
PushCallStack("internal::TwoSidedTrmmLVar2");
if( A.Height() != A.Width() )
throw std::logic_error( "A must be square." );
if( L.Height() != L.Width() )
throw std::logic_error( "Triangular matrices must be square." );
if( A.Height() != L.Height() )
throw std::logic_error( "A and L must be the same size." );
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F>
LTL(g), LTR(g), L00(g), L01(g), L02(g),
LBL(g), LBR(g), L10(g), L11(g), L12(g),
L20(g), L21(g), L22(g);
// Temporary distributions
DistMatrix<F,STAR,VR > A10_STAR_VR(g);
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,VC, STAR> A21_VC_STAR(g);
DistMatrix<F,STAR,STAR> L11_STAR_STAR(g);
DistMatrix<F,MC, STAR> L21_MC_STAR(g);
DistMatrix<F,STAR,MR > L21Adj_STAR_MR(g);
DistMatrix<F,VC, STAR> L21_VC_STAR(g);
DistMatrix<F,VR, STAR> L21_VR_STAR(g);
DistMatrix<F,STAR,MR > X10_STAR_MR(g);
DistMatrix<F,STAR,STAR> X11_STAR_STAR(g);
DistMatrix<F,MC, STAR> Z21_MC_STAR(g);
DistMatrix<F,MR, STAR> Z21_MR_STAR(g);
DistMatrix<F,MR, MC > Z21_MR_MC(g);
DistMatrix<F> Y21(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( L, LTL, LTR,
LBL, LBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( LTL, /**/ LTR, L00, /**/ L01, L02,
/*************/ /******************/
/**/ L10, /**/ L11, L12,
LBL, /**/ LBR, L20, /**/ L21, L22 );
A21_VC_STAR.AlignWith( A22 );
L21_MC_STAR.AlignWith( A20 );
L21_VC_STAR.AlignWith( A22 );
L21_VR_STAR.AlignWith( A22 );
L21Adj_STAR_MR.AlignWith( A22 );
X10_STAR_MR.AlignWith( A10 );
Y21.AlignWith( A21 );
Z21_MC_STAR.AlignWith( A22 );
Z21_MR_STAR.AlignWith( A22 );
//--------------------------------------------------------------------//
// A10 := L11' A10
L11_STAR_STAR = L11;
A10_STAR_VR = A10;
LocalTrmm
( LEFT, LOWER, ADJOINT, diag, F(1), L11_STAR_STAR, A10_STAR_VR );
A10 = A10_STAR_VR;
// A10 := A10 + L21' A20
L21_MC_STAR = L21;
X10_STAR_MR.ResizeTo( A10.Height(), A10.Width() );
LocalGemm( ADJOINT, NORMAL, F(1), L21_MC_STAR, A20, F(0), X10_STAR_MR );
A10.SumScatterUpdate( F(1), X10_STAR_MR );
// Y21 := A22 L21
L21_VC_STAR = L21_MC_STAR;
L21_VR_STAR = L21_VC_STAR;
L21Adj_STAR_MR.AdjointFrom( L21_VR_STAR );
Z21_MC_STAR.ResizeTo( A21.Height(), A21.Width() );
Z21_MR_STAR.ResizeTo( A21.Height(), A21.Width() );
Zero( Z21_MC_STAR );
Zero( Z21_MR_STAR );
LocalSymmetricAccumulateLL
( ADJOINT,
F(1), A22, L21_MC_STAR, L21Adj_STAR_MR, Z21_MC_STAR, Z21_MR_STAR );
Z21_MR_MC.SumScatterFrom( Z21_MR_STAR );
Y21 = Z21_MR_MC;
Y21.SumScatterUpdate( F(1), Z21_MC_STAR );
// A21 := A21 L11
A21_VC_STAR = A21;
LocalTrmm
( RIGHT, LOWER, NORMAL, diag, F(1), L11_STAR_STAR, A21_VC_STAR );
A21 = A21_VC_STAR;
// A21 := A21 + 1/2 Y21
Axpy( F(1)/F(2), Y21, A21 );
// A11 := L11' A11 L11
A11_STAR_STAR = A11;
LocalTwoSidedTrmm( LOWER, diag, A11_STAR_STAR, L11_STAR_STAR );
A11 = A11_STAR_STAR;
// A11 := A11 + (A21' L21 + L21' A21)
A21_VC_STAR = A21;
X11_STAR_STAR.ResizeTo( A11.Height(), A11.Width() );
Her2k
( LOWER, ADJOINT,
F(1), A21_VC_STAR.LocalMatrix(), L21_VC_STAR.LocalMatrix(),
F(0), X11_STAR_STAR.LocalMatrix() );
A11.SumScatterUpdate( F(1), X11_STAR_STAR );
// A21 := A21 + 1/2 Y21
Axpy( F(1)/F(2), Y21, A21 );
//--------------------------------------------------------------------//
A21_VC_STAR.FreeAlignments();
L21_MC_STAR.FreeAlignments();
L21_VC_STAR.FreeAlignments();
L21_VR_STAR.FreeAlignments();
L21Adj_STAR_MR.FreeAlignments();
X10_STAR_MR.FreeAlignments();
Y21.FreeAlignments();
Z21_MC_STAR.FreeAlignments();
Z21_MR_STAR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( LTL, /**/ LTR, L00, L01, /**/ L02,
/**/ L10, L11, /**/ L12,
/*************/ /******************/
LBL, /**/ LBR, L20, L21, /**/ L22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
TwoSidedTrsmLVar2
( UnitOrNonUnit diag, DistMatrix<F>& A, const DistMatrix<F>& L )
{
#ifndef RELEASE
CallStackEntry entry("internal::TwoSidedTrsmLVar2");
if( A.Height() != A.Width() )
LogicError("A must be square");
if( L.Height() != L.Width() )
LogicError("Triangular matrices must be square");
if( A.Height() != L.Height() )
LogicError("A and L must be the same size");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F>
LTL(g), LTR(g), L00(g), L01(g), L02(g),
LBL(g), LBR(g), L10(g), L11(g), L12(g),
L20(g), L21(g), L22(g);
// Temporary distributions
DistMatrix<F,MR, STAR> A10Adj_MR_STAR(g);
DistMatrix<F,STAR,VR > A10_STAR_VR(g);
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,VC, STAR> A21_VC_STAR(g);
DistMatrix<F,MR, STAR> F10Adj_MR_STAR(g);
DistMatrix<F,MR, STAR> L10Adj_MR_STAR(g);
DistMatrix<F,VC, STAR> L10Adj_VC_STAR(g);
DistMatrix<F,STAR,MC > L10_STAR_MC(g);
DistMatrix<F,STAR,STAR> L11_STAR_STAR(g);
DistMatrix<F,MC, STAR> X11_MC_STAR(g);
DistMatrix<F,MC, STAR> X21_MC_STAR(g);
DistMatrix<F,MC, STAR> Y10Adj_MC_STAR(g);
DistMatrix<F,MR, MC > Y10Adj_MR_MC(g);
DistMatrix<F> X11(g);
DistMatrix<F> Y10Adj(g);
Matrix<F> Y10Local;
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( L, LTL, LTR,
LBL, LBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( LTL, /**/ LTR, L00, /**/ L01, L02,
/*************/ /******************/
/**/ L10, /**/ L11, L12,
LBL, /**/ LBR, L20, /**/ L21, L22 );
A10Adj_MR_STAR.AlignWith( L10 );
F10Adj_MR_STAR.AlignWith( A00 );
L10Adj_MR_STAR.AlignWith( A00 );
L10Adj_VC_STAR.AlignWith( A00 );
L10_STAR_MC.AlignWith( A00 );
X11.AlignWith( A11 );
X11_MC_STAR.AlignWith( L10 );
X21_MC_STAR.AlignWith( A20 );
Y10Adj_MC_STAR.AlignWith( A00 );
Y10Adj_MR_MC.AlignWith( A10 );
//--------------------------------------------------------------------//
// Y10 := L10 A00
L10Adj_MR_STAR.AdjointFrom( L10 );
L10Adj_VC_STAR = L10Adj_MR_STAR;
L10_STAR_MC.AdjointFrom( L10Adj_VC_STAR );
Zeros( Y10Adj_MC_STAR, A10.Width(), A10.Height() );
Zeros( F10Adj_MR_STAR, A10.Width(), A10.Height() );
LocalSymmetricAccumulateRL
( ADJOINT,
F(1), A00, L10_STAR_MC, L10Adj_MR_STAR,
Y10Adj_MC_STAR, F10Adj_MR_STAR );
Y10Adj.SumScatterFrom( Y10Adj_MC_STAR );
Y10Adj_MR_MC = Y10Adj;
Y10Adj_MR_MC.SumScatterUpdate( F(1), F10Adj_MR_STAR );
Adjoint( Y10Adj_MR_MC.LockedMatrix(), Y10Local );
// X11 := A10 L10'
LocalGemm( NORMAL, NORMAL, F(1), A10, L10Adj_MR_STAR, X11_MC_STAR );
// A10 := A10 - Y10
Axpy( F(-1), Y10Local, A10.Matrix() );
A10Adj_MR_STAR.AdjointFrom( A10 );
// A11 := A11 - (X11 + L10 A10') = A11 - (A10 L10' + L10 A10')
LocalGemm
( NORMAL, NORMAL, F(1), L10, A10Adj_MR_STAR, F(1), X11_MC_STAR );
X11.SumScatterFrom( X11_MC_STAR );
MakeTriangular( LOWER, X11 );
Axpy( F(-1), X11, A11 );
// A10 := inv(L11) A10
L11_STAR_STAR = L11;
A10_STAR_VR.AdjointFrom( A10Adj_MR_STAR );
LocalTrsm
( LEFT, LOWER, NORMAL, diag, F(1), L11_STAR_STAR, A10_STAR_VR );
A10 = A10_STAR_VR;
// A11 := inv(L11) A11 inv(L11)'
A11_STAR_STAR = A11;
LocalTwoSidedTrsm( LOWER, diag, A11_STAR_STAR, L11_STAR_STAR );
A11 = A11_STAR_STAR;
// A21 := A21 - A20 L10'
LocalGemm( NORMAL, NORMAL, F(1), A20, L10Adj_MR_STAR, X21_MC_STAR );
A21.SumScatterUpdate( F(-1), X21_MC_STAR );
// A21 := A21 inv(L11)'
A21_VC_STAR = A21;
LocalTrsm
( RIGHT, LOWER, ADJOINT, diag, F(1), L11_STAR_STAR, A21_VC_STAR );
A21 = A21_VC_STAR;
//--------------------------------------------------------------------//
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( LTL, /**/ LTR, L00, L01, /**/ L02,
/**/ L10, L11, /**/ L12,
/**********************************/
LBL, /**/ LBR, L20, L21, /**/ L22 );
}
}
inline void
TwoSidedTrsmUVar5
( UnitOrNonUnit diag, DistMatrix<F>& A, const DistMatrix<F>& U )
{
#ifndef RELEASE
CallStackEntry entry("internal::TwoSidedTrsmUVar5");
if( A.Height() != A.Width() )
LogicError("A must be square");
if( U.Height() != U.Width() )
LogicError("Triangular matrices must be square");
if( A.Height() != U.Height() )
LogicError("A and U must be the same size");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F>
UTL(g), UTR(g), U00(g), U01(g), U02(g),
UBL(g), UBR(g), U10(g), U11(g), U12(g),
U20(g), U21(g), U22(g);
// Temporary distributions
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,STAR,MC > A12_STAR_MC(g);
DistMatrix<F,STAR,MR > A12_STAR_MR(g);
DistMatrix<F,STAR,VC > A12_STAR_VC(g);
DistMatrix<F,STAR,VR > A12_STAR_VR(g);
DistMatrix<F,STAR,STAR> U11_STAR_STAR(g);
DistMatrix<F,STAR,MC > U12_STAR_MC(g);
DistMatrix<F,STAR,MR > U12_STAR_MR(g);
DistMatrix<F,STAR,VC > U12_STAR_VC(g);
DistMatrix<F,STAR,VR > U12_STAR_VR(g);
DistMatrix<F,STAR,VR > Y12_STAR_VR(g);
DistMatrix<F> Y12(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
A12_STAR_MC.AlignWith( A22 );
A12_STAR_MR.AlignWith( A22 );
A12_STAR_VC.AlignWith( A22 );
A12_STAR_VR.AlignWith( A22 );
U12_STAR_MC.AlignWith( A22 );
U12_STAR_MR.AlignWith( A22 );
U12_STAR_VC.AlignWith( A22 );
U12_STAR_VR.AlignWith( A22 );
Y12.AlignWith( A12 );
Y12_STAR_VR.AlignWith( A12 );
//--------------------------------------------------------------------//
// A11 := inv(U11)' A11 inv(U11)
U11_STAR_STAR = U11;
A11_STAR_STAR = A11;
LocalTwoSidedTrsm( UPPER, diag, A11_STAR_STAR, U11_STAR_STAR );
A11 = A11_STAR_STAR;
// Y12 := A11 U12
U12_STAR_VR = U12;
Zeros( Y12_STAR_VR, A12.Height(), A12.Width() );
Hemm
( LEFT, UPPER,
F(1), A11_STAR_STAR.Matrix(), U12_STAR_VR.Matrix(),
F(0), Y12_STAR_VR.Matrix() );
Y12 = Y12_STAR_VR;
// A12 := inv(U11)' A12
A12_STAR_VR = A12;
LocalTrsm
( LEFT, UPPER, ADJOINT, diag, F(1), U11_STAR_STAR, A12_STAR_VR );
A12 = A12_STAR_VR;
// A12 := A12 - 1/2 Y12
Axpy( F(-1)/F(2), Y12, A12 );
// A22 := A22 - (A12' U12 + U12' A12)
A12_STAR_VR = A12;
A12_STAR_VC = A12_STAR_VR;
U12_STAR_VC = U12_STAR_VR;
A12_STAR_MC = A12_STAR_VC;
U12_STAR_MC = U12_STAR_VC;
A12_STAR_MR = A12_STAR_VR;
U12_STAR_MR = U12_STAR_VR;
LocalTrr2k
( UPPER, ADJOINT, ADJOINT,
F(-1), U12_STAR_MC, A12_STAR_MR,
A12_STAR_MC, U12_STAR_MR,
F(1), A22 );
// A12 := A12 - 1/2 Y12
Axpy( F(-1)/F(2), Y12, A12 );
// A12 := A12 inv(U22)
//
// This is the bottleneck because A12 only has blocksize rows
Trsm( RIGHT, UPPER, NORMAL, diag, F(1), U22, A12 );
//--------------------------------------------------------------------//
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/*************/ /******************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
}
}
inline void
TrtrsmLLN
( UnitOrNonUnit diag, F alpha, const DistMatrix<F>& L, DistMatrix<F>& X,
bool checkIfSingular )
{
#ifndef RELEASE
CallStackEntry entry("internal::TrtrsmLLN");
#endif
const Grid& g = L.Grid();
// Matrix views
DistMatrix<F>
LTL(g), LTR(g), L00(g), L01(g), L02(g),
LBL(g), LBR(g), L10(g), L11(g), L12(g),
L20(g), L21(g), L22(g);
DistMatrix<F>
XTL(g), XTR(g), X00(g), X01(g), X02(g),
XBL(g), XBR(g), X10(g), X11(g), X12(g),
X20(g), X21(g), X22(g);
// Temporary distributions
DistMatrix<F,STAR,STAR> L11_STAR_STAR(g);
DistMatrix<F,MC, STAR> L21_MC_STAR(g);
DistMatrix<F,STAR,MR > X10_STAR_MR(g);
DistMatrix<F,STAR,VR > X10_STAR_VR(g);
DistMatrix<F,STAR,MR > X11_STAR_MR(g);
DistMatrix<F,STAR,STAR> X11_STAR_STAR(g);
// Start the algorithm
ScaleTrapezoid( alpha, LOWER, X );
LockedPartitionDownDiagonal
( L, LTL, LTR,
LBL, LBR, 0 );
PartitionDownDiagonal
( X, XTL, XTR,
XBL, XBR, 0 );
while( XBR.Height() > 0 )
{
LockedRepartitionDownDiagonal
( LTL, /**/ LTR, L00, /**/ L01, L02,
/*************/ /******************/
/**/ L10, /**/ L11, L12,
LBL, /**/ LBR, L20, /**/ L21, L22 );
RepartitionDownDiagonal
( XTL, /**/ XTR, X00, /**/ X01, X02,
/*************/ /******************/
/**/ X10, /**/ X11, X12,
XBL, /**/ XBR, X20, /**/ X21, X22 );
L21_MC_STAR.AlignWith( X20 );
X10_STAR_MR.AlignWith( X20 );
X11_STAR_MR.AlignWith( X21 );
//--------------------------------------------------------------------//
L11_STAR_STAR = L11;
X11_STAR_STAR = X11;
X10_STAR_VR = X10;
LocalTrsm
( LEFT, LOWER, NORMAL, diag, F(1), L11_STAR_STAR, X10_STAR_VR,
checkIfSingular );
LocalTrtrsm
( LEFT, LOWER, NORMAL, diag, F(1), L11_STAR_STAR, X11_STAR_STAR,
checkIfSingular );
X11 = X11_STAR_STAR;
X11_STAR_MR = X11_STAR_STAR;
MakeTriangular( LOWER, X11_STAR_MR );
X10_STAR_MR = X10_STAR_VR;
X10 = X10_STAR_MR;
L21_MC_STAR = L21;
LocalGemm
( NORMAL, NORMAL, F(-1), L21_MC_STAR, X10_STAR_MR, F(1), X20 );
LocalGemm
( NORMAL, NORMAL, F(-1), L21_MC_STAR, X11_STAR_MR, F(1), X21 );
//--------------------------------------------------------------------//
SlideLockedPartitionDownDiagonal
( LTL, /**/ LTR, L00, L01, /**/ L02,
/**/ L10, L11, /**/ L12,
/*************/ /******************/
LBL, /**/ LBR, L20, L21, /**/ L22 );
SlidePartitionDownDiagonal
( XTL, /**/ XTR, X00, X01, /**/ X02,
/**/ X10, X11, /**/ X12,
/*************/ /******************/
XBL, /**/ XBR, X20, X21, /**/ X22 );
}
}
inline void
TwoSidedTrsmUVar1
( UnitOrNonUnit diag, DistMatrix<F>& A, const DistMatrix<F>& U )
{
#ifndef RELEASE
PushCallStack("internal::TwoSidedTrsmUVar1");
if( A.Height() != A.Width() )
throw std::logic_error("A must be square");
if( U.Height() != U.Width() )
throw std::logic_error("Triangular matrices must be square");
if( A.Height() != U.Height() )
throw std::logic_error("A and U must be the same size");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F>
UTL(g), UTR(g), U00(g), U01(g), U02(g),
UBL(g), UBR(g), U10(g), U11(g), U12(g),
U20(g), U21(g), U22(g);
// Temporary distributions
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,VC, STAR> A01_VC_STAR(g);
DistMatrix<F,STAR,STAR> U11_STAR_STAR(g);
DistMatrix<F,MC, STAR> U01_MC_STAR(g);
DistMatrix<F,VC, STAR> U01_VC_STAR(g);
DistMatrix<F,VR, STAR> U01_VR_STAR(g);
DistMatrix<F,STAR,MR > U01Adj_STAR_MR(g);
DistMatrix<F,STAR,STAR> X11_STAR_STAR(g);
DistMatrix<F,MR, MC > Z01_MR_MC(g);
DistMatrix<F,MC, STAR> Z01_MC_STAR(g);
DistMatrix<F,MR, STAR> Z01_MR_STAR(g);
DistMatrix<F> Y01(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
A01_VC_STAR.AlignWith( A01 );
U01_MC_STAR.AlignWith( A00 );
U01_VR_STAR.AlignWith( A00 );
U01_VC_STAR.AlignWith( A00 );
U01Adj_STAR_MR.AlignWith( A00 );
Y01.AlignWith( A01 );
Z01_MR_MC.AlignWith( A01 );
Z01_MC_STAR.AlignWith( A00 );
Z01_MR_STAR.AlignWith( A00 );
//--------------------------------------------------------------------//
// Y01 := A00 U01
U01_MC_STAR = U01;
U01_VR_STAR = U01_MC_STAR;
U01Adj_STAR_MR.AdjointFrom( U01_VR_STAR );
Z01_MC_STAR.ResizeTo( A01.Height(), A01.Width() );
Z01_MR_STAR.ResizeTo( A01.Height(), A01.Width() );
Zero( Z01_MC_STAR );
Zero( Z01_MR_STAR );
LocalSymmetricAccumulateLU
( ADJOINT,
F(1), A00, U01_MC_STAR, U01Adj_STAR_MR, Z01_MC_STAR, Z01_MR_STAR );
Z01_MR_MC.SumScatterFrom( Z01_MR_STAR );
Y01 = Z01_MR_MC;
Y01.SumScatterUpdate( F(1), Z01_MC_STAR );
// A01 := inv(U00)' A01
//
// This is the bottleneck because A01 only has blocksize columns
Trsm( LEFT, UPPER, ADJOINT, diag, F(1), U00, A01 );
// A01 := A01 - 1/2 Y01
Axpy( F(-1)/F(2), Y01, A01 );
// A11 := A11 - (U01' A01 + A01' U01)
A01_VC_STAR = A01;
U01_VC_STAR = U01_MC_STAR;
X11_STAR_STAR.ResizeTo( A11.Height(), A11.Width() );
Her2k
( UPPER, ADJOINT,
F(-1), A01_VC_STAR.LocalMatrix(), U01_VC_STAR.LocalMatrix(),
F(0), X11_STAR_STAR.LocalMatrix() );
A11.SumScatterUpdate( F(1), X11_STAR_STAR );
// A11 := inv(U11)' A11 inv(U11)
A11_STAR_STAR = A11;
U11_STAR_STAR = U11;
LocalTwoSidedTrsm( UPPER, diag, A11_STAR_STAR, U11_STAR_STAR );
A11 = A11_STAR_STAR;
// A01 := A01 - 1/2 Y01
Axpy( F(-1)/F(2), Y01, A01 );
// A01 := A01 inv(U11)
A01_VC_STAR = A01;
LocalTrsm
( RIGHT, UPPER, NORMAL, diag, F(1), U11_STAR_STAR, A01_VC_STAR );
A01 = A01_VC_STAR;
//--------------------------------------------------------------------//
A01_VC_STAR.FreeAlignments();
U01_MC_STAR.FreeAlignments();
U01_VR_STAR.FreeAlignments();
U01_VC_STAR.FreeAlignments();
U01Adj_STAR_MR.FreeAlignments();
Y01.FreeAlignments();
Z01_MR_MC.FreeAlignments();
Z01_MC_STAR.FreeAlignments();
Z01_MR_STAR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/*************/ /******************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
TrtrsmLLN
( UnitOrNonUnit diag, F alpha, const Matrix<F>& L, Matrix<F>& X,
bool checkIfSingular=true )
{
#ifndef RELEASE
CallStackEntry entry("internal::TrtrsmLLN");
#endif
// Matrix views
Matrix<F>
LTL, LTR, L00, L01, L02,
LBL, LBR, L10, L11, L12,
L20, L21, L22;
Matrix<F>
XTL, XTR, X00, X01, X02,
XBL, XBR, X10, X11, X12,
X20, X21, X22;
Matrix<F> Z11;
// Start the algorithm
ScaleTrapezoid( alpha, LOWER, X );
LockedPartitionDownDiagonal
( L, LTL, LTR,
LBL, LBR, 0 );
PartitionDownDiagonal
( X, XTL, XTR,
XBL, XBR, 0 );
while( XBR.Height() > 0 )
{
LockedRepartitionDownDiagonal
( LTL, /**/ LTR, L00, /**/ L01, L02,
/*************/ /******************/
/**/ L10, /**/ L11, L12,
LBL, /**/ LBR, L20, /**/ L21, L22 );
RepartitionDownDiagonal
( XTL, /**/ XTR, X00, /**/ X01, X02,
/*************/ /******************/
/**/ X10, /**/ X11, X12,
XBL, /**/ XBR, X20, /**/ X21, X22 );
//--------------------------------------------------------------------//
Trsm( LEFT, LOWER, NORMAL, diag, F(1), L11, X10, checkIfSingular );
TrtrsmLLNUnb( diag, F(1), L11, X11 );
Gemm( NORMAL, NORMAL, F(-1), L21, X10, F(1), X20 );
Z11 = X11;
MakeTriangular( LOWER, Z11 );
Gemm( NORMAL, NORMAL, F(-1), L21, Z11, F(1), X21 );
//--------------------------------------------------------------------//
SlideLockedPartitionDownDiagonal
( LTL, /**/ LTR, L00, L01, /**/ L02,
/**/ L10, L11, /**/ L12,
/*************/ /******************/
LBL, /**/ LBR, L20, L21, /**/ L22 );
SlidePartitionDownDiagonal
( XTL, /**/ XTR, X00, X01, /**/ X02,
/**/ X10, X11, /**/ X12,
/*************/ /******************/
XBL, /**/ XBR, X20, X21, /**/ X22 );
}
}
inline void
TwoSidedTrsmUVar1( UnitOrNonUnit diag, Matrix<F>& A, const Matrix<F>& U )
{
#ifndef RELEASE
PushCallStack("internal::TwoSidedTrsmUVar1");
if( A.Height() != A.Width() )
throw std::logic_error("A must be square");
if( U.Height() != U.Width() )
throw std::logic_error("Triangular matrices must be square");
if( A.Height() != U.Height() )
throw std::logic_error("A and U must be the same size");
#endif
// Matrix views
Matrix<F>
ATL, ATR, A00, A01, A02,
ABL, ABR, A10, A11, A12,
A20, A21, A22;
Matrix<F>
UTL, UTR, U00, U01, U02,
UBL, UBR, U10, U11, U12,
U20, U21, U22;
// Temporary products
Matrix<F> Y01;
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
//--------------------------------------------------------------------//
// Y01 := A00 U01
Zeros( A01.Height(), A01.Width(), Y01 );
Hemm( LEFT, UPPER, F(1), A00, U01, F(0), Y01 );
// A01 := inv(U00)' A01
Trsm( LEFT, UPPER, ADJOINT, diag, F(1), U00, A01 );
// A01 := A01 - 1/2 Y01
Axpy( F(-1)/F(2), Y01, A01 );
// A11 := A11 - (U01' A01 + A01' U01)
Her2k( UPPER, ADJOINT, F(-1), U01, A01, F(1), A11 );
// A11 := inv(U11)' A11 inv(U11)
TwoSidedTrsmUUnb( diag, A11, U11 );
// A01 := A01 - 1/2 Y01
Axpy( F(-1)/F(2), Y01, A01 );
// A01 := A01 inv(U11)
Trsm( RIGHT, UPPER, NORMAL, diag, F(1), U11, A01 );
//--------------------------------------------------------------------//
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/*************/ /******************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
HemmRUC
( T alpha, const DistMatrix<T>& A,
const DistMatrix<T>& B,
T beta, DistMatrix<T>& C )
{
#ifndef RELEASE
PushCallStack("internal::HemmRUC");
if( A.Grid() != B.Grid() || B.Grid() != C.Grid() )
throw std::logic_error("{A,B,C} must be distributed on the same grid");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<T>
ATL(g), ATR(g), A00(g), A01(g), A02(g), AColPan(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g), ARowPan(g),
A20(g), A21(g), A22(g);
DistMatrix<T> BL(g), BR(g),
B0(g), B1(g), B2(g);
DistMatrix<T> CL(g), CR(g),
C0(g), C1(g), C2(g),
CLeft(g), CRight(g);
// Temporary distributions
DistMatrix<T,MC,STAR> B1_MC_STAR(g);
DistMatrix<T,VR, STAR> AColPan_VR_STAR(g);
DistMatrix<T,STAR,MR > AColPanAdj_STAR_MR(g);
DistMatrix<T,MR, STAR> ARowPanAdj_MR_STAR(g);
B1_MC_STAR.AlignWith( C );
// Start the algorithm
Scale( beta, C );
LockedPartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionRight( B, BL, BR, 0 );
PartitionRight( C, CL, CR, 0 );
while( CR.Width() > 0 )
{
LockedRepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionRight
( BL, /**/ BR,
B0, /**/ B1, B2 );
RepartitionRight
( CL, /**/ CR,
C0, /**/ C1, C2 );
ARowPan.LockedView1x2( A11, A12 );
AColPan.LockedView2x1
( A01,
A11 );
CLeft.View1x2( C0, C1 );
CRight.View1x2( C1, C2 );
AColPan_VR_STAR.AlignWith( CLeft );
AColPanAdj_STAR_MR.AlignWith( CLeft );
ARowPanAdj_MR_STAR.AlignWith( CRight );
//--------------------------------------------------------------------//
B1_MC_STAR = B1;
AColPan_VR_STAR = AColPan;
AColPanAdj_STAR_MR.AdjointFrom( AColPan_VR_STAR );
ARowPanAdj_MR_STAR.AdjointFrom( ARowPan );
MakeTrapezoidal( LEFT, LOWER, 0, ARowPanAdj_MR_STAR );
MakeTrapezoidal( RIGHT, LOWER, -1, AColPanAdj_STAR_MR );
LocalGemm
( NORMAL, ADJOINT,
alpha, B1_MC_STAR, ARowPanAdj_MR_STAR, T(1), CRight );
LocalGemm
( NORMAL, NORMAL,
alpha, B1_MC_STAR, AColPanAdj_STAR_MR, T(1), CLeft );
//--------------------------------------------------------------------//
AColPan_VR_STAR.FreeAlignments();
AColPanAdj_STAR_MR.FreeAlignments();
ARowPanAdj_MR_STAR.FreeAlignments();
SlideLockedPartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionRight
( BL, /**/ BR,
B0, B1, /**/ B2 );
SlidePartitionRight
( CL, /**/ CR,
C0, C1, /**/ C2 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
internal::HegstLUVar2( DistMatrix<F,MC,MR>& A, const DistMatrix<F,MC,MR>& U )
{
#ifndef RELEASE
PushCallStack("internal::HegstLUVar2");
if( A.Height() != A.Width() )
throw std::logic_error("A must be square");
if( U.Height() != U.Width() )
throw std::logic_error("Triangular matrices must be square");
if( A.Height() != U.Height() )
throw std::logic_error("A and U must be the same size");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F,MC,MR>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F,MC,MR>
UTL(g), UTR(g), U00(g), U01(g), U02(g),
UBL(g), UBR(g), U10(g), U11(g), U12(g),
U20(g), U21(g), U22(g);
// Temporary distributions
DistMatrix<F,VC, STAR> A01_VC_STAR(g);
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,STAR,VR > A12_STAR_VR(g);
DistMatrix<F,STAR,STAR> U11_STAR_STAR(g);
DistMatrix<F,STAR,MC > U12_STAR_MC(g);
DistMatrix<F,STAR,VR > U12_STAR_VR(g);
DistMatrix<F,MR, STAR> U12Adj_MR_STAR(g);
DistMatrix<F,VC, STAR> U12Adj_VC_STAR(g);
DistMatrix<F,MC, STAR> X01_MC_STAR(g);
DistMatrix<F,STAR,STAR> X11_STAR_STAR(g);
DistMatrix<F,MC, MR > Y12(g);
DistMatrix<F,MC, MR > Z12Adj(g);
DistMatrix<F,MR, MC > Z12Adj_MR_MC(g);
DistMatrix<F,MC, STAR> Z12Adj_MC_STAR(g);
DistMatrix<F,MR, STAR> Z12Adj_MR_STAR(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
A12_STAR_VR.AlignWith( A12 );
U12_STAR_MC.AlignWith( A22 );
U12_STAR_VR.AlignWith( A12 );
U12Adj_MR_STAR.AlignWith( A22 );
U12Adj_VC_STAR.AlignWith( A22 );
X01_MC_STAR.AlignWith( A01 );
Y12.AlignWith( A12 );
Z12Adj.AlignWith( A12 );
Z12Adj_MR_MC.AlignWith( A12 );
Z12Adj_MC_STAR.AlignWith( A22 );
Z12Adj_MR_STAR.AlignWith( A22 );
//--------------------------------------------------------------------//
// A01 := A01 U11'
U11_STAR_STAR = U11;
A01_VC_STAR = A01;
internal::LocalTrmm
( RIGHT, UPPER, ADJOINT, NON_UNIT, (F)1, U11_STAR_STAR, A01_VC_STAR );
A01 = A01_VC_STAR;
// A01 := A01 + A02 U12'
U12Adj_MR_STAR.AdjointFrom( U12 );
X01_MC_STAR.ResizeTo( A01.Height(), A01.Width() );
internal::LocalGemm
( NORMAL, NORMAL,
(F)1, A02, U12Adj_MR_STAR, (F)0, X01_MC_STAR );
A01.SumScatterUpdate( (F)1, X01_MC_STAR );
// Y12 := U12 A22
U12Adj_VC_STAR = U12Adj_MR_STAR;
U12_STAR_MC.AdjointFrom( U12Adj_VC_STAR );
Z12Adj_MC_STAR.ResizeTo( A12.Width(), A12.Height() );
Z12Adj_MR_STAR.ResizeTo( A12.Width(), A12.Height() );
Zero( Z12Adj_MC_STAR );
Zero( Z12Adj_MR_STAR );
internal::LocalSymmetricAccumulateRU
( ADJOINT,
(F)1, A22, U12_STAR_MC, U12Adj_MR_STAR,
Z12Adj_MC_STAR, Z12Adj_MR_STAR );
Z12Adj.SumScatterFrom( Z12Adj_MC_STAR );
Z12Adj_MR_MC = Z12Adj;
Z12Adj_MR_MC.SumScatterUpdate( (F)1, Z12Adj_MR_STAR );
Y12.ResizeTo( A12.Height(), A12.Width() );
Adjoint( Z12Adj_MR_MC.LockedLocalMatrix(), Y12.LocalMatrix() );
// A12 := U11 A12
A12_STAR_VR = A12;
U11_STAR_STAR = U11;
internal::LocalTrmm
( LEFT, UPPER, NORMAL, NON_UNIT, (F)1, U11_STAR_STAR, A12_STAR_VR );
A12 = A12_STAR_VR;
// A12 := A12 + 1/2 Y12
Axpy( (F)0.5, Y12, A12 );
// A11 := U11 A11 U11'
A11_STAR_STAR = A11;
internal::LocalHegst( LEFT, UPPER, A11_STAR_STAR, U11_STAR_STAR );
A11 = A11_STAR_STAR;
// A11 := A11 + (A12 U12' + U12 A12')
A12_STAR_VR = A12;
U12_STAR_VR = U12;
X11_STAR_STAR.ResizeTo( A11.Height(), A11.Width() );
Her2k
( UPPER, NORMAL,
(F)1, A12_STAR_VR.LocalMatrix(), U12_STAR_VR.LocalMatrix(),
(F)0, X11_STAR_STAR.LocalMatrix() );
A11.SumScatterUpdate( (F)1, X11_STAR_STAR );
// A12 := A12 + 1/2 Y12
Axpy( (F)0.5, Y12, A12 );
//--------------------------------------------------------------------//
A12_STAR_VR.FreeAlignments();
U12_STAR_MC.FreeAlignments();
U12_STAR_VR.FreeAlignments();
U12Adj_MR_STAR.FreeAlignments();
U12Adj_VC_STAR.FreeAlignments();
X01_MC_STAR.FreeAlignments();
Y12.FreeAlignments();
Z12Adj.FreeAlignments();
Z12Adj_MR_MC.FreeAlignments();
Z12Adj_MC_STAR.FreeAlignments();
Z12Adj_MR_STAR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/*************/ /******************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
TwoSidedTrmmUVar5
( UnitOrNonUnit diag, DistMatrix<F>& A, const DistMatrix<F>& U )
{
#ifndef RELEASE
CallStackEntry entry("internal::TwoSidedTrmmUVar5");
if( A.Height() != A.Width() )
LogicError("A must be square");
if( U.Height() != U.Width() )
LogicError("Triangular matrices must be square");
if( A.Height() != U.Height() )
LogicError("A and U must be the same size");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F>
UTL(g), UTR(g), U00(g), U01(g), U02(g),
UBL(g), UBR(g), U10(g), U11(g), U12(g),
U20(g), U21(g), U22(g);
// Temporary distributions
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,MC, STAR> A01_MC_STAR(g);
DistMatrix<F,MR, STAR> A01_MR_STAR(g);
DistMatrix<F,VC, STAR> A01_VC_STAR(g);
DistMatrix<F,STAR,STAR> U11_STAR_STAR(g);
DistMatrix<F,MC, STAR> U01_MC_STAR(g);
DistMatrix<F,MR, STAR> U01_MR_STAR(g);
DistMatrix<F,VC, STAR> U01_VC_STAR(g);
DistMatrix<F,VC, STAR> Y01_VC_STAR(g);
DistMatrix<F> Y01(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
A01_MC_STAR.AlignWith( A00 );
A01_MR_STAR.AlignWith( A00 );
A01_VC_STAR.AlignWith( A00 );
U01_MC_STAR.AlignWith( A00 );
U01_MR_STAR.AlignWith( A00 );
U01_VC_STAR.AlignWith( A00 );
Y01.AlignWith( A01 );
Y01_VC_STAR.AlignWith( A01 );
//--------------------------------------------------------------------//
// Y01 := U01 A11
A11_STAR_STAR = A11;
U01_VC_STAR = U01;
Zeros( Y01_VC_STAR, A01.Height(), A01.Width() );
Hemm
( RIGHT, UPPER,
F(1), A11_STAR_STAR.Matrix(), U01_VC_STAR.Matrix(),
F(0), Y01_VC_STAR.Matrix() );
Y01 = Y01_VC_STAR;
// A01 := U00 A01
Trmm( LEFT, UPPER, NORMAL, diag, F(1), U00, A01 );
// A01 := A01 + 1/2 Y01
Axpy( F(1)/F(2), Y01, A01 );
// A00 := A00 + (U01 A01' + A01 U01')
A01_MC_STAR = A01;
U01_MC_STAR = U01;
A01_VC_STAR = A01_MC_STAR;
A01_MR_STAR = A01_VC_STAR;
U01_MR_STAR = U01_MC_STAR;
LocalTrr2k
( UPPER, ADJOINT, ADJOINT,
F(1), U01_MC_STAR, A01_MR_STAR,
A01_MC_STAR, U01_MR_STAR,
F(1), A00 );
// A01 := A01 + 1/2 Y01
Axpy( F(1)/F(2), Y01_VC_STAR, A01_VC_STAR );
// A01 := A01 U11'
U11_STAR_STAR = U11;
LocalTrmm
( RIGHT, UPPER, ADJOINT, diag, F(1), U11_STAR_STAR, A01_VC_STAR );
A01 = A01_VC_STAR;
// A11 := U11 A11 U11'
LocalTwoSidedTrmm( UPPER, diag, A11_STAR_STAR, U11_STAR_STAR );
A11 = A11_STAR_STAR;
//--------------------------------------------------------------------//
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/*************/ /******************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
}
}
inline void
ApplyPackedReflectorsRLHF
( Conjugation conjugation, int offset,
const Matrix<Complex<R> >& H,
const Matrix<Complex<R> >& t,
Matrix<Complex<R> >& A )
{
#ifndef RELEASE
PushCallStack("internal::ApplyPackedReflectorsRLHF");
if( offset > 0 || offset < -H.Width() )
throw std::logic_error("Transforms out of bounds");
if( H.Width() != A.Width() )
throw std::logic_error
("Width of transforms must equal width of target matrix");
if( t.Height() != H.DiagonalLength( offset ) )
throw std::logic_error("t must be the same length as H's offset diag");
#endif
typedef Complex<R> C;
Matrix<C>
HTL, HTR, H00, H01, H02, HPan, HPanCopy,
HBL, HBR, H10, H11, H12,
H20, H21, H22;
Matrix<C> ALeft;
Matrix<C>
tT, t0,
tB, t1,
t2;
Matrix<C> SInv, Z;
LockedPartitionDownDiagonal
( H, HTL, HTR,
HBL, HBR, 0 );
LockedPartitionDown
( t, tT,
tB, 0 );
while( HTL.Height() < H.Height() && HTL.Width() < H.Width() )
{
LockedRepartitionDownDiagonal
( HTL, /**/ HTR, H00, /**/ H01, H02,
/*************/ /******************/
/**/ H10, /**/ H11, H12,
HBL, /**/ HBR, H20, /**/ H21, H22 );
const int HPanWidth = H10.Width() + H11.Width();
const int HPanOffset =
std::min( H11.Height(), std::max(-offset-H00.Height(),0) );
const int HPanHeight = H11.Height()-HPanOffset;
HPan.LockedView( H, H00.Height()+HPanOffset, 0, HPanHeight, HPanWidth );
LockedRepartitionDown
( tT, t0,
/**/ /**/
t1,
tB, t2, HPanHeight );
ALeft.View( A, 0, 0, A.Height(), HPanWidth );
Zeros( ALeft.Height(), HPan.Height(), Z );
Zeros( HPan.Height(), HPan.Height(), SInv );
//--------------------------------------------------------------------//
HPanCopy = HPan;
MakeTrapezoidal( RIGHT, LOWER, offset, HPanCopy );
SetDiagonalToOne( RIGHT, offset, HPanCopy );
Herk( UPPER, NORMAL, C(1), HPanCopy, C(0), SInv );
FixDiagonal( conjugation, t1, SInv );
Gemm( NORMAL, ADJOINT, C(1), ALeft, HPanCopy, C(0), Z );
Trsm( RIGHT, UPPER, NORMAL, NON_UNIT, C(1), SInv, Z );
Gemm( NORMAL, NORMAL, C(-1), Z, HPanCopy, C(1), ALeft );
//--------------------------------------------------------------------//
SlideLockedPartitionDownDiagonal
( HTL, /**/ HTR, H00, H01, /**/ H02,
/**/ H10, H11, /**/ H12,
/*************/ /******************/
HBL, /**/ HBR, H20, H21, /**/ H22 );
SlideLockedPartitionDown
( tT, t0,
t1,
/**/ /**/
tB, t2 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
internal::HegstRLVar3( DistMatrix<F,MC,MR>& A, const DistMatrix<F,MC,MR>& L )
{
#ifndef RELEASE
PushCallStack("internal::HegstRLVar4");
if( A.Height() != A.Width() )
throw std::logic_error("A must be square");
if( L.Height() != L.Width() )
throw std::logic_error("Triangular matrices must be square");
if( A.Height() != L.Height() )
throw std::logic_error("A and L must be the same size");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F,MC,MR>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F,MC,MR>
YTL(g), YTR(g), Y00(g), Y01(g), Y02(g),
YBL(g), YBR(g), Y10(g), Y11(g), Y12(g),
Y20(g), Y21(g), Y22(g);
DistMatrix<F,MC,MR>
LTL(g), LTR(g), L00(g), L01(g), L02(g),
LBL(g), LBR(g), L10(g), L11(g), L12(g),
L20(g), L21(g), L22(g);
// Temporary distributions
DistMatrix<F,STAR,MR > A11_STAR_MR(g);
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,VC, STAR> A21_VC_STAR(g);
DistMatrix<F,STAR,VR > A10_STAR_VR(g);
DistMatrix<F,STAR,MR > A10_STAR_MR(g);
DistMatrix<F,STAR,STAR> L11_STAR_STAR(g);
DistMatrix<F,STAR,VR > L10_STAR_VR(g);
DistMatrix<F,STAR,MR > L10_STAR_MR(g);
DistMatrix<F,MC, STAR> L21_MC_STAR(g);
DistMatrix<F,STAR,STAR> X11_STAR_STAR(g);
DistMatrix<F,MC, STAR> X21_MC_STAR(g);
DistMatrix<F,MC, STAR> Z21_MC_STAR(g);
// We will use an entire extra matrix as temporary storage. If this is not
// acceptable, use HegstRLVar4 instead.
DistMatrix<F,MC,MR> Y(g);
Y.AlignWith( A );
Y.ResizeTo( A.Height(), A.Width() );
Zero( Y );
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
PartitionDownDiagonal
( Y, YTL, YTR,
YBL, YBR, 0 );
LockedPartitionDownDiagonal
( L, LTL, LTR,
LBL, LBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
RepartitionDownDiagonal
( YTL, /**/ YTR, Y00, /**/ Y01, Y02,
/*************/ /******************/
/**/ Y10, /**/ Y11, Y12,
YBL, /**/ YBR, Y20, /**/ Y21, Y22 );
LockedRepartitionDownDiagonal
( LTL, /**/ LTR, L00, /**/ L01, L02,
/*************/ /******************/
/**/ L10, /**/ L11, L12,
LBL, /**/ LBR, L20, /**/ L21, L22 );
A11_STAR_MR.AlignWith( Y21 );
A21_VC_STAR.AlignWith( A21 );
A10_STAR_VR.AlignWith( A10 );
A10_STAR_MR.AlignWith( A10 );
L10_STAR_VR.AlignWith( A10 );
L10_STAR_MR.AlignWith( A10 );
L21_MC_STAR.AlignWith( Y21 );
X21_MC_STAR.AlignWith( A20 );
Z21_MC_STAR.AlignWith( L20 );
//--------------------------------------------------------------------//
// A10 := A10 - 1/2 Y10
Axpy( (F)-0.5, Y10, A10 );
// A11 := A11 - (A10 L10' + L10 A10')
A10_STAR_VR = A10;
L10_STAR_VR = L10;
X11_STAR_STAR.ResizeTo( A11.Height(), A11.Width() );
Her2k
( LOWER, NORMAL,
(F)1, A10_STAR_VR.LocalMatrix(), L10_STAR_VR.LocalMatrix(),
(F)0, X11_STAR_STAR.LocalMatrix() );
MakeTrapezoidal( LEFT, LOWER, 0, X11_STAR_STAR );
A11.SumScatterUpdate( (F)-1, X11_STAR_STAR );
// A11 := inv(L11) A11 inv(L11)'
A11_STAR_STAR = A11;
L11_STAR_STAR = L11;
internal::LocalHegst( RIGHT, LOWER, A11_STAR_STAR, L11_STAR_STAR );
A11 = A11_STAR_STAR;
// A21 := A21 - A20 L10'
L10_STAR_MR = L10_STAR_VR;
X21_MC_STAR.ResizeTo( A21.Height(), A21.Width() );
internal::LocalGemm
( NORMAL, ADJOINT, (F)1, A20, L10_STAR_MR, (F)0, X21_MC_STAR );
A21.SumScatterUpdate( (F)-1, X21_MC_STAR );
// A21 := A21 inv(L11)'
A21_VC_STAR = A21;
internal::LocalTrsm
( RIGHT, LOWER, ADJOINT, NON_UNIT, (F)1, L11_STAR_STAR, A21_VC_STAR );
A21 = A21_VC_STAR;
// A10 := A10 - 1/2 Y10
Axpy( (F)-0.5, Y10, A10 );
// A10 := inv(L11) A10
A10_STAR_VR = A10;
internal::LocalTrsm
( LEFT, LOWER, NORMAL, NON_UNIT,
(F)1, L11_STAR_STAR, A10_STAR_VR );
// Y20 := Y20 + L21 A10
A10_STAR_MR = A10_STAR_VR;
A10 = A10_STAR_MR;
L21_MC_STAR = L21;
internal::LocalGemm
( NORMAL, NORMAL, (F)1, L21_MC_STAR, A10_STAR_MR, (F)1, Y20 );
// Y21 := L21 A11
//
// Symmetrize A11[* ,* ] by copying the lower triangle into the upper
// so that we can call a local gemm instead of worrying about
// reproducing a hemm with nonsymmetric local matrices.
{
const int height = A11_STAR_STAR.LocalHeight();
const int ldim = A11_STAR_STAR.LocalLDim();
F* A11Buffer = A11_STAR_STAR.LocalBuffer();
for( int i=1; i<height; ++i )
for( int j=0; j<i; ++j )
A11Buffer[j+i*ldim] = Conj(A11Buffer[i+j*ldim]);
}
A11_STAR_MR = A11_STAR_STAR;
internal::LocalGemm
( NORMAL, NORMAL, (F)1, L21_MC_STAR, A11_STAR_MR, (F)0, Y21 );
// Y21 := Y21 + L20 A10'
Z21_MC_STAR.ResizeTo( A21.Height(), A21.Width() );
internal::LocalGemm
( NORMAL, ADJOINT, (F)1, L20, A10_STAR_MR, (F)0, Z21_MC_STAR );
Y21.SumScatterUpdate( (F)1, Z21_MC_STAR );
//--------------------------------------------------------------------//
A11_STAR_MR.FreeAlignments();
A21_VC_STAR.FreeAlignments();
A10_STAR_VR.FreeAlignments();
A10_STAR_MR.FreeAlignments();
L10_STAR_VR.FreeAlignments();
L10_STAR_MR.FreeAlignments();
L21_MC_STAR.FreeAlignments();
X21_MC_STAR.FreeAlignments();
Z21_MC_STAR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlidePartitionDownDiagonal
( YTL, /**/ YTR, Y00, Y01, /**/ Y02,
/**/ Y10, Y11, /**/ Y12,
/*************/ /******************/
YBL, /**/ YBR, Y20, Y21, /**/ Y22 );
SlideLockedPartitionDownDiagonal
( LTL, /**/ LTR, L00, L01, /**/ L02,
/**/ L10, L11, /**/ L12,
/**********************************/
LBL, /**/ LBR, L20, L21, /**/ L22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
ApplyPackedReflectorsRLHF
( Conjugation conjugation, int offset,
const DistMatrix<Complex<R> >& H,
const DistMatrix<Complex<R>,MD,STAR>& t,
DistMatrix<Complex<R> >& A )
{
#ifndef RELEASE
PushCallStack("internal::ApplyPackedReflectorsRLHF");
if( H.Grid() != t.Grid() || t.Grid() != A.Grid() )
throw std::logic_error
("{H,t,A} must be distributed over the same grid");
if( offset > 0 || offset < -H.Width() )
throw std::logic_error("Transforms out of bounds");
if( H.Width() != A.Width() )
throw std::logic_error
("Width of transforms must equal width of target matrix");
if( t.Height() != H.DiagonalLength( offset ) )
throw std::logic_error("t must be the same length as H's offset diag");
if( !t.AlignedWithDiagonal( H, offset ) )
throw std::logic_error("t must be aligned with H's 'offset' diagonal");
#endif
typedef Complex<R> C;
const Grid& g = H.Grid();
DistMatrix<C>
HTL(g), HTR(g), H00(g), H01(g), H02(g), HPan(g), HPanCopy(g),
HBL(g), HBR(g), H10(g), H11(g), H12(g),
H20(g), H21(g), H22(g);
DistMatrix<C> ALeft(g);
DistMatrix<C,MD,STAR>
tT(g), t0(g),
tB(g), t1(g),
t2(g);
DistMatrix<C,STAR,VR > HPan_STAR_VR(g);
DistMatrix<C,STAR,MR > HPan_STAR_MR(g);
DistMatrix<C,STAR,STAR> t1_STAR_STAR(g);
DistMatrix<C,STAR,STAR> SInv_STAR_STAR(g);
DistMatrix<C,STAR,MC > ZAdj_STAR_MC(g);
DistMatrix<C,STAR,VC > ZAdj_STAR_VC(g);
LockedPartitionDownDiagonal
( H, HTL, HTR,
HBL, HBR, 0 );
LockedPartitionDown
( t, tT,
tB, 0 );
while( HTL.Height() < H.Height() && HTL.Width() < H.Width() )
{
LockedRepartitionDownDiagonal
( HTL, /**/ HTR, H00, /**/ H01, H02,
/*************/ /******************/
/**/ H10, /**/ H11, H12,
HBL, /**/ HBR, H20, /**/ H21, H22 );
const int HPanWidth = H10.Width() + H11.Width();
const int HPanOffset =
std::min( H11.Height(), std::max(-offset-H00.Height(),0) );
const int HPanHeight = H11.Height()-HPanOffset;
HPan.LockedView( H, H00.Height()+HPanOffset, 0, HPanHeight, HPanWidth );
LockedRepartitionDown
( tT, t0,
/**/ /**/
t1,
tB, t2, HPanHeight );
ALeft.View( A, 0, 0, A.Height(), HPanWidth );
HPan_STAR_MR.AlignWith( ALeft );
ZAdj_STAR_MC.AlignWith( ALeft );
ZAdj_STAR_VC.AlignWith( ALeft );
Zeros( HPan.Height(), ALeft.Height(), ZAdj_STAR_MC );
Zeros( HPan.Height(), HPan.Height(), SInv_STAR_STAR );
//--------------------------------------------------------------------//
HPanCopy = HPan;
MakeTrapezoidal( RIGHT, LOWER, offset, HPanCopy );
SetDiagonalToOne( RIGHT, offset, HPanCopy );
HPan_STAR_VR = HPanCopy;
Herk
( UPPER, NORMAL,
C(1), HPan_STAR_VR.LockedLocalMatrix(),
C(0), SInv_STAR_STAR.LocalMatrix() );
SInv_STAR_STAR.SumOverGrid();
t1_STAR_STAR = t1;
FixDiagonal( conjugation, t1_STAR_STAR, SInv_STAR_STAR );
HPan_STAR_MR = HPan_STAR_VR;
LocalGemm
( NORMAL, ADJOINT,
C(1), HPan_STAR_MR, ALeft, C(0), ZAdj_STAR_MC );
ZAdj_STAR_VC.SumScatterFrom( ZAdj_STAR_MC );
LocalTrsm
( LEFT, UPPER, ADJOINT, NON_UNIT,
C(1), SInv_STAR_STAR, ZAdj_STAR_VC );
ZAdj_STAR_MC = ZAdj_STAR_VC;
LocalGemm
( ADJOINT, NORMAL,
C(-1), ZAdj_STAR_MC, HPan_STAR_MR, C(1), ALeft );
//--------------------------------------------------------------------//
HPan_STAR_MR.FreeAlignments();
ZAdj_STAR_MC.FreeAlignments();
ZAdj_STAR_VC.FreeAlignments();
SlideLockedPartitionDownDiagonal
( HTL, /**/ HTR, H00, H01, /**/ H02,
/**/ H10, H11, /**/ H12,
/*************/ /******************/
HBL, /**/ HBR, H20, H21, /**/ H22 );
SlideLockedPartitionDown
( tT, t0,
t1,
/**/ /**/
tB, t2 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
TwoSidedTrsmLVar5( UnitOrNonUnit diag, Matrix<F>& A, const Matrix<F>& L )
{
#ifndef RELEASE
CallStackEntry entry("internal::TwoSidedTrsmLVar5");
if( A.Height() != A.Width() )
LogicError("A must be square");
if( L.Height() != L.Width() )
LogicError("Triangular matrices must be square");
if( A.Height() != L.Height() )
LogicError("A and L must be the same size");
#endif
// Matrix views
Matrix<F>
ATL, ATR, A00, A01, A02,
ABL, ABR, A10, A11, A12,
A20, A21, A22;
Matrix<F>
LTL, LTR, L00, L01, L02,
LBL, LBR, L10, L11, L12,
L20, L21, L22;
// Temporary products
Matrix<F> Y21;
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( L, LTL, LTR,
LBL, LBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( LTL, /**/ LTR, L00, /**/ L01, L02,
/*************/ /******************/
/**/ L10, /**/ L11, L12,
LBL, /**/ LBR, L20, /**/ L21, L22 );
//--------------------------------------------------------------------//
// A11 := inv(L11) A11 inv(L11)'
TwoSidedTrsmLUnb( diag, A11, L11 );
// Y21 := L21 A11
Zeros( Y21, A21.Height(), A21.Width() );
Hemm( RIGHT, LOWER, F(1), A11, L21, F(0), Y21 );
// A21 := A21 inv(L11)'
Trsm( RIGHT, LOWER, ADJOINT, diag, F(1), L11, A21 );
// A21 := A21 - 1/2 Y21
Axpy( F(-1)/F(2), Y21, A21 );
// A22 := A22 - (L21 A21' + A21 L21')
Her2k( LOWER, NORMAL, F(-1), L21, A21, F(1), A22 );
// A21 := A21 - 1/2 Y21
Axpy( F(-1)/F(2), Y21, A21 );
// A21 := inv(L22) A21
Trsm( LEFT, LOWER, NORMAL, diag, F(1), L22, A21 );
//--------------------------------------------------------------------//
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( LTL, /**/ LTR, L00, L01, /**/ L02,
/**/ L10, L11, /**/ L12,
/**********************************/
LBL, /**/ LBR, L20, L21, /**/ L22 );
}
}
inline void
ApplyPackedReflectorsRLHF
( int offset, const Matrix<R>& H, Matrix<R>& A )
{
#ifndef RELEASE
PushCallStack("internal::ApplyPackedReflectorsRLHF");
if( offset > 0 || offset < -H.Width() )
throw std::logic_error("Transforms out of bounds");
if( H.Width() != A.Width() )
throw std::logic_error
("Width of transforms must equal width of target matrix");
#endif
Matrix<R>
HTL, HTR, H00, H01, H02, HPan, HPanCopy,
HBL, HBR, H10, H11, H12,
H20, H21, H22;
Matrix<R> ALeft;
Matrix<R> SInv, Z;
LockedPartitionDownDiagonal
( H, HTL, HTR,
HBL, HBR, 0 );
while( HTL.Height() < H.Height() && HTL.Width() < H.Width() )
{
LockedRepartitionDownDiagonal
( HTL, /**/ HTR, H00, /**/ H01, H02,
/*************/ /******************/
/**/ H10, /**/ H11, H12,
HBL, /**/ HBR, H20, /**/ H21, H22 );
const int HPanWidth = H10.Width() + H11.Width();
const int HPanOffset =
std::min( H11.Height(), std::max(-offset-H00.Height(),0) );
const int HPanHeight = H11.Height()-HPanOffset;
HPan.LockedView( H, H00.Height()+HPanOffset, 0, HPanHeight, HPanWidth );
ALeft.View( A, 0, 0, A.Height(), HPanWidth );
Zeros( ALeft.Height(), HPan.Height(), Z );
Zeros( HPan.Height(), HPan.Height(), SInv );
//--------------------------------------------------------------------//
HPanCopy = HPan;
MakeTrapezoidal( RIGHT, LOWER, offset, HPanCopy );
SetDiagonalToOne( RIGHT, offset, HPanCopy );
Syrk( UPPER, NORMAL, R(1), HPanCopy, R(0), SInv );
HalveMainDiagonal( SInv );
Gemm( NORMAL, TRANSPOSE, R(1), ALeft, HPanCopy, R(0), Z );
Trsm( RIGHT, UPPER, NORMAL, NON_UNIT, R(1), SInv, Z );
Gemm( NORMAL, NORMAL, R(-1), Z, HPanCopy, R(1), ALeft );
//--------------------------------------------------------------------//
SlideLockedPartitionDownDiagonal
( HTL, /**/ HTR, H00, H01, /**/ H02,
/**/ H10, H11, /**/ H12,
/*************/ /******************/
HBL, /**/ HBR, H20, H21, /**/ H22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
SymmLLC
( T alpha, const DistMatrix<T,MC,MR>& A,
const DistMatrix<T,MC,MR>& B,
T beta, DistMatrix<T,MC,MR>& C )
{
#ifndef RELEASE
PushCallStack("internal::SymmLLC");
if( A.Grid() != B.Grid() || B.Grid() != C.Grid() )
throw std::logic_error
("{A,B,C} must be distributed over the same grid");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<T,MC,MR>
ATL(g), ATR(g), A00(g), A01(g), A02(g), AColPan(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g), ARowPan(g),
A20(g), A21(g), A22(g);
DistMatrix<T,MC,MR>
BT(g), B0(g),
BB(g), B1(g),
B2(g);
DistMatrix<T,MC,MR>
CT(g), C0(g), CAbove(g),
CB(g), C1(g), CBelow(g),
C2(g);
// Temporary distributions
DistMatrix<T,MC, STAR> AColPan_MC_STAR(g);
DistMatrix<T,STAR,MC > ARowPan_STAR_MC(g);
DistMatrix<T,MR, STAR> B1Trans_MR_STAR(g);
B1Trans_MR_STAR.AlignWith( C );
// Start the algorithm
Scale( beta, C );
LockedPartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDown
( B, BT,
BB, 0 );
PartitionDown
( C, CT,
CB, 0 );
while( CB.Height() > 0 )
{
LockedRepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDown
( BT, B0,
/**/ /**/
B1,
BB, B2 );
RepartitionDown
( CT, C0,
/**/ /**/
C1,
CB, C2 );
ARowPan.LockedView1x2( A10, A11 );
AColPan.LockedView2x1
( A11,
A21 );
CAbove.View2x1
( C0,
C1 );
CBelow.View2x1
( C1,
C2 );
AColPan_MC_STAR.AlignWith( CBelow );
ARowPan_STAR_MC.AlignWith( CAbove );
//--------------------------------------------------------------------//
AColPan_MC_STAR = AColPan;
ARowPan_STAR_MC = ARowPan;
MakeTrapezoidal( LEFT, LOWER, 0, AColPan_MC_STAR );
MakeTrapezoidal( RIGHT, LOWER, -1, ARowPan_STAR_MC );
B1Trans_MR_STAR.TransposeFrom( B1 );
LocalGemm
( NORMAL, TRANSPOSE,
alpha, AColPan_MC_STAR, B1Trans_MR_STAR, T(1), CBelow );
LocalGemm
( TRANSPOSE, TRANSPOSE,
alpha, ARowPan_STAR_MC, B1Trans_MR_STAR, T(1), CAbove );
//--------------------------------------------------------------------//
AColPan_MC_STAR.FreeAlignments();
ARowPan_STAR_MC.FreeAlignments();
SlideLockedPartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDown
( BT, B0,
B1,
/**/ /**/
BB, B2 );
SlidePartitionDown
( CT, C0,
C1,
/**/ /**/
CB, C2 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
ApplyPackedReflectorsRLHF
( int offset,
const DistMatrix<R>& H,
DistMatrix<R>& A )
{
#ifndef RELEASE
PushCallStack("internal::ApplyPackedReflectorsRLHF");
if( H.Grid() != A.Grid() )
throw std::logic_error("{H,A} must be distributed over the same grid");
if( offset > 0 || offset < -H.Width() )
throw std::logic_error("Transforms out of bounds");
if( H.Width() != A.Width() )
throw std::logic_error
("Width of transforms must equal width of target matrix");
#endif
const Grid& g = H.Grid();
DistMatrix<R>
HTL(g), HTR(g), H00(g), H01(g), H02(g), HPan(g), HPanCopy(g),
HBL(g), HBR(g), H10(g), H11(g), H12(g),
H20(g), H21(g), H22(g);
DistMatrix<R> ALeft(g);
DistMatrix<R,STAR,VR > HPan_STAR_VR(g);
DistMatrix<R,STAR,MR > HPan_STAR_MR(g);
DistMatrix<R,STAR,STAR> SInv_STAR_STAR(g);
DistMatrix<R,STAR,MC > ZTrans_STAR_MC(g);
DistMatrix<R,STAR,VC > ZTrans_STAR_VC(g);
LockedPartitionDownDiagonal
( H, HTL, HTR,
HBL, HBR, 0 );
while( HTL.Height() < H.Height() && HTL.Width() < H.Width() )
{
LockedRepartitionDownDiagonal
( HTL, /**/ HTR, H00, /**/ H01, H02,
/*************/ /******************/
/**/ H10, /**/ H11, H12,
HBL, /**/ HBR, H20, /**/ H21, H22 );
const int HPanWidth = H10.Width() + H11.Width();
const int HPanOffset =
std::min( H11.Height(), std::max(-offset-H00.Height(),0) );
const int HPanHeight = H11.Height()-HPanOffset;
HPan.LockedView( H, H00.Height()+HPanOffset, 0, HPanHeight, HPanWidth );
ALeft.View( A, 0, 0, A.Height(), HPanWidth );
HPan_STAR_MR.AlignWith( ALeft );
ZTrans_STAR_MC.AlignWith( ALeft );
ZTrans_STAR_VC.AlignWith( ALeft );
Zeros( HPan.Height(), ALeft.Height(), ZTrans_STAR_MC );
Zeros( HPan.Height(), HPan.Height(), SInv_STAR_STAR );
//--------------------------------------------------------------------//
HPanCopy = HPan;
MakeTrapezoidal( RIGHT, LOWER, offset, HPanCopy );
SetDiagonalToOne( RIGHT, offset, HPanCopy );
HPan_STAR_VR = HPanCopy;
Syrk
( UPPER, NORMAL,
R(1), HPan_STAR_VR.LockedLocalMatrix(),
R(0), SInv_STAR_STAR.LocalMatrix() );
SInv_STAR_STAR.SumOverGrid();
HalveMainDiagonal( SInv_STAR_STAR );
HPan_STAR_MR = HPan_STAR_VR;
LocalGemm
( NORMAL, TRANSPOSE,
R(1), HPan_STAR_MR, ALeft, R(0), ZTrans_STAR_MC );
ZTrans_STAR_VC.SumScatterFrom( ZTrans_STAR_MC );
LocalTrsm
( LEFT, UPPER, TRANSPOSE, NON_UNIT,
R(1), SInv_STAR_STAR, ZTrans_STAR_VC );
ZTrans_STAR_MC = ZTrans_STAR_VC;
LocalGemm
( TRANSPOSE, NORMAL,
R(-1), ZTrans_STAR_MC, HPan_STAR_MR, R(1), ALeft );
//--------------------------------------------------------------------//
HPan_STAR_MR.FreeAlignments();
ZTrans_STAR_MC.FreeAlignments();
ZTrans_STAR_VC.FreeAlignments();
SlideLockedPartitionDownDiagonal
( HTL, /**/ HTR, H00, H01, /**/ H02,
/**/ H10, H11, /**/ H12,
/*************/ /******************/
HBL, /**/ HBR, H20, H21, /**/ H22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
TrmmRLNCOld
( UnitOrNonUnit diag,
T alpha, const DistMatrix<T>& L,
DistMatrix<T>& X )
{
#ifndef RELEASE
PushCallStack("internal::TrmmRLNCOld");
if( L.Grid() != X.Grid() )
throw std::logic_error
("L and X must be distributed over the same grid");
if( L.Height() != L.Width() || X.Width() != L.Height() )
{
std::ostringstream msg;
msg << "Nonconformal TrmmRLNC: \n"
<< " L ~ " << L.Height() << " x " << L.Width() << "\n"
<< " X ~ " << X.Height() << " x " << X.Width() << "\n";
throw std::logic_error( msg.str().c_str() );
}
#endif
const Grid& g = L.Grid();
// Matrix views
DistMatrix<T>
LTL(g), LTR(g), L00(g), L01(g), L02(g),
LBL(g), LBR(g), L10(g), L11(g), L12(g),
L20(g), L21(g), L22(g);
DistMatrix<T> XL(g), XR(g),
X0(g), X1(g), X2(g);
// Temporary distributions
DistMatrix<T,STAR,STAR> L11_STAR_STAR(g);
DistMatrix<T,MR, STAR> L21_MR_STAR(g);
DistMatrix<T,VC, STAR> X1_VC_STAR(g);
DistMatrix<T,MC, STAR> D1_MC_STAR(g);
// Start the algorithm
Scale( alpha, X );
LockedPartitionDownDiagonal
( L, LTL, LTR,
LBL, LBR, 0 );
PartitionRight( X, XL, XR, 0 );
while( XR.Width() > 0 )
{
LockedRepartitionDownDiagonal
( LTL, /**/ LTR, L00, /**/ L01, L02,
/*************/ /******************/
/**/ L10, /**/ L11, L12,
LBL, /**/ LBR, L20, /**/ L21, L22 );
RepartitionRight
( XL, /**/ XR,
X0, /**/ X1, X2 );
L21_MR_STAR.AlignWith( X2 );
D1_MC_STAR.AlignWith( X1 );
Zeros( X1.Height(), X1.Width(), D1_MC_STAR );
//--------------------------------------------------------------------//
X1_VC_STAR = X1;
L11_STAR_STAR = L11;
LocalTrmm
( RIGHT, LOWER, NORMAL, diag, T(1), L11_STAR_STAR, X1_VC_STAR );
X1 = X1_VC_STAR;
L21_MR_STAR = L21;
LocalGemm( NORMAL, NORMAL, T(1), X2, L21_MR_STAR, T(0), D1_MC_STAR );
X1.SumScatterUpdate( T(1), D1_MC_STAR );
//--------------------------------------------------------------------//
L21_MR_STAR.FreeAlignments();
D1_MC_STAR.FreeAlignments();
SlideLockedPartitionDownDiagonal
( LTL, /**/ LTR, L00, L01, /**/ L02,
/**/ L10, L11, /**/ L12,
/*************/ /******************/
LBL, /**/ LBR, L20, L21, /**/ L22 );
SlidePartitionRight
( XL, /**/ XR,
X0, X1, /**/ X2 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
TwoSidedTrmmLVar4
( UnitOrNonUnit diag, DistMatrix<F>& A, const DistMatrix<F>& L )
{
#ifndef RELEASE
PushCallStack("internal::TwoSidedTrmmLVar4");
if( A.Height() != A.Width() )
throw std::logic_error("A must be square");
if( L.Height() != L.Width() )
throw std::logic_error("Triangular matrices must be square");
if( A.Height() != L.Height() )
throw std::logic_error("A and L must be the same size");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F>
LTL(g), LTR(g), L00(g), L01(g), L02(g),
LBL(g), LBR(g), L10(g), L11(g), L12(g),
L20(g), L21(g), L22(g);
// Temporary distributions
DistMatrix<F,STAR,VR > A10_STAR_VR(g);
DistMatrix<F,STAR,MR > A10_STAR_MR(g);
DistMatrix<F,STAR,MC > A10_STAR_MC(g);
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,VC, STAR> A21_VC_STAR(g);
DistMatrix<F,MC, STAR> A21_MC_STAR(g);
DistMatrix<F,STAR,VR > L10_STAR_VR(g);
DistMatrix<F,MR, STAR> L10Adj_MR_STAR(g);
DistMatrix<F,STAR,MC > L10_STAR_MC(g);
DistMatrix<F,STAR,STAR> L11_STAR_STAR(g);
DistMatrix<F,STAR,VR > Y10_STAR_VR(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( L, LTL, LTR,
LBL, LBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( LTL, /**/ LTR, L00, /**/ L01, L02,
/*************/ /******************/
/**/ L10, /**/ L11, L12,
LBL, /**/ LBR, L20, /**/ L21, L22 );
A10_STAR_VR.AlignWith( A00 );
A10_STAR_MR.AlignWith( A00 );
A10_STAR_MC.AlignWith( A00 );
A21_MC_STAR.AlignWith( A20 );
L10_STAR_VR.AlignWith( A00 );
L10Adj_MR_STAR.AlignWith( A00 );
L10_STAR_MC.AlignWith( A00 );
Y10_STAR_VR.AlignWith( A10 );
//--------------------------------------------------------------------//
// Y10 := A11 L10
A11_STAR_STAR = A11;
L10Adj_MR_STAR.AdjointFrom( L10 );
L10_STAR_VR.AdjointFrom( L10Adj_MR_STAR );
Y10_STAR_VR.ResizeTo( A10.Height(), A10.Width() );
Zero( Y10_STAR_VR );
Hemm
( LEFT, LOWER,
F(1), A11_STAR_STAR.LockedLocalMatrix(),
L10_STAR_VR.LockedLocalMatrix(),
F(0), Y10_STAR_VR.LocalMatrix() );
// A10 := A10 + 1/2 Y10
A10_STAR_VR = A10;
Axpy( F(1)/F(2), Y10_STAR_VR, A10_STAR_VR );
// A00 := A00 + (A10' L10 + L10' A10)
A10_STAR_MR = A10_STAR_VR;
A10_STAR_MC = A10_STAR_VR;
L10_STAR_MC = L10_STAR_VR;
LocalTrr2k
( LOWER, ADJOINT, ADJOINT, ADJOINT,
F(1), A10_STAR_MC, L10Adj_MR_STAR,
L10_STAR_MC, A10_STAR_MR,
F(1), A00 );
// A10 := A10 + 1/2 Y10
Axpy( F(1)/F(2), Y10_STAR_VR, A10_STAR_VR );
// A10 := L11' A10
L11_STAR_STAR = L11;
LocalTrmm
( LEFT, LOWER, ADJOINT, diag, F(1), L11_STAR_STAR, A10_STAR_VR );
A10 = A10_STAR_VR;
// A20 := A20 + A21 L10
A21_MC_STAR = A21;
LocalGemm
( NORMAL, ADJOINT, F(1), A21_MC_STAR, L10Adj_MR_STAR, F(1), A20 );
// A11 := L11' A11 L11
LocalTwoSidedTrmm( LOWER, diag, A11_STAR_STAR, L11_STAR_STAR );
A11 = A11_STAR_STAR;
// A21 := A21 L11
A21_VC_STAR = A21_MC_STAR;
LocalTrmm
( RIGHT, LOWER, NORMAL, diag, F(1), L11_STAR_STAR, A21_VC_STAR );
A21 = A21_VC_STAR;
//--------------------------------------------------------------------//
A10_STAR_VR.FreeAlignments();
A10_STAR_MR.FreeAlignments();
A10_STAR_MC.FreeAlignments();
A21_MC_STAR.FreeAlignments();
L10_STAR_VR.FreeAlignments();
L10Adj_MR_STAR.FreeAlignments();
L10_STAR_MC.FreeAlignments();
Y10_STAR_VR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( LTL, /**/ LTR, L00, L01, /**/ L02,
/**/ L10, L11, /**/ L12,
/*************/ /******************/
LBL, /**/ LBR, L20, L21, /**/ L22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
