inline void
TrdtrmmUVar1( Orientation orientation, DistMatrix<F>& U )
{
#ifndef RELEASE
PushCallStack("internal::TrdtrmmUVar1");
if( U.Height() != U.Width() )
throw std::logic_error("U must be square");
if( orientation == NORMAL )
throw std::logic_error("Orientation must be (conjugate-)transpose");
#endif
const Grid& g = U.Grid();
// Matrix views
DistMatrix<F>
UTL(g), UTR(g), U00(g), U01(g), U02(g),
UBL(g), UBR(g), U10(g), U11(g), U12(g),
U20(g), U21(g), U22(g);
DistMatrix<F,MD,STAR> d1(g);
// Temporary distributions
DistMatrix<F,MC, STAR> S01_MC_STAR(g);
DistMatrix<F,VC, STAR> S01_VC_STAR(g);
DistMatrix<F,VR, STAR> U01_VR_STAR(g);
DistMatrix<F,STAR,MR > U01AdjOrTrans_STAR_MR(g);
DistMatrix<F,STAR,STAR> U11_STAR_STAR(g);
S01_MC_STAR.AlignWith( U );
S01_VC_STAR.AlignWith( U );
U01_VR_STAR.AlignWith( U );
U01AdjOrTrans_STAR_MR.AlignWith( U );
PartitionDownDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
while( UTL.Height() < U.Height() && UTL.Width() < U.Height() )
{
RepartitionDownDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
//--------------------------------------------------------------------//
U11.GetDiagonal( d1 );
S01_MC_STAR = U01;
S01_VC_STAR = S01_MC_STAR;
U01_VR_STAR = S01_VC_STAR;
if( orientation == TRANSPOSE )
{
DiagonalSolve( RIGHT, NORMAL, d1, U01_VR_STAR );
U01AdjOrTrans_STAR_MR.TransposeFrom( U01_VR_STAR );
}
else
{
DiagonalSolve( RIGHT, ADJOINT, d1, U01_VR_STAR );
U01AdjOrTrans_STAR_MR.AdjointFrom( U01_VR_STAR );
}
LocalTrrk( UPPER, F(1), S01_MC_STAR, U01AdjOrTrans_STAR_MR, F(1), U00 );
U11_STAR_STAR = U11;
LocalTrmm
( RIGHT, UPPER, ADJOINT, UNIT, F(1), U11_STAR_STAR, U01_VR_STAR );
U01 = U01_VR_STAR;
LocalTrdtrmm( orientation, UPPER, U11_STAR_STAR );
U11 = U11_STAR_STAR;
//--------------------------------------------------------------------//
d1.FreeAlignments();
SlidePartitionDownDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/*************/ /******************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
TrmmLLTCOld
( Orientation orientation,
UnitOrNonUnit diag,
T alpha,
const DistMatrix<T>& L,
DistMatrix<T>& X )
{
#ifndef RELEASE
PushCallStack("internal::TrmmLLTCOld");
if( L.Grid() != X.Grid() )
throw std::logic_error
("L and X must be distributed over the same grid");
if( orientation == NORMAL )
throw std::logic_error("TrmmLLT expects a (Conjugate)Transpose option");
if( L.Height() != L.Width() || L.Height() != X.Height() )
{
std::ostringstream msg;
msg << "Nonconformal TrmmLLTC: \n"
<< " L ~ " << L.Height() << " x " << L.Width() << "\n"
<< " X ~ " << X.Height() << " x " << X.Width() << "\n";
throw std::logic_error( msg.str().c_str() );
}
#endif
const Grid& g = L.Grid();
// Matrix views
DistMatrix<T>
LTL(g), LTR(g), L00(g), L01(g), L02(g),
LBL(g), LBR(g), L10(g), L11(g), L12(g),
L20(g), L21(g), L22(g);
DistMatrix<T> XT(g), X0(g),
XB(g), X1(g),
X2(g);
// Temporary distributions
DistMatrix<T,STAR,STAR> L11_STAR_STAR(g);
DistMatrix<T,MC, STAR> L21_MC_STAR(g);
DistMatrix<T,STAR,VR > X1_STAR_VR(g);
DistMatrix<T,MR, STAR> D1AdjOrTrans_MR_STAR(g);
DistMatrix<T,MR, MC > D1AdjOrTrans_MR_MC(g);
DistMatrix<T,MC, MR > D1(g);
// Start the algorithm
Scale( alpha, X );
LockedPartitionDownDiagonal
( L, LTL, LTR,
LBL, LBR, 0 );
PartitionDown
( X, XT,
XB, 0 );
while( XB.Height() > 0 )
{
LockedRepartitionDownDiagonal
( LTL, /**/ LTR, L00, /**/ L01, L02,
/*************/ /******************/
/**/ L10, /**/ L11, L12,
LBL, /**/ LBR, L20, /**/ L21, L22 );
RepartitionDown
( XT, X0,
/**/ /**/
X1,
XB, X2 );
L21_MC_STAR.AlignWith( X2 );
D1AdjOrTrans_MR_STAR.AlignWith( X1 );
D1AdjOrTrans_MR_MC.AlignWith( X1 );
D1.AlignWith( X1 );
Zeros( X1.Width(), X1.Height(), D1AdjOrTrans_MR_STAR );
Zeros( X1.Height(), X1.Width(), D1 );
//--------------------------------------------------------------------//
X1_STAR_VR = X1;
L11_STAR_STAR = L11;
LocalTrmm
( LEFT, LOWER, orientation, diag, T(1), L11_STAR_STAR, X1_STAR_VR );
X1 = X1_STAR_VR;
L21_MC_STAR = L21;
LocalGemm
( orientation, NORMAL,
T(1), X2, L21_MC_STAR, T(0), D1AdjOrTrans_MR_STAR );
D1AdjOrTrans_MR_MC.SumScatterFrom( D1AdjOrTrans_MR_STAR );
if( orientation == TRANSPOSE )
Transpose( D1AdjOrTrans_MR_MC.LocalMatrix(), D1.LocalMatrix() );
else
Adjoint( D1AdjOrTrans_MR_MC.LocalMatrix(), D1.LocalMatrix() );
Axpy( T(1), D1, X1 );
//--------------------------------------------------------------------//
D1.FreeAlignments();
D1AdjOrTrans_MR_MC.FreeAlignments();
D1AdjOrTrans_MR_STAR.FreeAlignments();
L21_MC_STAR.FreeAlignments();
SlideLockedPartitionDownDiagonal
( LTL, /**/ LTR, L00, L01, /**/ L02,
/**/ L10, L11, /**/ L12,
/*************/ /******************/
LBL, /**/ LBR, L20, L21, /**/ L22 );
SlidePartitionDown
( XT, X0,
X1,
/**/ /**/
XB, X2 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
TwoSidedTrmmUVar5
( UnitOrNonUnit diag, DistMatrix<F>& A, const DistMatrix<F>& U )
{
#ifndef RELEASE
PushCallStack("internal::TwoSidedTrmmUVar5");
if( A.Height() != A.Width() )
throw std::logic_error("A must be square");
if( U.Height() != U.Width() )
throw std::logic_error("Triangular matrices must be square");
if( A.Height() != U.Height() )
throw std::logic_error("A and U must be the same size");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F>
UTL(g), UTR(g), U00(g), U01(g), U02(g),
UBL(g), UBR(g), U10(g), U11(g), U12(g),
U20(g), U21(g), U22(g);
// Temporary distributions
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,MC, STAR> A01_MC_STAR(g);
DistMatrix<F,MR, STAR> A01_MR_STAR(g);
DistMatrix<F,VC, STAR> A01_VC_STAR(g);
DistMatrix<F,STAR,STAR> U11_STAR_STAR(g);
DistMatrix<F,MC, STAR> U01_MC_STAR(g);
DistMatrix<F,MR, STAR> U01_MR_STAR(g);
DistMatrix<F,VC, STAR> U01_VC_STAR(g);
DistMatrix<F,VC, STAR> Y01_VC_STAR(g);
DistMatrix<F> Y01(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
A01_MC_STAR.AlignWith( A00 );
A01_MR_STAR.AlignWith( A00 );
A01_VC_STAR.AlignWith( A00 );
U01_MC_STAR.AlignWith( A00 );
U01_MR_STAR.AlignWith( A00 );
U01_VC_STAR.AlignWith( A00 );
Y01.AlignWith( A01 );
Y01_VC_STAR.AlignWith( A01 );
//--------------------------------------------------------------------//
// Y01 := U01 A11
A11_STAR_STAR = A11;
U01_VC_STAR = U01;
Y01_VC_STAR.ResizeTo( A01.Height(), A01.Width() );
Hemm
( RIGHT, UPPER,
F(1), A11_STAR_STAR.LocalMatrix(), U01_VC_STAR.LocalMatrix(),
F(0), Y01_VC_STAR.LocalMatrix() );
Y01 = Y01_VC_STAR;
// A01 := U00 A01
Trmm( LEFT, UPPER, NORMAL, diag, F(1), U00, A01 );
// A01 := A01 + 1/2 Y01
Axpy( F(1)/F(2), Y01, A01 );
// A00 := A00 + (U01 A01' + A01 U01')
A01_MC_STAR = A01;
U01_MC_STAR = U01;
A01_VC_STAR = A01_MC_STAR;
A01_MR_STAR = A01_VC_STAR;
U01_MR_STAR = U01_MC_STAR;
LocalTrr2k
( UPPER, ADJOINT, ADJOINT,
F(1), U01_MC_STAR, A01_MR_STAR,
A01_MC_STAR, U01_MR_STAR,
F(1), A00 );
// A01 := A01 + 1/2 Y01
Axpy( F(1)/F(2), Y01_VC_STAR, A01_VC_STAR );
// A01 := A01 U11'
U11_STAR_STAR = U11;
LocalTrmm
( RIGHT, UPPER, ADJOINT, diag, F(1), U11_STAR_STAR, A01_VC_STAR );
A01 = A01_VC_STAR;
// A11 := U11 A11 U11'
LocalTwoSidedTrmm( UPPER, diag, A11_STAR_STAR, U11_STAR_STAR );
A11 = A11_STAR_STAR;
//--------------------------------------------------------------------//
A01_MC_STAR.FreeAlignments();
A01_MR_STAR.FreeAlignments();
A01_VC_STAR.FreeAlignments();
U01_MC_STAR.FreeAlignments();
U01_MR_STAR.FreeAlignments();
U01_VC_STAR.FreeAlignments();
Y01.FreeAlignments();
Y01_VC_STAR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/*************/ /******************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
TrmmLLNC
( UnitOrNonUnit diag,
T alpha, const DistMatrix<T>& L,
DistMatrix<T>& X )
{
#ifndef RELEASE
CallStackEntry entry("internal::TrmmLLNC");
if( L.Grid() != X.Grid() )
throw std::logic_error
("L and X must be distributed over the same grid");
if( L.Height() != L.Width() || L.Width() != X.Height() )
{
std::ostringstream msg;
msg << "Nonconformal TrmmLLNC: \n"
<< " L ~ " << L.Height() << " x " << L.Width() << "\n"
<< " X ~ " << X.Height() << " x " << X.Width() << "\n";
throw std::logic_error( msg.str().c_str() );
}
#endif
const Grid& g = L.Grid();
// Matrix views
DistMatrix<T>
LTL(g), LTR(g), L00(g), L01(g), L02(g),
LBL(g), LBR(g), L10(g), L11(g), L12(g),
L20(g), L21(g), L22(g);
DistMatrix<T> XT(g), X0(g),
XB(g), X1(g),
X2(g);
// Temporary distributions
DistMatrix<T,MC, STAR> L21_MC_STAR(g);
DistMatrix<T,STAR,STAR> L11_STAR_STAR(g);
DistMatrix<T,STAR,VR > X1_STAR_VR(g);
DistMatrix<T,MR, STAR> X1Trans_MR_STAR(g);
// Start the algorithm
Scale( alpha, X );
LockedPartitionUpDiagonal
( L, LTL, LTR,
LBL, LBR, 0 );
PartitionUp
( X, XT,
XB, 0 );
while( XT.Height() > 0 )
{
LockedRepartitionUpDiagonal
( LTL, /**/ LTR, L00, L01, /**/ L02,
/**/ L10, L11, /**/ L12,
/*************/ /******************/
LBL, /**/ LBR, L20, L21, /**/ L22 );
RepartitionUp
( XT, X0,
X1,
/**/ /**/
XB, X2 );
L21_MC_STAR.AlignWith( X2 );
X1Trans_MR_STAR.AlignWith( X2 );
X1_STAR_VR.AlignWith( X1 );
//--------------------------------------------------------------------//
L21_MC_STAR = L21;
X1Trans_MR_STAR.TransposeFrom( X1 );
LocalGemm
( NORMAL, TRANSPOSE, T(1), L21_MC_STAR, X1Trans_MR_STAR, T(1), X2 );
L11_STAR_STAR = L11;
X1_STAR_VR.TransposeFrom( X1Trans_MR_STAR );
LocalTrmm( LEFT, LOWER, NORMAL, diag, T(1), L11_STAR_STAR, X1_STAR_VR );
X1 = X1_STAR_VR;
//--------------------------------------------------------------------//
L21_MC_STAR.FreeAlignments();
X1Trans_MR_STAR.FreeAlignments();
X1_STAR_VR.FreeAlignments();
SlideLockedPartitionUpDiagonal
( LTL, /**/ LTR, L00, /**/ L01, L02,
/*************/ /******************/
/**/ L10, /**/ L11, L12,
LBL, /**/ LBR, L20, /**/ L21, L22 );
SlidePartitionUp
( XT, X0,
/**/ /**/
X1,
XB, X2 );
}
}
inline void
TrmmRLNCOld
( UnitOrNonUnit diag,
T alpha, const DistMatrix<T>& L,
DistMatrix<T>& X )
{
#ifndef RELEASE
PushCallStack("internal::TrmmRLNCOld");
if( L.Grid() != X.Grid() )
throw std::logic_error
("L and X must be distributed over the same grid");
if( L.Height() != L.Width() || X.Width() != L.Height() )
{
std::ostringstream msg;
msg << "Nonconformal TrmmRLNC: \n"
<< " L ~ " << L.Height() << " x " << L.Width() << "\n"
<< " X ~ " << X.Height() << " x " << X.Width() << "\n";
throw std::logic_error( msg.str().c_str() );
}
#endif
const Grid& g = L.Grid();
// Matrix views
DistMatrix<T>
LTL(g), LTR(g), L00(g), L01(g), L02(g),
LBL(g), LBR(g), L10(g), L11(g), L12(g),
L20(g), L21(g), L22(g);
DistMatrix<T> XL(g), XR(g),
X0(g), X1(g), X2(g);
// Temporary distributions
DistMatrix<T,STAR,STAR> L11_STAR_STAR(g);
DistMatrix<T,MR, STAR> L21_MR_STAR(g);
DistMatrix<T,VC, STAR> X1_VC_STAR(g);
DistMatrix<T,MC, STAR> D1_MC_STAR(g);
// Start the algorithm
Scale( alpha, X );
LockedPartitionDownDiagonal
( L, LTL, LTR,
LBL, LBR, 0 );
PartitionRight( X, XL, XR, 0 );
while( XR.Width() > 0 )
{
LockedRepartitionDownDiagonal
( LTL, /**/ LTR, L00, /**/ L01, L02,
/*************/ /******************/
/**/ L10, /**/ L11, L12,
LBL, /**/ LBR, L20, /**/ L21, L22 );
RepartitionRight
( XL, /**/ XR,
X0, /**/ X1, X2 );
L21_MR_STAR.AlignWith( X2 );
D1_MC_STAR.AlignWith( X1 );
Zeros( X1.Height(), X1.Width(), D1_MC_STAR );
//--------------------------------------------------------------------//
X1_VC_STAR = X1;
L11_STAR_STAR = L11;
LocalTrmm
( RIGHT, LOWER, NORMAL, diag, T(1), L11_STAR_STAR, X1_VC_STAR );
X1 = X1_VC_STAR;
L21_MR_STAR = L21;
LocalGemm( NORMAL, NORMAL, T(1), X2, L21_MR_STAR, T(0), D1_MC_STAR );
X1.SumScatterUpdate( T(1), D1_MC_STAR );
//--------------------------------------------------------------------//
L21_MR_STAR.FreeAlignments();
D1_MC_STAR.FreeAlignments();
SlideLockedPartitionDownDiagonal
( LTL, /**/ LTR, L00, L01, /**/ L02,
/**/ L10, L11, /**/ L12,
/*************/ /******************/
LBL, /**/ LBR, L20, L21, /**/ L22 );
SlidePartitionRight
( XL, /**/ XR,
X0, X1, /**/ X2 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
TwoSidedTrmmLVar4
( UnitOrNonUnit diag, DistMatrix<F>& A, const DistMatrix<F>& L )
{
#ifndef RELEASE
CallStackEntry entry("internal::TwoSidedTrmmLVar4");
if( A.Height() != A.Width() )
LogicError("A must be square");
if( L.Height() != L.Width() )
LogicError("Triangular matrices must be square");
if( A.Height() != L.Height() )
LogicError("A and L must be the same size");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F>
LTL(g), LTR(g), L00(g), L01(g), L02(g),
LBL(g), LBR(g), L10(g), L11(g), L12(g),
L20(g), L21(g), L22(g);
// Temporary distributions
DistMatrix<F,STAR,VR > A10_STAR_VR(g);
DistMatrix<F,STAR,MR > A10_STAR_MR(g);
DistMatrix<F,STAR,MC > A10_STAR_MC(g);
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,VC, STAR> A21_VC_STAR(g);
DistMatrix<F,MC, STAR> A21_MC_STAR(g);
DistMatrix<F,STAR,VR > L10_STAR_VR(g);
DistMatrix<F,MR, STAR> L10Adj_MR_STAR(g);
DistMatrix<F,STAR,MC > L10_STAR_MC(g);
DistMatrix<F,STAR,STAR> L11_STAR_STAR(g);
DistMatrix<F,STAR,VR > Y10_STAR_VR(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( L, LTL, LTR,
LBL, LBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( LTL, /**/ LTR, L00, /**/ L01, L02,
/*************/ /******************/
/**/ L10, /**/ L11, L12,
LBL, /**/ LBR, L20, /**/ L21, L22 );
A10_STAR_VR.AlignWith( A00 );
A10_STAR_MR.AlignWith( A00 );
A10_STAR_MC.AlignWith( A00 );
A21_MC_STAR.AlignWith( A20 );
L10_STAR_VR.AlignWith( A00 );
L10Adj_MR_STAR.AlignWith( A00 );
L10_STAR_MC.AlignWith( A00 );
Y10_STAR_VR.AlignWith( A10 );
//--------------------------------------------------------------------//
// Y10 := A11 L10
A11_STAR_STAR = A11;
L10Adj_MR_STAR.AdjointFrom( L10 );
L10_STAR_VR.AdjointFrom( L10Adj_MR_STAR );
Zeros( Y10_STAR_VR, A10.Height(), A10.Width() );
Hemm
( LEFT, LOWER,
F(1), A11_STAR_STAR.LockedMatrix(), L10_STAR_VR.LockedMatrix(),
F(0), Y10_STAR_VR.Matrix() );
// A10 := A10 + 1/2 Y10
A10_STAR_VR = A10;
Axpy( F(1)/F(2), Y10_STAR_VR, A10_STAR_VR );
// A00 := A00 + (A10' L10 + L10' A10)
A10_STAR_MR = A10_STAR_VR;
A10_STAR_MC = A10_STAR_VR;
L10_STAR_MC = L10_STAR_VR;
LocalTrr2k
( LOWER, ADJOINT, ADJOINT, ADJOINT,
F(1), A10_STAR_MC, L10Adj_MR_STAR,
L10_STAR_MC, A10_STAR_MR,
F(1), A00 );
// A10 := A10 + 1/2 Y10
Axpy( F(1)/F(2), Y10_STAR_VR, A10_STAR_VR );
// A10 := L11' A10
L11_STAR_STAR = L11;
LocalTrmm
( LEFT, LOWER, ADJOINT, diag, F(1), L11_STAR_STAR, A10_STAR_VR );
A10 = A10_STAR_VR;
// A20 := A20 + A21 L10
A21_MC_STAR = A21;
LocalGemm
( NORMAL, ADJOINT, F(1), A21_MC_STAR, L10Adj_MR_STAR, F(1), A20 );
// A11 := L11' A11 L11
LocalTwoSidedTrmm( LOWER, diag, A11_STAR_STAR, L11_STAR_STAR );
A11 = A11_STAR_STAR;
// A21 := A21 L11
A21_VC_STAR = A21_MC_STAR;
LocalTrmm
( RIGHT, LOWER, NORMAL, diag, F(1), L11_STAR_STAR, A21_VC_STAR );
A21 = A21_VC_STAR;
//--------------------------------------------------------------------//
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( LTL, /**/ LTR, L00, L01, /**/ L02,
/**/ L10, L11, /**/ L12,
/*************/ /******************/
LBL, /**/ LBR, L20, L21, /**/ L22 );
}
}
inline void
TrmmRUNC
( UnitOrNonUnit diag,
T alpha, const DistMatrix<T>& U,
DistMatrix<T>& X )
{
#ifndef RELEASE
CallStackEntry entry("internal::TrmmRUNC");
if( U.Grid() != X.Grid() )
throw std::logic_error
("U and X must be distributed over the same grid");
if( U.Height() != U.Width() || X.Width() != U.Height() )
{
std::ostringstream msg;
msg << "Nonconformal TrmmRUNC: \n"
<< " U ~ " << U.Height() << " x " << U.Width() << "\n"
<< " X ~ " << X.Height() << " x " << X.Width() << "\n";
throw std::logic_error( msg.str().c_str() );
}
#endif
const Grid& g = U.Grid();
// Matrix views
DistMatrix<T>
UTL(g), UTR(g), U00(g), U01(g), U02(g),
UBL(g), UBR(g), U10(g), U11(g), U12(g),
U20(g), U21(g), U22(g);
DistMatrix<T> XL(g), XR(g),
X0(g), X1(g), X2(g);
// Temporary distributions
DistMatrix<T,MR, STAR> U12Trans_MR_STAR(g);
DistMatrix<T,STAR,STAR> U11_STAR_STAR(g);
DistMatrix<T,VC, STAR> X1_VC_STAR(g);
DistMatrix<T,MC, STAR> X1_MC_STAR(g);
// Start the algorithm
Scale( alpha, X );
LockedPartitionUpDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
PartitionLeft( X, XL, XR, 0 );
while( XL.Width() > 0 )
{
LockedRepartitionUpDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/*************/ /******************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
RepartitionLeft
( XL, /**/ XR,
X0, X1, /**/ X2 );
X1_MC_STAR.AlignWith( X2 );
U12Trans_MR_STAR.AlignWith( X2 );
X1_VC_STAR.AlignWith( X1 );
//--------------------------------------------------------------------//
X1_MC_STAR = X1;
U12Trans_MR_STAR.TransposeFrom( U12 );
LocalGemm
( NORMAL, TRANSPOSE, T(1), X1_MC_STAR, U12Trans_MR_STAR, T(1), X2 );
U11_STAR_STAR = U11;
X1_VC_STAR = X1_MC_STAR;
LocalTrmm
( RIGHT, UPPER, NORMAL, diag, T(1), U11_STAR_STAR, X1_VC_STAR );
X1 = X1_VC_STAR;
//--------------------------------------------------------------------//
X1_MC_STAR.FreeAlignments();
U12Trans_MR_STAR.FreeAlignments();
X1_VC_STAR.FreeAlignments();
SlideLockedPartitionUpDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
SlidePartitionLeft
( XL, /**/ XR,
X0, /**/ X1, X2 );
}
}
inline void
TwoSidedTrmmLVar2
( UnitOrNonUnit diag, DistMatrix<F>& A, const DistMatrix<F>& L )
{
#ifndef RELEASE
PushCallStack("internal::TwoSidedTrmmLVar2");
if( A.Height() != A.Width() )
throw std::logic_error( "A must be square." );
if( L.Height() != L.Width() )
throw std::logic_error( "Triangular matrices must be square." );
if( A.Height() != L.Height() )
throw std::logic_error( "A and L must be the same size." );
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F>
LTL(g), LTR(g), L00(g), L01(g), L02(g),
LBL(g), LBR(g), L10(g), L11(g), L12(g),
L20(g), L21(g), L22(g);
// Temporary distributions
DistMatrix<F,STAR,VR > A10_STAR_VR(g);
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,VC, STAR> A21_VC_STAR(g);
DistMatrix<F,STAR,STAR> L11_STAR_STAR(g);
DistMatrix<F,MC, STAR> L21_MC_STAR(g);
DistMatrix<F,STAR,MR > L21Adj_STAR_MR(g);
DistMatrix<F,VC, STAR> L21_VC_STAR(g);
DistMatrix<F,VR, STAR> L21_VR_STAR(g);
DistMatrix<F,STAR,MR > X10_STAR_MR(g);
DistMatrix<F,STAR,STAR> X11_STAR_STAR(g);
DistMatrix<F,MC, STAR> Z21_MC_STAR(g);
DistMatrix<F,MR, STAR> Z21_MR_STAR(g);
DistMatrix<F,MR, MC > Z21_MR_MC(g);
DistMatrix<F> Y21(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( L, LTL, LTR,
LBL, LBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( LTL, /**/ LTR, L00, /**/ L01, L02,
/*************/ /******************/
/**/ L10, /**/ L11, L12,
LBL, /**/ LBR, L20, /**/ L21, L22 );
A21_VC_STAR.AlignWith( A22 );
L21_MC_STAR.AlignWith( A20 );
L21_VC_STAR.AlignWith( A22 );
L21_VR_STAR.AlignWith( A22 );
L21Adj_STAR_MR.AlignWith( A22 );
X10_STAR_MR.AlignWith( A10 );
Y21.AlignWith( A21 );
Z21_MC_STAR.AlignWith( A22 );
Z21_MR_STAR.AlignWith( A22 );
//--------------------------------------------------------------------//
// A10 := L11' A10
L11_STAR_STAR = L11;
A10_STAR_VR = A10;
LocalTrmm
( LEFT, LOWER, ADJOINT, diag, F(1), L11_STAR_STAR, A10_STAR_VR );
A10 = A10_STAR_VR;
// A10 := A10 + L21' A20
L21_MC_STAR = L21;
X10_STAR_MR.ResizeTo( A10.Height(), A10.Width() );
LocalGemm( ADJOINT, NORMAL, F(1), L21_MC_STAR, A20, F(0), X10_STAR_MR );
A10.SumScatterUpdate( F(1), X10_STAR_MR );
// Y21 := A22 L21
L21_VC_STAR = L21_MC_STAR;
L21_VR_STAR = L21_VC_STAR;
L21Adj_STAR_MR.AdjointFrom( L21_VR_STAR );
Z21_MC_STAR.ResizeTo( A21.Height(), A21.Width() );
Z21_MR_STAR.ResizeTo( A21.Height(), A21.Width() );
Zero( Z21_MC_STAR );
Zero( Z21_MR_STAR );
LocalSymmetricAccumulateLL
( ADJOINT,
F(1), A22, L21_MC_STAR, L21Adj_STAR_MR, Z21_MC_STAR, Z21_MR_STAR );
Z21_MR_MC.SumScatterFrom( Z21_MR_STAR );
Y21 = Z21_MR_MC;
Y21.SumScatterUpdate( F(1), Z21_MC_STAR );
// A21 := A21 L11
A21_VC_STAR = A21;
LocalTrmm
( RIGHT, LOWER, NORMAL, diag, F(1), L11_STAR_STAR, A21_VC_STAR );
A21 = A21_VC_STAR;
// A21 := A21 + 1/2 Y21
Axpy( F(1)/F(2), Y21, A21 );
// A11 := L11' A11 L11
A11_STAR_STAR = A11;
LocalTwoSidedTrmm( LOWER, diag, A11_STAR_STAR, L11_STAR_STAR );
A11 = A11_STAR_STAR;
// A11 := A11 + (A21' L21 + L21' A21)
A21_VC_STAR = A21;
X11_STAR_STAR.ResizeTo( A11.Height(), A11.Width() );
Her2k
( LOWER, ADJOINT,
F(1), A21_VC_STAR.LocalMatrix(), L21_VC_STAR.LocalMatrix(),
F(0), X11_STAR_STAR.LocalMatrix() );
A11.SumScatterUpdate( F(1), X11_STAR_STAR );
// A21 := A21 + 1/2 Y21
Axpy( F(1)/F(2), Y21, A21 );
//--------------------------------------------------------------------//
A21_VC_STAR.FreeAlignments();
L21_MC_STAR.FreeAlignments();
L21_VC_STAR.FreeAlignments();
L21_VR_STAR.FreeAlignments();
L21Adj_STAR_MR.FreeAlignments();
X10_STAR_MR.FreeAlignments();
Y21.FreeAlignments();
Z21_MC_STAR.FreeAlignments();
Z21_MR_STAR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( LTL, /**/ LTR, L00, L01, /**/ L02,
/**/ L10, L11, /**/ L12,
/*************/ /******************/
LBL, /**/ LBR, L20, L21, /**/ L22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
TwoSidedTrmmUVar1
( UnitOrNonUnit diag, DistMatrix<F>& A, const DistMatrix<F>& U )
{
#ifndef RELEASE
PushCallStack("internal::TwoSidedTrmmUVar1");
if( A.Height() != A.Width() )
throw std::logic_error("A must be square");
if( U.Height() != U.Width() )
throw std::logic_error("Triangular matrices must be square");
if( A.Height() != U.Height() )
throw std::logic_error("A and U must be the same size");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F>
UTL(g), UTR(g), U00(g), U01(g), U02(g),
UBL(g), UBR(g), U10(g), U11(g), U12(g),
U20(g), U21(g), U22(g);
// Temporary distributions
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,STAR,VR > A12_STAR_VR(g);
DistMatrix<F,STAR,STAR> U11_STAR_STAR(g);
DistMatrix<F,STAR,MC > U12_STAR_MC(g);
DistMatrix<F,STAR,VR > U12_STAR_VR(g);
DistMatrix<F,MR, STAR> U12Adj_MR_STAR(g);
DistMatrix<F,VC, STAR> U12Adj_VC_STAR(g);
DistMatrix<F,STAR,STAR> X11_STAR_STAR(g);
DistMatrix<F,MR, MC > Z12Adj_MR_MC(g);
DistMatrix<F,MC, STAR> Z12Adj_MC_STAR(g);
DistMatrix<F,MR, STAR> Z12Adj_MR_STAR(g);
DistMatrix<F> Z12Adj(g);
DistMatrix<F> Y12(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
A12_STAR_VR.AlignWith( A12 );
U12_STAR_MC.AlignWith( A22 );
U12_STAR_VR.AlignWith( A12 );
U12Adj_MR_STAR.AlignWith( A22 );
U12Adj_VC_STAR.AlignWith( A22 );
X11_STAR_STAR.ResizeTo( A11.Height(), A11.Width() );
Y12.AlignWith( A12 );
Z12Adj.AlignWith( A12 );
Z12Adj_MR_MC.AlignWith( A12 );
Z12Adj_MC_STAR.AlignWith( A22 );
Z12Adj_MR_STAR.AlignWith( A22 );
//--------------------------------------------------------------------//
// Y12 := U12 A22
U12Adj_MR_STAR.AdjointFrom( U12 );
U12Adj_VC_STAR = U12Adj_MR_STAR;
U12_STAR_MC.AdjointFrom( U12Adj_VC_STAR );
Z12Adj_MC_STAR.ResizeTo( A12.Width(), A12.Height() );
Z12Adj_MR_STAR.ResizeTo( A12.Width(), A12.Height() );
Zero( Z12Adj_MC_STAR );
Zero( Z12Adj_MR_STAR );
LocalSymmetricAccumulateRU
( ADJOINT,
F(1), A22, U12_STAR_MC, U12Adj_MR_STAR,
Z12Adj_MC_STAR, Z12Adj_MR_STAR );
Z12Adj.SumScatterFrom( Z12Adj_MC_STAR );
Z12Adj_MR_MC = Z12Adj;
Z12Adj_MR_MC.SumScatterUpdate( F(1), Z12Adj_MR_STAR );
Y12.ResizeTo( A12.Height(), A12.Width() );
Adjoint( Z12Adj_MR_MC.LockedLocalMatrix(), Y12.LocalMatrix() );
// A12 := U11 A12
A12_STAR_VR = A12;
U11_STAR_STAR = U11;
LocalTrmm
( LEFT, UPPER, NORMAL, diag, F(1), U11_STAR_STAR, A12_STAR_VR );
A12 = A12_STAR_VR;
// A12 := A12 + 1/2 Y12
Axpy( F(1)/F(2), Y12, A12 );
// A11 := U11 A11 U11'
A11_STAR_STAR = A11;
LocalTwoSidedTrmm( UPPER, diag, A11_STAR_STAR, U11_STAR_STAR );
A11 = A11_STAR_STAR;
// A11 := A11 + (U12 A12' + A12 U12')
A12_STAR_VR = A12;
U12_STAR_VR = U12;
Her2k
( UPPER, NORMAL,
F(1), A12_STAR_VR.LocalMatrix(), U12_STAR_VR.LocalMatrix(),
F(0), X11_STAR_STAR.LocalMatrix() );
A11.SumScatterUpdate( F(1), X11_STAR_STAR );
// A12 := A12 + 1/2 Y12
Axpy( F(1)/F(2), Y12, A12 );
// A12 := A12 U22'
Trmm( RIGHT, UPPER, ADJOINT, diag, F(1), U22, A12 );
//--------------------------------------------------------------------//
A12_STAR_VR.FreeAlignments();
U12_STAR_MC.FreeAlignments();
U12_STAR_VR.FreeAlignments();
U12Adj_MR_STAR.FreeAlignments();
U12Adj_VC_STAR.FreeAlignments();
Y12.FreeAlignments();
Z12Adj.FreeAlignments();
Z12Adj_MR_MC.FreeAlignments();
Z12Adj_MC_STAR.FreeAlignments();
Z12Adj_MR_STAR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/*************/ /******************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
TrtrmmUVar1( Orientation orientation, DistMatrix<T>& U )
{
#ifndef RELEASE
PushCallStack("internal::TrtrmmUVar1");
if( U.Height() != U.Width() )
throw std::logic_error("U must be square");
#endif
const Grid& g = U.Grid();
// Matrix views
DistMatrix<T>
UTL(g), UTR(g), U00(g), U01(g), U02(g),
UBL(g), UBR(g), U10(g), U11(g), U12(g),
U20(g), U21(g), U22(g);
// Temporary distributions
DistMatrix<T,MC, STAR> U01_MC_STAR(g);
DistMatrix<T,VC, STAR> U01_VC_STAR(g);
DistMatrix<T,VR, STAR> U01_VR_STAR(g);
DistMatrix<T,STAR,MR > U01AdjOrTrans_STAR_MR(g);
DistMatrix<T,STAR,STAR> U11_STAR_STAR(g);
U01_MC_STAR.AlignWith( U );
U01_VC_STAR.AlignWith( U );
U01_VR_STAR.AlignWith( U );
U01AdjOrTrans_STAR_MR.AlignWith( U );
PartitionDownDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
while( UTL.Height() < U.Height() && UTL.Width() < U.Height() )
{
RepartitionDownDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
//--------------------------------------------------------------------//
U01_MC_STAR = U01;
U01_VC_STAR = U01_MC_STAR;
U01_VR_STAR = U01_VC_STAR;
if( orientation == ADJOINT )
U01AdjOrTrans_STAR_MR.AdjointFrom( U01_VR_STAR );
else
U01AdjOrTrans_STAR_MR.TransposeFrom( U01_VR_STAR );
LocalTrrk( UPPER, T(1), U01_MC_STAR, U01AdjOrTrans_STAR_MR, T(1), U00 );
U11_STAR_STAR = U11;
LocalTrmm
( RIGHT, UPPER, orientation, NON_UNIT,
T(1), U11_STAR_STAR, U01_VC_STAR );
U01 = U01_VC_STAR;
LocalTrtrmm( orientation, UPPER, U11_STAR_STAR );
U11 = U11_STAR_STAR;
//--------------------------------------------------------------------//
SlidePartitionDownDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/*************/ /******************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
TrdtrmmLVar1( Orientation orientation, DistMatrix<F>& L )
{
#ifndef RELEASE
CallStackEntry entry("internal::TrdtrmmLVar1");
if( L.Height() != L.Width() )
LogicError("L must be square");
if( orientation == NORMAL )
LogicError("Orientation must be (conjugate-)transpose");
#endif
const Grid& g = L.Grid();
// Matrix views
DistMatrix<F>
LTL(g), LTR(g), L00(g), L01(g), L02(g),
LBL(g), LBR(g), L10(g), L11(g), L12(g),
L20(g), L21(g), L22(g);
DistMatrix<F,MD,STAR> d1(g);
// Temporary distributions
DistMatrix<F,STAR,VR > L10_STAR_VR(g);
DistMatrix<F,STAR,VC > S10_STAR_VC(g);
DistMatrix<F,STAR,MC > S10_STAR_MC(g);
DistMatrix<F,STAR,MR > L10_STAR_MR(g);
DistMatrix<F,STAR,STAR> L11_STAR_STAR(g);
L10_STAR_VR.AlignWith( L );
S10_STAR_VC.AlignWith( L );
S10_STAR_MC.AlignWith( L );
L10_STAR_MR.AlignWith( L );
PartitionDownDiagonal
( L, LTL, LTR,
LBL, LBR, 0 );
while( LTL.Height() < L.Height() && LTL.Width() < L.Height() )
{
RepartitionDownDiagonal
( LTL, /**/ LTR, L00, /**/ L01, L02,
/*************/ /******************/
/**/ L10, /**/ L11, L12,
LBL, /**/ LBR, L20, /**/ L21, L22 );
//--------------------------------------------------------------------//
L11.GetDiagonal( d1 );
L10_STAR_VR = L10;
S10_STAR_VC = L10_STAR_VR;
S10_STAR_MC = S10_STAR_VC;
DiagonalSolve( LEFT, NORMAL, d1, L10_STAR_VR, true );
L10_STAR_MR = L10_STAR_VR;
LocalTrrk
( LOWER, orientation, F(1), S10_STAR_MC, L10_STAR_MR, F(1), L00 );
L11_STAR_STAR = L11;
LocalTrmm
( LEFT, LOWER, orientation, UNIT, F(1), L11_STAR_STAR, L10_STAR_VR );
L10 = L10_STAR_VR;
LocalTrdtrmm( orientation, LOWER, L11_STAR_STAR );
L11 = L11_STAR_STAR;
//--------------------------------------------------------------------//
SlidePartitionDownDiagonal
( LTL, /**/ LTR, L00, L01, /**/ L02,
/**/ L10, L11, /**/ L12,
/*************/ /******************/
LBL, /**/ LBR, L20, L21, /**/ L22 );
}
}
