inline void
TwoSidedTrmmUVar5
( UnitOrNonUnit diag, DistMatrix<F>& A, const DistMatrix<F>& U )
{
#ifndef RELEASE
PushCallStack("internal::TwoSidedTrmmUVar5");
if( A.Height() != A.Width() )
throw std::logic_error("A must be square");
if( U.Height() != U.Width() )
throw std::logic_error("Triangular matrices must be square");
if( A.Height() != U.Height() )
throw std::logic_error("A and U must be the same size");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F>
UTL(g), UTR(g), U00(g), U01(g), U02(g),
UBL(g), UBR(g), U10(g), U11(g), U12(g),
U20(g), U21(g), U22(g);
// Temporary distributions
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,MC, STAR> A01_MC_STAR(g);
DistMatrix<F,MR, STAR> A01_MR_STAR(g);
DistMatrix<F,VC, STAR> A01_VC_STAR(g);
DistMatrix<F,STAR,STAR> U11_STAR_STAR(g);
DistMatrix<F,MC, STAR> U01_MC_STAR(g);
DistMatrix<F,MR, STAR> U01_MR_STAR(g);
DistMatrix<F,VC, STAR> U01_VC_STAR(g);
DistMatrix<F,VC, STAR> Y01_VC_STAR(g);
DistMatrix<F> Y01(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
A01_MC_STAR.AlignWith( A00 );
A01_MR_STAR.AlignWith( A00 );
A01_VC_STAR.AlignWith( A00 );
U01_MC_STAR.AlignWith( A00 );
U01_MR_STAR.AlignWith( A00 );
U01_VC_STAR.AlignWith( A00 );
Y01.AlignWith( A01 );
Y01_VC_STAR.AlignWith( A01 );
//--------------------------------------------------------------------//
// Y01 := U01 A11
A11_STAR_STAR = A11;
U01_VC_STAR = U01;
Y01_VC_STAR.ResizeTo( A01.Height(), A01.Width() );
Hemm
( RIGHT, UPPER,
F(1), A11_STAR_STAR.LocalMatrix(), U01_VC_STAR.LocalMatrix(),
F(0), Y01_VC_STAR.LocalMatrix() );
Y01 = Y01_VC_STAR;
// A01 := U00 A01
Trmm( LEFT, UPPER, NORMAL, diag, F(1), U00, A01 );
// A01 := A01 + 1/2 Y01
Axpy( F(1)/F(2), Y01, A01 );
// A00 := A00 + (U01 A01' + A01 U01')
A01_MC_STAR = A01;
U01_MC_STAR = U01;
A01_VC_STAR = A01_MC_STAR;
A01_MR_STAR = A01_VC_STAR;
U01_MR_STAR = U01_MC_STAR;
LocalTrr2k
( UPPER, ADJOINT, ADJOINT,
F(1), U01_MC_STAR, A01_MR_STAR,
A01_MC_STAR, U01_MR_STAR,
F(1), A00 );
// A01 := A01 + 1/2 Y01
Axpy( F(1)/F(2), Y01_VC_STAR, A01_VC_STAR );
// A01 := A01 U11'
U11_STAR_STAR = U11;
LocalTrmm
( RIGHT, UPPER, ADJOINT, diag, F(1), U11_STAR_STAR, A01_VC_STAR );
A01 = A01_VC_STAR;
// A11 := U11 A11 U11'
LocalTwoSidedTrmm( UPPER, diag, A11_STAR_STAR, U11_STAR_STAR );
A11 = A11_STAR_STAR;
//--------------------------------------------------------------------//
A01_MC_STAR.FreeAlignments();
A01_MR_STAR.FreeAlignments();
A01_VC_STAR.FreeAlignments();
U01_MC_STAR.FreeAlignments();
U01_MR_STAR.FreeAlignments();
U01_VC_STAR.FreeAlignments();
Y01.FreeAlignments();
Y01_VC_STAR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/*************/ /******************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
internal::HermitianTridiagU
( DistMatrix<Complex<R>,MC, MR >& A,
DistMatrix<Complex<R>,STAR,STAR>& t )
{
#ifndef RELEASE
PushCallStack("internal::HermitianTridiagU");
if( A.Grid() != t.Grid() )
throw std::logic_error("{A,t} must be distributed over the same grid");
if( A.Height() != A.Width() )
throw std::logic_error("A must be square");
if( t.Viewing() )
throw std::logic_error("t must not be a view");
#endif
typedef Complex<R> C;
const Grid& g = A.Grid();
DistMatrix<C,MD,STAR> tDiag(g);
tDiag.AlignWithDiagonal( A, 1 );
tDiag.ResizeTo( A.Height()-1, 1 );
if( g.InGrid() )
{
// Matrix views
DistMatrix<C,MC,MR>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<C,MD,STAR> tT(g), t0(g),
tB(g), t1(g),
t2(g);
// Temporary distributions
DistMatrix<C,MC, MR > WPan(g);
DistMatrix<C,STAR,STAR> t1_STAR_STAR(g);
DistMatrix<C,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<C,MC, STAR> APan_MC_STAR(g), A01_MC_STAR(g),
A11_MC_STAR(g);
DistMatrix<C,MR, STAR> APan_MR_STAR(g), A01_MR_STAR(g),
A11_MR_STAR(g);
DistMatrix<C,MC, STAR> WPan_MC_STAR(g), W01_MC_STAR(g),
W11_MC_STAR(g);
DistMatrix<C,MR, STAR> WPan_MR_STAR(g), W01_MR_STAR(g),
W11_MR_STAR(g);
PartitionUpDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
PartitionUp
( tDiag, tT,
tB, 0 );
while( ABR.Height() < A.Height() )
{
RepartitionUpDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
RepartitionUp
( tT, t0,
t1,
/**/ /**/
tB, t2 );
if( A00.Height() > 0 )
{
WPan.AlignWith( A01 );
APan_MC_STAR.AlignWith( A00 );
WPan_MC_STAR.AlignWith( A00 );
APan_MR_STAR.AlignWith( A00 );
WPan_MR_STAR.AlignWith( A00 );
//------------------------------------------------------------//
WPan.ResizeTo( ATL.Height(), A11.Width() );
APan_MC_STAR.ResizeTo( ATL.Height(), A11.Width() );
WPan_MC_STAR.ResizeTo( ATL.Height(), A11.Width() );
APan_MR_STAR.ResizeTo( ATL.Height(), A11.Width() );
WPan_MR_STAR.ResizeTo( ATL.Height(), A11.Width() );
internal::HermitianPanelTridiagU
( ATL, WPan, t1,
APan_MC_STAR, APan_MR_STAR, WPan_MC_STAR, WPan_MR_STAR );
PartitionUp
( APan_MC_STAR, A01_MC_STAR,
A11_MC_STAR, A11.Height() );
PartitionUp
( APan_MR_STAR, A01_MR_STAR,
A11_MR_STAR, A11.Height() );
PartitionUp
( WPan_MC_STAR, W01_MC_STAR,
W11_MC_STAR, A11.Height() );
PartitionUp
( WPan_MR_STAR, W01_MR_STAR,
W11_MR_STAR, A11.Height() );
internal::LocalTrr2k
( UPPER, ADJOINT, ADJOINT,
(C)-1, A01_MC_STAR, W01_MR_STAR,
W01_MC_STAR, A01_MR_STAR,
(C)1, A00 );
//------------------------------------------------------------//
WPan_MR_STAR.FreeAlignments();
APan_MR_STAR.FreeAlignments();
WPan_MC_STAR.FreeAlignments();
APan_MC_STAR.FreeAlignments();
WPan.FreeAlignments();
}
else
{
A11_STAR_STAR = A11;
t1_STAR_STAR.ResizeTo( t1.Height(), 1 );
HermitianTridiag
( UPPER, A11_STAR_STAR.LocalMatrix(),
t1_STAR_STAR.LocalMatrix() );
A11 = A11_STAR_STAR;
t1 = t1_STAR_STAR;
}
SlidePartitionUp
( tT, t0,
/**/ /**/
t1,
tB, t2 );
SlidePartitionUpDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
}
}
// Redistribute from matrix-diagonal form to fully replicated
t = tDiag;
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
internal::HermitianTridiagU( DistMatrix<R,MC,MR>& A )
{
#ifndef RELEASE
PushCallStack("internal::HermitianTridiagU");
if( A.Height() != A.Width() )
throw std::logic_error( "A must be square." );
#endif
const Grid& g = A.Grid();
if( g.InGrid() )
{
// Matrix views
DistMatrix<R,MC,MR>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
// Temporary distributions
DistMatrix<R,MC, MR > WPan(g);
DistMatrix<R,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<R,MC, STAR> APan_MC_STAR(g), A01_MC_STAR(g),
A11_MC_STAR(g);
DistMatrix<R,MR, STAR> APan_MR_STAR(g), A01_MR_STAR(g),
A11_MR_STAR(g);
DistMatrix<R,MC, STAR> WPan_MC_STAR(g), W01_MC_STAR(g),
W11_MC_STAR(g);
DistMatrix<R,MR, STAR> WPan_MR_STAR(g), W01_MR_STAR(g),
W11_MR_STAR(g);
PartitionUpDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
while( ABR.Height() < A.Height() )
{
RepartitionUpDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
if( A00.Height() > 0 )
{
WPan.AlignWith( A01 );
APan_MC_STAR.AlignWith( A00 );
WPan_MC_STAR.AlignWith( A00 );
APan_MR_STAR.AlignWith( A00 );
WPan_MR_STAR.AlignWith( A00 );
//------------------------------------------------------------//
WPan.ResizeTo( ATL.Height(), A11.Width() );
APan_MC_STAR.ResizeTo( ATL.Height(), A11.Width() );
WPan_MC_STAR.ResizeTo( ATL.Height(), A11.Width() );
APan_MR_STAR.ResizeTo( ATL.Height(), A11.Width() );
WPan_MR_STAR.ResizeTo( ATL.Height(), A11.Width() );
internal::HermitianPanelTridiagU
( ATL, WPan,
APan_MC_STAR, APan_MR_STAR, WPan_MC_STAR, WPan_MR_STAR );
PartitionUp
( APan_MC_STAR, A01_MC_STAR,
A11_MC_STAR, A11.Height() );
PartitionUp
( APan_MR_STAR, A01_MR_STAR,
A11_MR_STAR, A11.Height() );
PartitionUp
( WPan_MC_STAR, W01_MC_STAR,
W11_MC_STAR, A11.Height() );
PartitionUp
( WPan_MR_STAR, W01_MR_STAR,
W11_MR_STAR, A11.Height() );
internal::LocalTrr2k
( UPPER, TRANSPOSE, TRANSPOSE,
(R)-1, A01_MC_STAR, W01_MR_STAR,
W01_MC_STAR, A01_MR_STAR,
(R)1, A00 );
//------------------------------------------------------------//
WPan_MR_STAR.FreeAlignments();
APan_MR_STAR.FreeAlignments();
WPan_MC_STAR.FreeAlignments();
APan_MC_STAR.FreeAlignments();
WPan.FreeAlignments();
}
else
{
A11_STAR_STAR = A11;
HermitianTridiag( UPPER, A11_STAR_STAR.LocalMatrix() );
A11 = A11_STAR_STAR;
}
SlidePartitionUpDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
}
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
TwoSidedTrsmUVar2
( UnitOrNonUnit diag, DistMatrix<F>& A, const DistMatrix<F>& U )
{
#ifndef RELEASE
CallStackEntry entry("internal::TwoSidedTrsmUVar2");
if( A.Height() != A.Width() )
LogicError("A must be square");
if( U.Height() != U.Width() )
LogicError("Triangular matrices must be square");
if( A.Height() != U.Height() )
LogicError("A and U must be the same size");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F>
UTL(g), UTR(g), U00(g), U01(g), U02(g),
UBL(g), UBR(g), U10(g), U11(g), U12(g),
U20(g), U21(g), U22(g);
// Temporary distributions
DistMatrix<F,MC, STAR> A01_MC_STAR(g);
DistMatrix<F,VC, STAR> A01_VC_STAR(g);
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,STAR,VR > A12_STAR_VR(g);
DistMatrix<F,MC, STAR> F01_MC_STAR(g);
DistMatrix<F,MC, STAR> U01_MC_STAR(g);
DistMatrix<F,VR, STAR> U01_VR_STAR(g);
DistMatrix<F,STAR,MR > U01Adj_STAR_MR(g);
DistMatrix<F,STAR,STAR> U11_STAR_STAR(g);
DistMatrix<F,STAR,MR > X11_STAR_MR(g);
DistMatrix<F,MR, STAR> X12Adj_MR_STAR(g);
DistMatrix<F,MR, MC > X12Adj_MR_MC(g);
DistMatrix<F,MR, MC > Y01_MR_MC(g);
DistMatrix<F,MR, STAR> Y01_MR_STAR(g);
DistMatrix<F> X11(g);
DistMatrix<F> Y01(g);
Matrix<F> X12Local;
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
A01_MC_STAR.AlignWith( U01 );
Y01.AlignWith( A01 );
Y01_MR_STAR.AlignWith( A00 );
U01_MC_STAR.AlignWith( A00 );
U01_VR_STAR.AlignWith( A00 );
U01Adj_STAR_MR.AlignWith( A00 );
X11_STAR_MR.AlignWith( U01 );
X11.AlignWith( A11 );
X12Adj_MR_STAR.AlignWith( A02 );
X12Adj_MR_MC.AlignWith( A12 );
F01_MC_STAR.AlignWith( A00 );
//--------------------------------------------------------------------//
// Y01 := A00 U01
U01_MC_STAR = U01;
U01_VR_STAR = U01_MC_STAR;
U01Adj_STAR_MR.AdjointFrom( U01_VR_STAR );
Zeros( Y01_MR_STAR, A01.Height(), A01.Width() );
Zeros( F01_MC_STAR, A01.Height(), A01.Width() );
LocalSymmetricAccumulateLU
( ADJOINT,
F(1), A00, U01_MC_STAR, U01Adj_STAR_MR, F01_MC_STAR, Y01_MR_STAR );
Y01_MR_MC.SumScatterFrom( Y01_MR_STAR );
Y01 = Y01_MR_MC;
Y01.SumScatterUpdate( F(1), F01_MC_STAR );
// X11 := U01' A01
LocalGemm( ADJOINT, NORMAL, F(1), U01_MC_STAR, A01, X11_STAR_MR );
// A01 := A01 - Y01
Axpy( F(-1), Y01, A01 );
A01_MC_STAR = A01;
// A11 := A11 - triu(X11 + A01' U01) = A11 - (U01 A01 + A01' U01)
LocalGemm( ADJOINT, NORMAL, F(1), A01_MC_STAR, U01, F(1), X11_STAR_MR );
X11.SumScatterFrom( X11_STAR_MR );
MakeTriangular( UPPER, X11 );
Axpy( F(-1), X11, A11 );
// A01 := A01 inv(U11)
U11_STAR_STAR = U11;
A01_VC_STAR = A01_MC_STAR;
LocalTrsm
( RIGHT, UPPER, NORMAL, diag, F(1), U11_STAR_STAR, A01_VC_STAR );
A01 = A01_VC_STAR;
// A11 := inv(U11)' A11 inv(U11)
A11_STAR_STAR = A11;
LocalTwoSidedTrsm( UPPER, diag, A11_STAR_STAR, U11_STAR_STAR );
A11 = A11_STAR_STAR;
// A12 := A12 - A02' U01
LocalGemm( ADJOINT, NORMAL, F(1), A02, U01_MC_STAR, X12Adj_MR_STAR );
X12Adj_MR_MC.SumScatterFrom( X12Adj_MR_STAR );
Adjoint( X12Adj_MR_MC.LockedMatrix(), X12Local );
Axpy( F(-1), X12Local, A12.Matrix() );
// A12 := inv(U11)' A12
A12_STAR_VR = A12;
LocalTrsm
( LEFT, UPPER, ADJOINT, diag, F(1), U11_STAR_STAR, A12_STAR_VR );
A12 = A12_STAR_VR;
//--------------------------------------------------------------------//
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/*************/ /******************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
}
}