void LSquare
( DistMatrix<Complex<R> >& A,
DistMatrix<Complex<R>,STAR,STAR>& t )
{
#ifndef RELEASE
CallStackEntry entry("hermitian_tridiag::LSquare");
if( A.Grid() != t.Grid() )
throw std::logic_error("{A,t} must be distributed over the same grid");
#endif
const Grid& g = A.Grid();
#ifndef RELEASE
if( g.Height() != g.Width() )
throw std::logic_error("The process grid must be square");
if( A.Height() != A.Width() )
throw std::logic_error("A must be square");
if( t.Viewing() )
throw std::logic_error("t must not be a view");
#endif
typedef Complex<R> C;
DistMatrix<C,MD,STAR> tDiag(g);
tDiag.AlignWithDiagonal( A, -1 );
tDiag.ResizeTo( A.Height()-1, 1 );
// Matrix views
DistMatrix<C>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<C,MD,STAR> tT(g), t0(g),
tB(g), t1(g),
t2(g);
// Temporary distributions
DistMatrix<C> WPan(g);
DistMatrix<C,STAR,STAR> t1_STAR_STAR(g);
DistMatrix<C,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<C,MC, STAR> APan_MC_STAR(g), A11_MC_STAR(g),
A21_MC_STAR(g);
DistMatrix<C,MR, STAR> APan_MR_STAR(g), A11_MR_STAR(g),
A21_MR_STAR(g);
DistMatrix<C,MC, STAR> WPan_MC_STAR(g), W11_MC_STAR(g),
W21_MC_STAR(g);
DistMatrix<C,MR, STAR> WPan_MR_STAR(g), W11_MR_STAR(g),
W21_MR_STAR(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
PartitionDown
( tDiag, tT,
tB, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
RepartitionDown
( tT, t0,
/**/ /**/
t1,
tB, t2 );
if( A22.Height() > 0 )
{
WPan.AlignWith( A11 );
APan_MC_STAR.AlignWith( A11 );
WPan_MC_STAR.AlignWith( A11 );
APan_MR_STAR.AlignWith( A11 );
WPan_MR_STAR.AlignWith( A11 );
//----------------------------------------------------------------//
WPan.ResizeTo( ABR.Height(), A11.Width() );
APan_MC_STAR.ResizeTo( ABR.Height(), A11.Width() );
WPan_MC_STAR.ResizeTo( ABR.Height(), A11.Width() );
APan_MR_STAR.ResizeTo( ABR.Height(), A11.Width() );
WPan_MR_STAR.ResizeTo( ABR.Height(), A11.Width() );
hermitian_tridiag::PanelLSquare
( ABR, WPan, t1,
APan_MC_STAR, APan_MR_STAR, WPan_MC_STAR, WPan_MR_STAR );
PartitionDown
( APan_MC_STAR, A11_MC_STAR,
A21_MC_STAR, A11.Height() );
PartitionDown
( APan_MR_STAR, A11_MR_STAR,
A21_MR_STAR, A11.Height() );
PartitionDown
( WPan_MC_STAR, W11_MC_STAR,
W21_MC_STAR, A11.Height() );
PartitionDown
( WPan_MR_STAR, W11_MR_STAR,
W21_MR_STAR, A11.Height() );
LocalTrr2k
( LOWER, ADJOINT, ADJOINT,
C(-1), A21_MC_STAR, W21_MR_STAR,
W21_MC_STAR, A21_MR_STAR,
C(1), A22 );
//----------------------------------------------------------------//
WPan_MR_STAR.FreeAlignments();
APan_MR_STAR.FreeAlignments();
WPan_MC_STAR.FreeAlignments();
APan_MC_STAR.FreeAlignments();
WPan.FreeAlignments();
}
else
{
A11_STAR_STAR = A11;
t1_STAR_STAR.ResizeTo( t1.Height(), 1 );
HermitianTridiag
( LOWER, A11_STAR_STAR.Matrix(), t1_STAR_STAR.Matrix() );
A11 = A11_STAR_STAR;
t1 = t1_STAR_STAR;
}
SlidePartitionDown
( tT, t0,
t1,
/**/ /**/
tB, t2 );
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
}
// Redistribute from matrix-diagonal form to fully replicated
t = tDiag;
}
void LSquare( DistMatrix<R>& A )
{
#ifndef RELEASE
CallStackEntry entry("hermitian_tridiag::LSquare");
if( A.Height() != A.Width() )
throw std::logic_error("A must be square");
if( A.Grid().Height() != A.Grid().Width() )
throw std::logic_error("The process grid must be square");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<R>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
// Temporary distributions
DistMatrix<R> WPan(g);
DistMatrix<R,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<R,MC, STAR> APan_MC_STAR(g), A11_MC_STAR(g),
A21_MC_STAR(g);
DistMatrix<R,MR, STAR> APan_MR_STAR(g), A11_MR_STAR(g),
A21_MR_STAR(g);
DistMatrix<R,MC, STAR> WPan_MC_STAR(g), W11_MC_STAR(g),
W21_MC_STAR(g);
DistMatrix<R,MR, STAR> WPan_MR_STAR(g), W11_MR_STAR(g),
W21_MR_STAR(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
if( A22.Height() > 0 )
{
WPan.AlignWith( A11 );
APan_MC_STAR.AlignWith( A11 );
WPan_MC_STAR.AlignWith( A11 );
APan_MR_STAR.AlignWith( A11 );
WPan_MR_STAR.AlignWith( A11 );
//----------------------------------------------------------------//
WPan.ResizeTo( ABR.Height(), A11.Width() );
APan_MC_STAR.ResizeTo( ABR.Height(), A11.Width() );
WPan_MC_STAR.ResizeTo( ABR.Height(), A11.Width() );
APan_MR_STAR.ResizeTo( ABR.Height(), A11.Width() );
WPan_MR_STAR.ResizeTo( ABR.Height(), A11.Width() );
hermitian_tridiag::PanelLSquare
( ABR, WPan,
APan_MC_STAR, APan_MR_STAR, WPan_MC_STAR, WPan_MR_STAR );
PartitionDown
( APan_MC_STAR, A11_MC_STAR,
A21_MC_STAR, A11.Height() );
PartitionDown
( APan_MR_STAR, A11_MR_STAR,
A21_MR_STAR, A11.Height() );
PartitionDown
( WPan_MC_STAR, W11_MC_STAR,
W21_MC_STAR, A11.Height() );
PartitionDown
( WPan_MR_STAR, W11_MR_STAR,
W21_MR_STAR, A11.Height() );
LocalTrr2k
( LOWER, TRANSPOSE, TRANSPOSE,
R(-1), A21_MC_STAR, W21_MR_STAR,
W21_MC_STAR, A21_MR_STAR,
R(1), A22 );
//----------------------------------------------------------------//
WPan_MR_STAR.FreeAlignments();
APan_MR_STAR.FreeAlignments();
WPan_MC_STAR.FreeAlignments();
APan_MC_STAR.FreeAlignments();
WPan.FreeAlignments();
}
else
{
A11_STAR_STAR = A11;
HermitianTridiag( LOWER, A11_STAR_STAR.Matrix() );
A11 = A11_STAR_STAR;
}
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
}
}
inline void
LU( DistMatrix<F>& A )
{
#ifndef RELEASE
CallStackEntry entry("LU");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
// Temporary distributions
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,MC, STAR> A21_MC_STAR(g);
DistMatrix<F,STAR,VR > A12_STAR_VR(g);
DistMatrix<F,STAR,MR > A12_STAR_MR(g);
// Start the algorithm
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
while( ATL.Height() < A.Height() && ATL.Width() < A.Width() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
A12_STAR_VR.AlignWith( A22 );
A12_STAR_MR.AlignWith( A22 );
A21_MC_STAR.AlignWith( A22 );
A11_STAR_STAR.ResizeTo( A11.Height(), A11.Width() );
//--------------------------------------------------------------------//
A11_STAR_STAR = A11;
LocalLU( A11_STAR_STAR );
A11 = A11_STAR_STAR;
A21_MC_STAR = A21;
LocalTrsm
( RIGHT, UPPER, NORMAL, NON_UNIT, F(1), A11_STAR_STAR, A21_MC_STAR );
A21 = A21_MC_STAR;
// Perhaps we should give up perfectly distributing this operation since
// it's total contribution is only O(n^2)
A12_STAR_VR = A12;
LocalTrsm
( LEFT, LOWER, NORMAL, UNIT, F(1), A11_STAR_STAR, A12_STAR_VR );
A12_STAR_MR = A12_STAR_VR;
LocalGemm( NORMAL, NORMAL, F(-1), A21_MC_STAR, A12_STAR_MR, F(1), A22 );
A12 = A12_STAR_MR;
//--------------------------------------------------------------------//
A12_STAR_VR.FreeAlignments();
A12_STAR_MR.FreeAlignments();
A21_MC_STAR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
}
}
inline void
TwoSidedTrmmLVar4
( UnitOrNonUnit diag, DistMatrix<F>& A, const DistMatrix<F>& L )
{
#ifndef RELEASE
CallStackEntry entry("internal::TwoSidedTrmmLVar4");
if( A.Height() != A.Width() )
LogicError("A must be square");
if( L.Height() != L.Width() )
LogicError("Triangular matrices must be square");
if( A.Height() != L.Height() )
LogicError("A and L must be the same size");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F>
LTL(g), LTR(g), L00(g), L01(g), L02(g),
LBL(g), LBR(g), L10(g), L11(g), L12(g),
L20(g), L21(g), L22(g);
// Temporary distributions
DistMatrix<F,STAR,VR > A10_STAR_VR(g);
DistMatrix<F,STAR,MR > A10_STAR_MR(g);
DistMatrix<F,STAR,MC > A10_STAR_MC(g);
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,VC, STAR> A21_VC_STAR(g);
DistMatrix<F,MC, STAR> A21_MC_STAR(g);
DistMatrix<F,STAR,VR > L10_STAR_VR(g);
DistMatrix<F,MR, STAR> L10Adj_MR_STAR(g);
DistMatrix<F,STAR,MC > L10_STAR_MC(g);
DistMatrix<F,STAR,STAR> L11_STAR_STAR(g);
DistMatrix<F,STAR,VR > Y10_STAR_VR(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( L, LTL, LTR,
LBL, LBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( LTL, /**/ LTR, L00, /**/ L01, L02,
/*************/ /******************/
/**/ L10, /**/ L11, L12,
LBL, /**/ LBR, L20, /**/ L21, L22 );
A10_STAR_VR.AlignWith( A00 );
A10_STAR_MR.AlignWith( A00 );
A10_STAR_MC.AlignWith( A00 );
A21_MC_STAR.AlignWith( A20 );
L10_STAR_VR.AlignWith( A00 );
L10Adj_MR_STAR.AlignWith( A00 );
L10_STAR_MC.AlignWith( A00 );
Y10_STAR_VR.AlignWith( A10 );
//--------------------------------------------------------------------//
// Y10 := A11 L10
A11_STAR_STAR = A11;
L10Adj_MR_STAR.AdjointFrom( L10 );
L10_STAR_VR.AdjointFrom( L10Adj_MR_STAR );
Zeros( Y10_STAR_VR, A10.Height(), A10.Width() );
Hemm
( LEFT, LOWER,
F(1), A11_STAR_STAR.LockedMatrix(), L10_STAR_VR.LockedMatrix(),
F(0), Y10_STAR_VR.Matrix() );
// A10 := A10 + 1/2 Y10
A10_STAR_VR = A10;
Axpy( F(1)/F(2), Y10_STAR_VR, A10_STAR_VR );
// A00 := A00 + (A10' L10 + L10' A10)
A10_STAR_MR = A10_STAR_VR;
A10_STAR_MC = A10_STAR_VR;
L10_STAR_MC = L10_STAR_VR;
LocalTrr2k
( LOWER, ADJOINT, ADJOINT, ADJOINT,
F(1), A10_STAR_MC, L10Adj_MR_STAR,
L10_STAR_MC, A10_STAR_MR,
F(1), A00 );
// A10 := A10 + 1/2 Y10
Axpy( F(1)/F(2), Y10_STAR_VR, A10_STAR_VR );
// A10 := L11' A10
L11_STAR_STAR = L11;
LocalTrmm
( LEFT, LOWER, ADJOINT, diag, F(1), L11_STAR_STAR, A10_STAR_VR );
A10 = A10_STAR_VR;
// A20 := A20 + A21 L10
A21_MC_STAR = A21;
LocalGemm
( NORMAL, ADJOINT, F(1), A21_MC_STAR, L10Adj_MR_STAR, F(1), A20 );
// A11 := L11' A11 L11
LocalTwoSidedTrmm( LOWER, diag, A11_STAR_STAR, L11_STAR_STAR );
A11 = A11_STAR_STAR;
// A21 := A21 L11
A21_VC_STAR = A21_MC_STAR;
LocalTrmm
( RIGHT, LOWER, NORMAL, diag, F(1), L11_STAR_STAR, A21_VC_STAR );
A21 = A21_VC_STAR;
//--------------------------------------------------------------------//
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( LTL, /**/ LTR, L00, L01, /**/ L02,
/**/ L10, L11, /**/ L12,
/*************/ /******************/
LBL, /**/ LBR, L20, L21, /**/ L22 );
}
}
inline void
LU( DistMatrix<F>& A, DistMatrix<int,VC,STAR>& p )
{
#ifndef RELEASE
CallStackEntry entry("LU");
if( A.Grid() != p.Grid() )
throw std::logic_error("{A,p} must be distributed over the same grid");
if( p.Viewing() &&
(std::min(A.Height(),A.Width()) != p.Height() || p.Width() != 1) )
throw std::logic_error
("p must be a vector of the same height as the min dimension of A.");
#endif
const Grid& g = A.Grid();
if( !p.Viewing() )
p.ResizeTo( std::min(A.Height(),A.Width()), 1 );
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g), AB(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<int,VC,STAR>
pT(g), p0(g),
pB(g), p1(g),
p2(g);
// Temporary distributions
DistMatrix<F, STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F, MC, STAR> A21_MC_STAR(g);
DistMatrix<F, STAR,VR > A12_STAR_VR(g);
DistMatrix<F, STAR,MR > A12_STAR_MR(g);
DistMatrix<int,STAR,STAR> p1_STAR_STAR(g);
// Pivot composition
std::vector<int> image, preimage;
// Start the algorithm
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
PartitionDown
( p, pT,
pB, 0 );
while( ATL.Height() < A.Height() && ATL.Width() < A.Width() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
RepartitionDown
( pT, p0,
/**/ /**/
p1,
pB, p2 );
View1x2( AB, ABL, ABR );
const int pivotOffset = A01.Height();
A12_STAR_VR.AlignWith( A22 );
A12_STAR_MR.AlignWith( A22 );
A21_MC_STAR.AlignWith( A22 );
A11_STAR_STAR.ResizeTo( A11.Height(), A11.Width() );
p1_STAR_STAR.ResizeTo( p1.Height(), 1 );
//--------------------------------------------------------------------//
A21_MC_STAR = A21;
A11_STAR_STAR = A11;
lu::Panel( A11_STAR_STAR, A21_MC_STAR, p1_STAR_STAR, pivotOffset );
ComposePivots( p1_STAR_STAR, pivotOffset, image, preimage );
ApplyRowPivots( AB, image, preimage );
// Perhaps we should give up perfectly distributing this operation since
// it's total contribution is only O(n^2)
A12_STAR_VR = A12;
LocalTrsm
( LEFT, LOWER, NORMAL, UNIT, F(1), A11_STAR_STAR, A12_STAR_VR );
A12_STAR_MR = A12_STAR_VR;
LocalGemm( NORMAL, NORMAL, F(-1), A21_MC_STAR, A12_STAR_MR, F(1), A22 );
A11 = A11_STAR_STAR;
A12 = A12_STAR_MR;
A21 = A21_MC_STAR;
p1 = p1_STAR_STAR;
//--------------------------------------------------------------------//
A12_STAR_VR.FreeAlignments();
A12_STAR_MR.FreeAlignments();
A21_MC_STAR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlidePartitionDown
( pT, p0,
p1,
/**/ /**/
pB, p2 );
}
}
inline void
internal::HegstLLVar4( DistMatrix<F,MC,MR>& A, const DistMatrix<F,MC,MR>& L )
{
#ifndef RELEASE
PushCallStack("internal::HegstLLVar4");
if( A.Height() != A.Width() )
throw std::logic_error("A must be square");
if( L.Height() != L.Width() )
throw std::logic_error("Triangular matrices must be square");
if( A.Height() != L.Height() )
throw std::logic_error("A and L must be the same size");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F,MC,MR>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F,MC,MR>
LTL(g), LTR(g), L00(g), L01(g), L02(g),
LBL(g), LBR(g), L10(g), L11(g), L12(g),
L20(g), L21(g), L22(g);
// Temporary distributions
DistMatrix<F,STAR,VR > A10_STAR_VR(g);
DistMatrix<F,STAR,MR > A10_STAR_MR(g);
DistMatrix<F,STAR,MC > A10_STAR_MC(g);
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,VC, STAR> A21_VC_STAR(g);
DistMatrix<F,MC, STAR> A21_MC_STAR(g);
DistMatrix<F,STAR,VR > L10_STAR_VR(g);
DistMatrix<F,STAR,MR > L10_STAR_MR(g);
DistMatrix<F,STAR,MC > L10_STAR_MC(g);
DistMatrix<F,STAR,STAR> L11_STAR_STAR(g);
DistMatrix<F,STAR,VR > Y10_STAR_VR(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( L, LTL, LTR,
LBL, LBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( LTL, /**/ LTR, L00, /**/ L01, L02,
/*************/ /******************/
/**/ L10, /**/ L11, L12,
LBL, /**/ LBR, L20, /**/ L21, L22 );
A10_STAR_VR.AlignWith( A00 );
A10_STAR_MR.AlignWith( A00 );
A10_STAR_MC.AlignWith( A00 );
A21_MC_STAR.AlignWith( A20 );
L10_STAR_VR.AlignWith( A00 );
L10_STAR_MR.AlignWith( A00 );
L10_STAR_MC.AlignWith( A00 );
Y10_STAR_VR.AlignWith( A10 );
//--------------------------------------------------------------------//
// Y10 := A11 L10
A11_STAR_STAR = A11;
L10_STAR_VR = L10;
Y10_STAR_VR.ResizeTo( A10.Height(), A10.Width() );
Zero( Y10_STAR_VR );
Hemm
( LEFT, LOWER,
(F)0.5, A11_STAR_STAR.LockedLocalMatrix(),
L10_STAR_VR.LockedLocalMatrix(),
(F)0, Y10_STAR_VR.LocalMatrix() );
// A10 := A10 + 1/2 Y10
A10_STAR_VR = A10;
Axpy( (F)1, Y10_STAR_VR, A10_STAR_VR );
// A00 := A00 + (A10' L10 + L10' A10)
A10_STAR_MR = A10_STAR_VR;
A10_STAR_MC = A10_STAR_VR;
L10_STAR_MR = L10_STAR_VR;
L10_STAR_MC = L10_STAR_VR;
internal::LocalTrr2k
( LOWER, ADJOINT, ADJOINT,
(F)1, A10_STAR_MC, L10_STAR_MR,
L10_STAR_MC, A10_STAR_MR,
(F)1, A00 );
// A10 := A10 + 1/2 Y10
Axpy( (F)1, Y10_STAR_VR, A10_STAR_VR );
// A10 := L11' A10
L11_STAR_STAR = L11;
internal::LocalTrmm
( LEFT, LOWER, ADJOINT, NON_UNIT, (F)1, L11_STAR_STAR, A10_STAR_VR );
A10 = A10_STAR_VR;
// A20 := A20 + A21 L10
A21_MC_STAR = A21;
internal::LocalGemm
( NORMAL, NORMAL, (F)1, A21_MC_STAR, L10_STAR_MR, (F)1, A20 );
// A11 := L11' A11 L11
internal::LocalHegst
( LEFT, LOWER, A11_STAR_STAR, L11_STAR_STAR );
A11 = A11_STAR_STAR;
// A21 := A21 L11
A21_VC_STAR = A21_MC_STAR;
internal::LocalTrmm
( RIGHT, LOWER, NORMAL, NON_UNIT, (F)1, L11_STAR_STAR, A21_VC_STAR );
A21 = A21_VC_STAR;
//--------------------------------------------------------------------//
A10_STAR_VR.FreeAlignments();
A10_STAR_MR.FreeAlignments();
A10_STAR_MC.FreeAlignments();
A21_MC_STAR.FreeAlignments();
L10_STAR_VR.FreeAlignments();
L10_STAR_MR.FreeAlignments();
L10_STAR_MC.FreeAlignments();
Y10_STAR_VR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( LTL, /**/ LTR, L00, L01, /**/ L02,
/**/ L10, L11, /**/ L12,
/*************/ /******************/
LBL, /**/ LBR, L20, L21, /**/ L22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
TwoSidedTrsmLVar5
( UnitOrNonUnit diag, DistMatrix<F>& A, const DistMatrix<F>& L )
{
#ifndef RELEASE
CallStackEntry entry("internal::TwoSidedTrsmLVar5");
if( A.Height() != A.Width() )
LogicError("A must be square");
if( L.Height() != L.Width() )
LogicError("Triangular matrices must be square");
if( A.Height() != L.Height() )
LogicError("A and L must be the same size");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F>
LTL(g), LTR(g), L00(g), L01(g), L02(g),
LBL(g), LBR(g), L10(g), L11(g), L12(g),
L20(g), L21(g), L22(g);
// Temporary distributions
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,MC, STAR> A21_MC_STAR(g);
DistMatrix<F,VC, STAR> A21_VC_STAR(g);
DistMatrix<F,VR, STAR> A21_VR_STAR(g);
DistMatrix<F,STAR,MR > A21Adj_STAR_MR(g);
DistMatrix<F,STAR,STAR> L11_STAR_STAR(g);
DistMatrix<F,MC, STAR> L21_MC_STAR(g);
DistMatrix<F,VC, STAR> L21_VC_STAR(g);
DistMatrix<F,VR, STAR> L21_VR_STAR(g);
DistMatrix<F,STAR,MR > L21Adj_STAR_MR(g);
DistMatrix<F,VC, STAR> Y21_VC_STAR(g);
DistMatrix<F> Y21(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( L, LTL, LTR,
LBL, LBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( LTL, /**/ LTR, L00, /**/ L01, L02,
/*************/ /******************/
/**/ L10, /**/ L11, L12,
LBL, /**/ LBR, L20, /**/ L21, L22 );
A21_MC_STAR.AlignWith( A22 );
A21_VC_STAR.AlignWith( A22 );
A21_VR_STAR.AlignWith( A22 );
A21Adj_STAR_MR.AlignWith( A22 );
L21_MC_STAR.AlignWith( A22 );
L21_VC_STAR.AlignWith( A22 );
L21_VR_STAR.AlignWith( A22 );
L21Adj_STAR_MR.AlignWith( A22 );
Y21.AlignWith( A21 );
Y21_VC_STAR.AlignWith( A22 );
//--------------------------------------------------------------------//
// A11 := inv(L11) A11 inv(L11)'
L11_STAR_STAR = L11;
A11_STAR_STAR = A11;
LocalTwoSidedTrsm( LOWER, diag, A11_STAR_STAR, L11_STAR_STAR );
A11 = A11_STAR_STAR;
// Y21 := L21 A11
L21_VC_STAR = L21;
Zeros( Y21_VC_STAR, A21.Height(), A21.Width() );
Hemm
( RIGHT, LOWER,
F(1), A11_STAR_STAR.Matrix(), L21_VC_STAR.Matrix(),
F(0), Y21_VC_STAR.Matrix() );
Y21 = Y21_VC_STAR;
// A21 := A21 inv(L11)'
A21_VC_STAR = A21;
LocalTrsm
( RIGHT, LOWER, ADJOINT, diag, F(1), L11_STAR_STAR, A21_VC_STAR );
A21 = A21_VC_STAR;
// A21 := A21 - 1/2 Y21
Axpy( F(-1)/F(2), Y21, A21 );
// A22 := A22 - (L21 A21' + A21 L21')
A21_MC_STAR = A21;
L21_MC_STAR = L21;
A21_VC_STAR = A21_MC_STAR;
A21_VR_STAR = A21_VC_STAR;
L21_VR_STAR = L21_VC_STAR;
A21Adj_STAR_MR.AdjointFrom( A21_VR_STAR );
L21Adj_STAR_MR.AdjointFrom( L21_VR_STAR );
LocalTrr2k
( LOWER,
F(-1), L21_MC_STAR, A21Adj_STAR_MR,
A21_MC_STAR, L21Adj_STAR_MR,
F(1), A22 );
// A21 := A21 - 1/2 Y21
Axpy( F(-1)/F(2), Y21, A21 );
// A21 := inv(L22) A21
//
// This is the bottleneck because A21 only has blocksize columns
Trsm( LEFT, LOWER, NORMAL, diag, F(1), L22, A21 );
//--------------------------------------------------------------------//
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( LTL, /**/ LTR, L00, L01, /**/ L02,
/**/ L10, L11, /**/ L12,
/**********************************/
LBL, /**/ LBR, L20, L21, /**/ L22 );
}
}
inline void
RowEchelon( DistMatrix<F>& A, DistMatrix<F>& B )
{
#ifndef RELEASE
CallStackEntry entry("RowEchelon");
if( A.Grid() != B.Grid() )
LogicError("{A,B} must be distributed over the same grid");
if( A.Height() != B.Height() )
LogicError("A and B must be the same height");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g), APan(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F>
BT(g), B0(g),
BB(g), B1(g),
B2(g);
// Temporary distributions
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,STAR,VR > A12_STAR_VR(g);
DistMatrix<F,STAR,MR > A12_STAR_MR(g);
DistMatrix<F,MC, STAR> A21_MC_STAR(g);
DistMatrix<F,STAR,VR > B1_STAR_VR(g);
DistMatrix<F,STAR,MR > B1_STAR_MR(g);
DistMatrix<Int,STAR,STAR> p1_STAR_STAR(g);
// In case B's columns are not aligned with A's
const bool BAligned = ( B.ColShift() == A.ColShift() );
DistMatrix<F,MC,STAR> A21_MC_STAR_B(g);
// Pivot composition
std::vector<Int> image, preimage;
// Start the algorithm
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
PartitionDown
( B, BT,
BB, 0 );
while( ATL.Height() < A.Height() && ATL.Width() < A.Width() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
RepartitionDown
( BT, B0,
/**/ /**/
B1,
BB, B2 );
View2x1
( APan, A12,
A22 );
A12_STAR_VR.AlignWith( A22 );
A12_STAR_MR.AlignWith( A22 );
A21_MC_STAR.AlignWith( A22 );
B1_STAR_VR.AlignWith( B1 );
B1_STAR_MR.AlignWith( B1 );
if( ! BAligned )
A21_MC_STAR_B.AlignWith( B2 );
p1_STAR_STAR.ResizeTo( A11.Height(), 1 );
//--------------------------------------------------------------------//
A11_STAR_STAR = A11;
A21_MC_STAR = A21;
lu::Panel( A11_STAR_STAR, A21_MC_STAR, p1_STAR_STAR, A00.Height() );
ComposePivots( p1_STAR_STAR, A00.Height(), image, preimage );
ApplyRowPivots( APan, image, preimage );
ApplyRowPivots( BB, image, preimage );
A12_STAR_VR = A12;
B1_STAR_VR = B1;
LocalTrsm
( LEFT, LOWER, NORMAL, UNIT, F(1), A11_STAR_STAR, A12_STAR_VR );
LocalTrsm( LEFT, LOWER, NORMAL, UNIT, F(1), A11_STAR_STAR, B1_STAR_VR );
A12_STAR_MR = A12_STAR_VR;
B1_STAR_MR = B1_STAR_VR;
LocalGemm( NORMAL, NORMAL, F(-1), A21_MC_STAR, A12_STAR_MR, F(1), A22 );
if( BAligned )
{
LocalGemm
( NORMAL, NORMAL, F(-1), A21_MC_STAR, B1_STAR_MR, F(1), B2 );
}
else
{
A21_MC_STAR_B = A21_MC_STAR;
LocalGemm
( NORMAL, NORMAL, F(-1), A21_MC_STAR_B, B1_STAR_MR, F(1), B2 );
}
A11 = A11_STAR_STAR;
A12 = A12_STAR_MR;
B1 = B1_STAR_MR;
//--------------------------------------------------------------------//
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlidePartitionDown
( BT, B0,
B1,
/**/ /**/
BB, B2 );
}
}
inline void
internal::CholeskyLVar3Naive( DistMatrix<F,MC,MR>& A )
{
#ifndef RELEASE
PushCallStack("internal::CholeskyLVar3Naive");
if( A.Height() != A.Width() )
throw std::logic_error
("Can only compute Cholesky factor of square matrices");
if( A.Grid().VCRank() == 0 )
{
std::cout
<< "CholeskyLVar3Naive exists solely for academic purposes. Please "
"use CholeskyLVar3 in real applications." << std::endl;
}
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F,MC,MR>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
// Temporary matrices
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,VC, STAR> A21_VC_STAR(g);
DistMatrix<F,MC, STAR> A21_MC_STAR(g);
DistMatrix<F,MR, STAR> A21_MR_STAR(g);
// Start the algorithm
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
while( ABR.Height() > 0 )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
A21_VC_STAR.AlignWith( A22 );
A21_MC_STAR.AlignWith( A22 );
A21_MR_STAR.AlignWith( A22 );
//--------------------------------------------------------------------//
A11_STAR_STAR = A11;
internal::LocalCholesky( LOWER, A11_STAR_STAR );
A11 = A11_STAR_STAR;
A21_VC_STAR = A21;
internal::LocalTrsm
( RIGHT, LOWER, ADJOINT, NON_UNIT, (F)1, A11_STAR_STAR, A21_VC_STAR );
A21_MC_STAR = A21_VC_STAR;
A21_MR_STAR = A21_VC_STAR;
// (A21^T[* ,MC])^T A21^H[* ,MR] = A21[MC,* ] A21^H[* ,MR]
// = (A21 A21^H)[MC,MR]
internal::LocalTrrk
( LOWER, ADJOINT, (F)-1, A21_MC_STAR, A21_MR_STAR, (F)1, A22 );
A21 = A21_MC_STAR;
//--------------------------------------------------------------------//
A21_VC_STAR.FreeAlignments();
A21_MC_STAR.FreeAlignments();
A21_MR_STAR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
