inline void
CholeskyUVar2( DistMatrix<F>& A )
{
#ifndef RELEASE
CallStackEntry entry("hpd_inverse::CholeskyUVar2");
if( A.Height() != A.Width() )
throw std::logic_error("Nonsquare matrices cannot be triangular");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
// Temporary distributions
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,VC, STAR> A01_VC_STAR(g);
DistMatrix<F,VR, STAR> A01_VR_STAR(g);
DistMatrix<F,STAR,VR > A12_STAR_VR(g);
DistMatrix<F,STAR,MC > A01Trans_STAR_MC(g);
DistMatrix<F,MR, STAR> A01_MR_STAR(g);
DistMatrix<F,STAR,MR > A01Adj_STAR_MR(g);
DistMatrix<F,STAR,MR > A12_STAR_MR(g);
DistMatrix<F,STAR,MC > A12_STAR_MC(g);
// Start the algorithm
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
A01_VC_STAR.AlignWith( A00 );
A12_STAR_VR.AlignWith( A02 );
A01Trans_STAR_MC.AlignWith( A00 );
A01_VR_STAR.AlignWith( A00 );
A01Adj_STAR_MR.AlignWith( A00 );
A12_STAR_MR.AlignWith( A02 );
A12_STAR_MC.AlignWith( A22 );
//--------------------------------------------------------------------//
A11_STAR_STAR = A11;
LocalCholesky( UPPER, A11_STAR_STAR );
A01_VC_STAR = A01;
LocalTrsm
( RIGHT, UPPER, NORMAL, NON_UNIT, F(1), A11_STAR_STAR, A01_VC_STAR );
A12_STAR_VR = A12;
LocalTrsm
( LEFT, UPPER, ADJOINT, NON_UNIT, F(1), A11_STAR_STAR, A12_STAR_VR );
A01Trans_STAR_MC.TransposeFrom( A01_VC_STAR );
A01_VR_STAR = A01_VC_STAR;
A01Adj_STAR_MR.AdjointFrom( A01_VR_STAR );
LocalTrrk
( UPPER, TRANSPOSE,
F(1), A01Trans_STAR_MC, A01Adj_STAR_MR, F(1), A00 );
A12_STAR_MR = A12_STAR_VR;
LocalGemm
( TRANSPOSE, NORMAL, F(-1), A01Trans_STAR_MC, A12_STAR_MR, F(1), A02 );
A12_STAR_MC = A12_STAR_VR;
LocalTrrk
( UPPER, ADJOINT,
F(-1), A12_STAR_MC, A12_STAR_MR, F(1), A22 );
LocalTrsm
( RIGHT, UPPER, ADJOINT, NON_UNIT, F(1), A11_STAR_STAR, A01_VC_STAR );
LocalTrsm
( LEFT, UPPER, NORMAL, NON_UNIT, F(-1), A11_STAR_STAR, A12_STAR_VR );
LocalTriangularInverse( UPPER, NON_UNIT, A11_STAR_STAR );
LocalTrtrmm( ADJOINT, UPPER, A11_STAR_STAR );
A11 = A11_STAR_STAR;
A01 = A01_VC_STAR;
A12 = A12_STAR_VR;
//--------------------------------------------------------------------//
A01_VC_STAR.FreeAlignments();
A12_STAR_VR.FreeAlignments();
A01Trans_STAR_MC.FreeAlignments();
A01_VR_STAR.FreeAlignments();
A01Adj_STAR_MR.FreeAlignments();
A12_STAR_MR.FreeAlignments();
A12_STAR_MC.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
}
}
inline void
CholeskyUVar3( DistMatrix<F>& A )
{
#ifndef RELEASE
PushCallStack("internal::CholeskyUVar3");
if( A.Height() != A.Width() )
throw std::logic_error
("Can only compute Cholesky factor of square matrices");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
// Temporary matrix distributions
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,STAR,VR > A12_STAR_VR(g);
DistMatrix<F,STAR,MC > A12_STAR_MC(g);
DistMatrix<F,STAR,MR > A12_STAR_MR(g);
// Start the algorithm
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
while( ABR.Height() > 0 )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
A12_STAR_MC.AlignWith( A22 );
A12_STAR_MR.AlignWith( A22 );
A12_STAR_VR.AlignWith( A22 );
//--------------------------------------------------------------------//
A11_STAR_STAR = A11;
LocalCholesky( UPPER, A11_STAR_STAR );
A11 = A11_STAR_STAR;
A12_STAR_VR = A12;
LocalTrsm
( LEFT, UPPER, ADJOINT, NON_UNIT, F(1), A11_STAR_STAR, A12_STAR_VR );
A12_STAR_MC = A12_STAR_VR;
A12_STAR_MR = A12_STAR_VR;
LocalTrrk
( UPPER, ADJOINT, F(-1), A12_STAR_MC, A12_STAR_MR, F(1), A22 );
A12 = A12_STAR_MR;
//--------------------------------------------------------------------//
A12_STAR_MC.FreeAlignments();
A12_STAR_MR.FreeAlignments();
A12_STAR_VR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
LU( DistMatrix<F>& A )
{
#ifndef RELEASE
CallStackEntry entry("LU");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
// Temporary distributions
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,MC, STAR> A21_MC_STAR(g);
DistMatrix<F,STAR,VR > A12_STAR_VR(g);
DistMatrix<F,STAR,MR > A12_STAR_MR(g);
// Start the algorithm
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
while( ATL.Height() < A.Height() && ATL.Width() < A.Width() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
A12_STAR_VR.AlignWith( A22 );
A12_STAR_MR.AlignWith( A22 );
A21_MC_STAR.AlignWith( A22 );
A11_STAR_STAR.ResizeTo( A11.Height(), A11.Width() );
//--------------------------------------------------------------------//
A11_STAR_STAR = A11;
LocalLU( A11_STAR_STAR );
A11 = A11_STAR_STAR;
A21_MC_STAR = A21;
LocalTrsm
( RIGHT, UPPER, NORMAL, NON_UNIT, F(1), A11_STAR_STAR, A21_MC_STAR );
A21 = A21_MC_STAR;
// Perhaps we should give up perfectly distributing this operation since
// it's total contribution is only O(n^2)
A12_STAR_VR = A12;
LocalTrsm
( LEFT, LOWER, NORMAL, UNIT, F(1), A11_STAR_STAR, A12_STAR_VR );
A12_STAR_MR = A12_STAR_VR;
LocalGemm( NORMAL, NORMAL, F(-1), A21_MC_STAR, A12_STAR_MR, F(1), A22 );
A12 = A12_STAR_MR;
//--------------------------------------------------------------------//
A12_STAR_VR.FreeAlignments();
A12_STAR_MR.FreeAlignments();
A21_MC_STAR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
}
}
inline void
TwoSidedTrsmUVar5
( UnitOrNonUnit diag, DistMatrix<F>& A, const DistMatrix<F>& U )
{
#ifndef RELEASE
PushCallStack("internal::TwoSidedTrsmUVar5");
if( A.Height() != A.Width() )
throw std::logic_error("A must be square");
if( U.Height() != U.Width() )
throw std::logic_error("Triangular matrices must be square");
if( A.Height() != U.Height() )
throw std::logic_error("A and U must be the same size");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F>
UTL(g), UTR(g), U00(g), U01(g), U02(g),
UBL(g), UBR(g), U10(g), U11(g), U12(g),
U20(g), U21(g), U22(g);
// Temporary distributions
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,STAR,MC > A12_STAR_MC(g);
DistMatrix<F,STAR,MR > A12_STAR_MR(g);
DistMatrix<F,STAR,VC > A12_STAR_VC(g);
DistMatrix<F,STAR,VR > A12_STAR_VR(g);
DistMatrix<F,STAR,STAR> U11_STAR_STAR(g);
DistMatrix<F,STAR,MC > U12_STAR_MC(g);
DistMatrix<F,STAR,MR > U12_STAR_MR(g);
DistMatrix<F,STAR,VC > U12_STAR_VC(g);
DistMatrix<F,STAR,VR > U12_STAR_VR(g);
DistMatrix<F,STAR,VR > Y12_STAR_VR(g);
DistMatrix<F> Y12(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
A12_STAR_MC.AlignWith( A22 );
A12_STAR_MR.AlignWith( A22 );
A12_STAR_VC.AlignWith( A22 );
A12_STAR_VR.AlignWith( A22 );
U12_STAR_MC.AlignWith( A22 );
U12_STAR_MR.AlignWith( A22 );
U12_STAR_VC.AlignWith( A22 );
U12_STAR_VR.AlignWith( A22 );
Y12.AlignWith( A12 );
Y12_STAR_VR.AlignWith( A12 );
//--------------------------------------------------------------------//
// A11 := inv(U11)' A11 inv(U11)
U11_STAR_STAR = U11;
A11_STAR_STAR = A11;
LocalTwoSidedTrsm( UPPER, diag, A11_STAR_STAR, U11_STAR_STAR );
A11 = A11_STAR_STAR;
// Y12 := A11 U12
U12_STAR_VR = U12;
Y12_STAR_VR.ResizeTo( A12.Height(), A12.Width() );
Hemm
( LEFT, UPPER,
F(1), A11_STAR_STAR.LocalMatrix(), U12_STAR_VR.LocalMatrix(),
F(0), Y12_STAR_VR.LocalMatrix() );
Y12 = Y12_STAR_VR;
// A12 := inv(U11)' A12
A12_STAR_VR = A12;
LocalTrsm
( LEFT, UPPER, ADJOINT, diag, F(1), U11_STAR_STAR, A12_STAR_VR );
A12 = A12_STAR_VR;
// A12 := A12 - 1/2 Y12
Axpy( F(-1)/F(2), Y12, A12 );
// A22 := A22 - (A12' U12 + U12' A12)
A12_STAR_VR = A12;
A12_STAR_VC = A12_STAR_VR;
U12_STAR_VC = U12_STAR_VR;
A12_STAR_MC = A12_STAR_VC;
U12_STAR_MC = U12_STAR_VC;
A12_STAR_MR = A12_STAR_VR;
U12_STAR_MR = U12_STAR_VR;
LocalTrr2k
( UPPER, ADJOINT, ADJOINT,
F(-1), U12_STAR_MC, A12_STAR_MR,
A12_STAR_MC, U12_STAR_MR,
F(1), A22 );
// A12 := A12 - 1/2 Y12
Axpy( F(-1)/F(2), Y12, A12 );
// A12 := A12 inv(U22)
//
// This is the bottleneck because A12 only has blocksize rows
Trsm( RIGHT, UPPER, NORMAL, diag, F(1), U22, A12 );
//--------------------------------------------------------------------//
A12_STAR_MC.FreeAlignments();
A12_STAR_MR.FreeAlignments();
A12_STAR_VC.FreeAlignments();
A12_STAR_VR.FreeAlignments();
U12_STAR_MC.FreeAlignments();
U12_STAR_MR.FreeAlignments();
U12_STAR_VC.FreeAlignments();
U12_STAR_VR.FreeAlignments();
Y12.FreeAlignments();
Y12_STAR_VR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/*************/ /******************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
TwoSidedTrsmUVar4
( UnitOrNonUnit diag, DistMatrix<F>& A, const DistMatrix<F>& U )
{
#ifndef RELEASE
CallStackEntry entry("internal::TwoSidedTrsmUVar4");
if( A.Height() != A.Width() )
LogicError("A must be square");
if( U.Height() != U.Width() )
LogicError("Triangular matrices must be square");
if( A.Height() != U.Height() )
LogicError("A and U must be the same size");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F>
UTL(g), UTR(g), U00(g), U01(g), U02(g),
UBL(g), UBR(g), U10(g), U11(g), U12(g),
U20(g), U21(g), U22(g);
// Temporary distributions
DistMatrix<F,VC, STAR> A01_VC_STAR(g);
DistMatrix<F,STAR,MC > A01Trans_STAR_MC(g);
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,STAR,VR > A12_STAR_VR(g);
DistMatrix<F,STAR,VC > A12_STAR_VC(g);
DistMatrix<F,STAR,MC > A12_STAR_MC(g);
DistMatrix<F,STAR,MR > A12_STAR_MR(g);
DistMatrix<F,STAR,STAR> U11_STAR_STAR(g);
DistMatrix<F,MR, STAR> U12Trans_MR_STAR(g);
DistMatrix<F,VR, STAR> U12Trans_VR_STAR(g);
DistMatrix<F,STAR,VR > U12_STAR_VR(g);
DistMatrix<F,STAR,VC > U12_STAR_VC(g);
DistMatrix<F,STAR,MC > U12_STAR_MC(g);
DistMatrix<F,STAR,VR > Y12_STAR_VR(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
A01_VC_STAR.AlignWith( A02 );
A01Trans_STAR_MC.AlignWith( A02 );
A12_STAR_VR.AlignWith( A22 );
A12_STAR_VC.AlignWith( A22 );
A12_STAR_MC.AlignWith( A22 );
A12_STAR_MR.AlignWith( A22 );
U12Trans_MR_STAR.AlignWith( A02 );
U12Trans_VR_STAR.AlignWith( A02 );
U12_STAR_VR.AlignWith( A02 );
U12_STAR_VC.AlignWith( A22 );
U12_STAR_MC.AlignWith( A22 );
Y12_STAR_VR.AlignWith( A12 );
//--------------------------------------------------------------------//
// A01 := A01 inv(U11)
A01_VC_STAR = A01;
U11_STAR_STAR = U11;
LocalTrsm
( RIGHT, UPPER, NORMAL, diag, F(1), U11_STAR_STAR, A01_VC_STAR );
A01 = A01_VC_STAR;
// A11 := inv(U11)' A11 inv(U11)
A11_STAR_STAR = A11;
LocalTwoSidedTrsm( UPPER, diag, A11_STAR_STAR, U11_STAR_STAR );
A11 = A11_STAR_STAR;
// A02 := A02 - A01 U12
A01Trans_STAR_MC.TransposeFrom( A01_VC_STAR );
U12Trans_MR_STAR.TransposeFrom( U12 );
LocalGemm
( TRANSPOSE, TRANSPOSE,
F(-1), A01Trans_STAR_MC, U12Trans_MR_STAR, F(1), A02 );
// Y12 := A11 U12
U12Trans_VR_STAR = U12Trans_MR_STAR;
Zeros( U12_STAR_VR, A12.Height(), A12.Width() );
Transpose( U12Trans_VR_STAR.Matrix(), U12_STAR_VR.Matrix() );
Zeros( Y12_STAR_VR, A12.Height(), A12.Width() );
Hemm
( LEFT, UPPER,
F(1), A11_STAR_STAR.Matrix(), U12_STAR_VR.Matrix(),
F(0), Y12_STAR_VR.Matrix() );
// A12 := inv(U11)' A12
A12_STAR_VR = A12;
LocalTrsm
( LEFT, UPPER, ADJOINT, diag, F(1), U11_STAR_STAR, A12_STAR_VR );
// A12 := A12 - 1/2 Y12
Axpy( F(-1)/F(2), Y12_STAR_VR, A12_STAR_VR );
// A22 := A22 - (A12' U12 + U12' A12)
A12_STAR_MR = A12_STAR_VR;
A12_STAR_VC = A12_STAR_VR;
U12_STAR_VC = U12_STAR_VR;
A12_STAR_MC = A12_STAR_VC;
U12_STAR_MC = U12_STAR_VC;
LocalTrr2k
( UPPER, ADJOINT, TRANSPOSE, ADJOINT,
F(-1), A12_STAR_MC, U12Trans_MR_STAR,
U12_STAR_MC, A12_STAR_MR,
F(1), A22 );
// A12 := A12 - 1/2 Y12
Axpy( F(-1)/F(2), Y12_STAR_VR, A12_STAR_VR );
A12 = A12_STAR_VR;
//--------------------------------------------------------------------//
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/**********************************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
}
}
inline void
LU( DistMatrix<F>& A, DistMatrix<int,VC,STAR>& p )
{
#ifndef RELEASE
CallStackEntry entry("LU");
if( A.Grid() != p.Grid() )
throw std::logic_error("{A,p} must be distributed over the same grid");
if( p.Viewing() &&
(std::min(A.Height(),A.Width()) != p.Height() || p.Width() != 1) )
throw std::logic_error
("p must be a vector of the same height as the min dimension of A.");
#endif
const Grid& g = A.Grid();
if( !p.Viewing() )
p.ResizeTo( std::min(A.Height(),A.Width()), 1 );
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g), AB(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<int,VC,STAR>
pT(g), p0(g),
pB(g), p1(g),
p2(g);
// Temporary distributions
DistMatrix<F, STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F, MC, STAR> A21_MC_STAR(g);
DistMatrix<F, STAR,VR > A12_STAR_VR(g);
DistMatrix<F, STAR,MR > A12_STAR_MR(g);
DistMatrix<int,STAR,STAR> p1_STAR_STAR(g);
// Pivot composition
std::vector<int> image, preimage;
// Start the algorithm
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
PartitionDown
( p, pT,
pB, 0 );
while( ATL.Height() < A.Height() && ATL.Width() < A.Width() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
RepartitionDown
( pT, p0,
/**/ /**/
p1,
pB, p2 );
View1x2( AB, ABL, ABR );
const int pivotOffset = A01.Height();
A12_STAR_VR.AlignWith( A22 );
A12_STAR_MR.AlignWith( A22 );
A21_MC_STAR.AlignWith( A22 );
A11_STAR_STAR.ResizeTo( A11.Height(), A11.Width() );
p1_STAR_STAR.ResizeTo( p1.Height(), 1 );
//--------------------------------------------------------------------//
A21_MC_STAR = A21;
A11_STAR_STAR = A11;
lu::Panel( A11_STAR_STAR, A21_MC_STAR, p1_STAR_STAR, pivotOffset );
ComposePivots( p1_STAR_STAR, pivotOffset, image, preimage );
ApplyRowPivots( AB, image, preimage );
// Perhaps we should give up perfectly distributing this operation since
// it's total contribution is only O(n^2)
A12_STAR_VR = A12;
LocalTrsm
( LEFT, LOWER, NORMAL, UNIT, F(1), A11_STAR_STAR, A12_STAR_VR );
A12_STAR_MR = A12_STAR_VR;
LocalGemm( NORMAL, NORMAL, F(-1), A21_MC_STAR, A12_STAR_MR, F(1), A22 );
A11 = A11_STAR_STAR;
A12 = A12_STAR_MR;
A21 = A21_MC_STAR;
p1 = p1_STAR_STAR;
//--------------------------------------------------------------------//
A12_STAR_VR.FreeAlignments();
A12_STAR_MR.FreeAlignments();
A21_MC_STAR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlidePartitionDown
( pT, p0,
p1,
/**/ /**/
pB, p2 );
}
}
inline void
internal::CholeskyUVar3Square( DistMatrix<F,MC,MR>& A )
{
#ifndef RELEASE
PushCallStack("internal::CholeskyUVar3Square");
if( A.Height() != A.Width() )
throw std::logic_error
("Can only compute Cholesky factor of square matrices.");
if( A.Grid().Height() != A.Grid().Width() )
throw std::logic_error
("CholeskyUVar3Square assumes a square process grid.");
#endif
const Grid& g = A.Grid();
// Find the process holding our transposed data
const int r = g.Height();
int transposeRank;
{
const int colAlignment = A.ColAlignment();
const int rowAlignment = A.RowAlignment();
const int colShift = A.ColShift();
const int rowShift = A.RowShift();
const int transposeRow = (colAlignment+rowShift) % r;
const int transposeCol = (rowAlignment+colShift) % r;
transposeRank = transposeRow + r*transposeCol;
}
const bool onDiagonal = ( transposeRank == g.VCRank() );
// Matrix views
DistMatrix<F,MC,MR>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
// Temporary matrix distributions
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,STAR,VR > A12_STAR_VR(g);
DistMatrix<F,STAR,MC > A12_STAR_MC(g);
DistMatrix<F,STAR,MR > A12_STAR_MR(g);
// Start the algorithm
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
while( ABR.Height() > 0 )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
A12_STAR_MC.AlignWith( A22 );
A12_STAR_MR.AlignWith( A22 );
A12_STAR_VR.AlignWith( A22 );
//--------------------------------------------------------------------//
A11_STAR_STAR = A11;
internal::LocalCholesky( UPPER, A11_STAR_STAR );
A11 = A11_STAR_STAR;
A12_STAR_VR = A12;
internal::LocalTrsm
( LEFT, UPPER, ADJOINT, NON_UNIT, (F)1, A11_STAR_STAR, A12_STAR_VR );
A12_STAR_MR = A12_STAR_VR;
// SendRecv to form A12[* ,MC] from A12[* ,MR]
A12_STAR_MC.ResizeTo( A12.Height(), A12.Width() );
{
if( onDiagonal )
{
const int size = A11.Height()*A22.LocalWidth();
MemCopy
( A12_STAR_MC.LocalBuffer(),
A12_STAR_MR.LocalBuffer(), size );
}
else
{
const int sendSize = A11.Height()*A22.LocalWidth();
const int recvSize = A11.Width()*A22.LocalHeight();
// We know that the ldim is the height since we have manually
// created both temporary matrices.
mpi::SendRecv
( A12_STAR_MR.LocalBuffer(), sendSize, transposeRank, 0,
A12_STAR_MC.LocalBuffer(), recvSize, transposeRank, 0,
g.VCComm() );
}
}
internal::LocalTrrk
( UPPER, ADJOINT, (F)-1, A12_STAR_MC, A12_STAR_MR, (F)1, A22 );
A12 = A12_STAR_MR;
//--------------------------------------------------------------------//
A12_STAR_MC.FreeAlignments();
A12_STAR_MR.FreeAlignments();
A12_STAR_VR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
TwoSidedTrmmUVar4
( UnitOrNonUnit diag, DistMatrix<F>& A, const DistMatrix<F>& U )
{
#ifndef RELEASE
PushCallStack("internal::TwoSidedTrmmUVar4");
if( A.Height() != A.Width() )
throw std::logic_error("A must be square");
if( U.Height() != U.Width() )
throw std::logic_error("Triangular matrices must be square");
if( A.Height() != U.Height() )
throw std::logic_error("A and U must be the same size");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F>
UTL(g), UTR(g), U00(g), U01(g), U02(g),
UBL(g), UBR(g), U10(g), U11(g), U12(g),
U20(g), U21(g), U22(g);
// Temporary distributions
DistMatrix<F,VC, STAR> A01_VC_STAR(g);
DistMatrix<F,VR, STAR> A01_VR_STAR(g);
DistMatrix<F,STAR,MC > A01Adj_STAR_MC(g);
DistMatrix<F,STAR,MR > A01Adj_STAR_MR(g);
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,STAR,VR > A12_STAR_VR(g);
DistMatrix<F,MR, STAR> A12Adj_MR_STAR(g);
DistMatrix<F,VC, STAR> U01_VC_STAR(g);
DistMatrix<F,VR, STAR> U01_VR_STAR(g);
DistMatrix<F,STAR,MC > U01Adj_STAR_MC(g);
DistMatrix<F,STAR,MR > U01Adj_STAR_MR(g);
DistMatrix<F,STAR,STAR> U11_STAR_STAR(g);
DistMatrix<F,VC, STAR> Y01_VC_STAR(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
A01_VC_STAR.AlignWith( A00 );
A01_VR_STAR.AlignWith( A00 );
A01Adj_STAR_MC.AlignWith( A00 );
A01Adj_STAR_MR.AlignWith( A00 );
A12Adj_MR_STAR.AlignWith( A02 );
U01_VC_STAR.AlignWith( A00 );
U01_VR_STAR.AlignWith( A00 );
U01Adj_STAR_MC.AlignWith( A00 );
U01Adj_STAR_MR.AlignWith( A00 );
Y01_VC_STAR.AlignWith( A01 );
//--------------------------------------------------------------------//
// Y01 := U01 A11
A11_STAR_STAR = A11;
U01_VC_STAR = U01;
Zeros( A01.Height(), A01.Width(), Y01_VC_STAR );
Hemm
( RIGHT, UPPER,
F(1), A11_STAR_STAR.LockedMatrix(), U01_VC_STAR.LockedMatrix(),
F(0), Y01_VC_STAR.Matrix() );
// A01 := A01 + 1/2 Y01
A01_VC_STAR = A01;
Axpy( F(1)/F(2), Y01_VC_STAR, A01_VC_STAR );
// A00 := A00 + (U01 A01' + A01 U01')
A01Adj_STAR_MC.AdjointFrom( A01_VC_STAR );
U01Adj_STAR_MC.AdjointFrom( U01_VC_STAR );
A01_VR_STAR = A01_VC_STAR;
U01_VR_STAR = U01_VC_STAR;
A01Adj_STAR_MR.AdjointFrom( A01_VR_STAR );
U01Adj_STAR_MR.AdjointFrom( U01_VR_STAR );
LocalTrr2k
( UPPER, ADJOINT, ADJOINT,
F(1), U01Adj_STAR_MC, A01Adj_STAR_MR,
A01Adj_STAR_MC, U01Adj_STAR_MR,
F(1), A00 );
// A01 := A01 + 1/2 Y01
Axpy( F(1)/F(2), Y01_VC_STAR, A01_VC_STAR );
// A01 := A01 U11'
U11_STAR_STAR = U11;
LocalTrmm
( RIGHT, UPPER, ADJOINT, diag, F(1), U11_STAR_STAR, A01_VC_STAR );
A01 = A01_VC_STAR;
// A02 := A02 + U01 A12
A12Adj_MR_STAR.AdjointFrom( A12 );
LocalGemm
( ADJOINT, ADJOINT, F(1), U01Adj_STAR_MC, A12Adj_MR_STAR, F(1), A02 );
// A11 := U11 A11 U11'
LocalTwoSidedTrmm( UPPER, diag, A11_STAR_STAR, U11_STAR_STAR );
A11 = A11_STAR_STAR;
// A12 := U11 A12
A12_STAR_VR.AdjointFrom( A12Adj_MR_STAR );
LocalTrmm
( LEFT, UPPER, NORMAL, diag, F(1), U11_STAR_STAR, A12_STAR_VR );
A12 = A12_STAR_VR;
//--------------------------------------------------------------------//
A01_VC_STAR.FreeAlignments();
A01_VR_STAR.FreeAlignments();
A01Adj_STAR_MC.FreeAlignments();
A01Adj_STAR_MR.FreeAlignments();
A12Adj_MR_STAR.FreeAlignments();
U01_VC_STAR.FreeAlignments();
U01_VR_STAR.FreeAlignments();
U01Adj_STAR_MC.FreeAlignments();
U01Adj_STAR_MR.FreeAlignments();
Y01_VC_STAR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/*************/ /******************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
internal::HegstLUVar2( DistMatrix<F,MC,MR>& A, const DistMatrix<F,MC,MR>& U )
{
#ifndef RELEASE
PushCallStack("internal::HegstLUVar2");
if( A.Height() != A.Width() )
throw std::logic_error("A must be square");
if( U.Height() != U.Width() )
throw std::logic_error("Triangular matrices must be square");
if( A.Height() != U.Height() )
throw std::logic_error("A and U must be the same size");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F,MC,MR>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F,MC,MR>
UTL(g), UTR(g), U00(g), U01(g), U02(g),
UBL(g), UBR(g), U10(g), U11(g), U12(g),
U20(g), U21(g), U22(g);
// Temporary distributions
DistMatrix<F,VC, STAR> A01_VC_STAR(g);
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,STAR,VR > A12_STAR_VR(g);
DistMatrix<F,STAR,STAR> U11_STAR_STAR(g);
DistMatrix<F,STAR,MC > U12_STAR_MC(g);
DistMatrix<F,STAR,VR > U12_STAR_VR(g);
DistMatrix<F,MR, STAR> U12Adj_MR_STAR(g);
DistMatrix<F,VC, STAR> U12Adj_VC_STAR(g);
DistMatrix<F,MC, STAR> X01_MC_STAR(g);
DistMatrix<F,STAR,STAR> X11_STAR_STAR(g);
DistMatrix<F,MC, MR > Y12(g);
DistMatrix<F,MC, MR > Z12Adj(g);
DistMatrix<F,MR, MC > Z12Adj_MR_MC(g);
DistMatrix<F,MC, STAR> Z12Adj_MC_STAR(g);
DistMatrix<F,MR, STAR> Z12Adj_MR_STAR(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
A12_STAR_VR.AlignWith( A12 );
U12_STAR_MC.AlignWith( A22 );
U12_STAR_VR.AlignWith( A12 );
U12Adj_MR_STAR.AlignWith( A22 );
U12Adj_VC_STAR.AlignWith( A22 );
X01_MC_STAR.AlignWith( A01 );
Y12.AlignWith( A12 );
Z12Adj.AlignWith( A12 );
Z12Adj_MR_MC.AlignWith( A12 );
Z12Adj_MC_STAR.AlignWith( A22 );
Z12Adj_MR_STAR.AlignWith( A22 );
//--------------------------------------------------------------------//
// A01 := A01 U11'
U11_STAR_STAR = U11;
A01_VC_STAR = A01;
internal::LocalTrmm
( RIGHT, UPPER, ADJOINT, NON_UNIT, (F)1, U11_STAR_STAR, A01_VC_STAR );
A01 = A01_VC_STAR;
// A01 := A01 + A02 U12'
U12Adj_MR_STAR.AdjointFrom( U12 );
X01_MC_STAR.ResizeTo( A01.Height(), A01.Width() );
internal::LocalGemm
( NORMAL, NORMAL,
(F)1, A02, U12Adj_MR_STAR, (F)0, X01_MC_STAR );
A01.SumScatterUpdate( (F)1, X01_MC_STAR );
// Y12 := U12 A22
U12Adj_VC_STAR = U12Adj_MR_STAR;
U12_STAR_MC.AdjointFrom( U12Adj_VC_STAR );
Z12Adj_MC_STAR.ResizeTo( A12.Width(), A12.Height() );
Z12Adj_MR_STAR.ResizeTo( A12.Width(), A12.Height() );
Zero( Z12Adj_MC_STAR );
Zero( Z12Adj_MR_STAR );
internal::LocalSymmetricAccumulateRU
( ADJOINT,
(F)1, A22, U12_STAR_MC, U12Adj_MR_STAR,
Z12Adj_MC_STAR, Z12Adj_MR_STAR );
Z12Adj.SumScatterFrom( Z12Adj_MC_STAR );
Z12Adj_MR_MC = Z12Adj;
Z12Adj_MR_MC.SumScatterUpdate( (F)1, Z12Adj_MR_STAR );
Y12.ResizeTo( A12.Height(), A12.Width() );
Adjoint( Z12Adj_MR_MC.LockedLocalMatrix(), Y12.LocalMatrix() );
// A12 := U11 A12
A12_STAR_VR = A12;
U11_STAR_STAR = U11;
internal::LocalTrmm
( LEFT, UPPER, NORMAL, NON_UNIT, (F)1, U11_STAR_STAR, A12_STAR_VR );
A12 = A12_STAR_VR;
// A12 := A12 + 1/2 Y12
Axpy( (F)0.5, Y12, A12 );
// A11 := U11 A11 U11'
A11_STAR_STAR = A11;
internal::LocalHegst( LEFT, UPPER, A11_STAR_STAR, U11_STAR_STAR );
A11 = A11_STAR_STAR;
// A11 := A11 + (A12 U12' + U12 A12')
A12_STAR_VR = A12;
U12_STAR_VR = U12;
X11_STAR_STAR.ResizeTo( A11.Height(), A11.Width() );
Her2k
( UPPER, NORMAL,
(F)1, A12_STAR_VR.LocalMatrix(), U12_STAR_VR.LocalMatrix(),
(F)0, X11_STAR_STAR.LocalMatrix() );
A11.SumScatterUpdate( (F)1, X11_STAR_STAR );
// A12 := A12 + 1/2 Y12
Axpy( (F)0.5, Y12, A12 );
//--------------------------------------------------------------------//
A12_STAR_VR.FreeAlignments();
U12_STAR_MC.FreeAlignments();
U12_STAR_VR.FreeAlignments();
U12Adj_MR_STAR.FreeAlignments();
U12Adj_VC_STAR.FreeAlignments();
X01_MC_STAR.FreeAlignments();
Y12.FreeAlignments();
Z12Adj.FreeAlignments();
Z12Adj_MR_MC.FreeAlignments();
Z12Adj_MC_STAR.FreeAlignments();
Z12Adj_MR_STAR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/*************/ /******************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
TwoSidedTrsmUVar2
( UnitOrNonUnit diag, DistMatrix<F>& A, const DistMatrix<F>& U )
{
#ifndef RELEASE
CallStackEntry entry("internal::TwoSidedTrsmUVar2");
if( A.Height() != A.Width() )
LogicError("A must be square");
if( U.Height() != U.Width() )
LogicError("Triangular matrices must be square");
if( A.Height() != U.Height() )
LogicError("A and U must be the same size");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F>
UTL(g), UTR(g), U00(g), U01(g), U02(g),
UBL(g), UBR(g), U10(g), U11(g), U12(g),
U20(g), U21(g), U22(g);
// Temporary distributions
DistMatrix<F,MC, STAR> A01_MC_STAR(g);
DistMatrix<F,VC, STAR> A01_VC_STAR(g);
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,STAR,VR > A12_STAR_VR(g);
DistMatrix<F,MC, STAR> F01_MC_STAR(g);
DistMatrix<F,MC, STAR> U01_MC_STAR(g);
DistMatrix<F,VR, STAR> U01_VR_STAR(g);
DistMatrix<F,STAR,MR > U01Adj_STAR_MR(g);
DistMatrix<F,STAR,STAR> U11_STAR_STAR(g);
DistMatrix<F,STAR,MR > X11_STAR_MR(g);
DistMatrix<F,MR, STAR> X12Adj_MR_STAR(g);
DistMatrix<F,MR, MC > X12Adj_MR_MC(g);
DistMatrix<F,MR, MC > Y01_MR_MC(g);
DistMatrix<F,MR, STAR> Y01_MR_STAR(g);
DistMatrix<F> X11(g);
DistMatrix<F> Y01(g);
Matrix<F> X12Local;
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
A01_MC_STAR.AlignWith( U01 );
Y01.AlignWith( A01 );
Y01_MR_STAR.AlignWith( A00 );
U01_MC_STAR.AlignWith( A00 );
U01_VR_STAR.AlignWith( A00 );
U01Adj_STAR_MR.AlignWith( A00 );
X11_STAR_MR.AlignWith( U01 );
X11.AlignWith( A11 );
X12Adj_MR_STAR.AlignWith( A02 );
X12Adj_MR_MC.AlignWith( A12 );
F01_MC_STAR.AlignWith( A00 );
//--------------------------------------------------------------------//
// Y01 := A00 U01
U01_MC_STAR = U01;
U01_VR_STAR = U01_MC_STAR;
U01Adj_STAR_MR.AdjointFrom( U01_VR_STAR );
Zeros( Y01_MR_STAR, A01.Height(), A01.Width() );
Zeros( F01_MC_STAR, A01.Height(), A01.Width() );
LocalSymmetricAccumulateLU
( ADJOINT,
F(1), A00, U01_MC_STAR, U01Adj_STAR_MR, F01_MC_STAR, Y01_MR_STAR );
Y01_MR_MC.SumScatterFrom( Y01_MR_STAR );
Y01 = Y01_MR_MC;
Y01.SumScatterUpdate( F(1), F01_MC_STAR );
// X11 := U01' A01
LocalGemm( ADJOINT, NORMAL, F(1), U01_MC_STAR, A01, X11_STAR_MR );
// A01 := A01 - Y01
Axpy( F(-1), Y01, A01 );
A01_MC_STAR = A01;
// A11 := A11 - triu(X11 + A01' U01) = A11 - (U01 A01 + A01' U01)
LocalGemm( ADJOINT, NORMAL, F(1), A01_MC_STAR, U01, F(1), X11_STAR_MR );
X11.SumScatterFrom( X11_STAR_MR );
MakeTriangular( UPPER, X11 );
Axpy( F(-1), X11, A11 );
// A01 := A01 inv(U11)
U11_STAR_STAR = U11;
A01_VC_STAR = A01_MC_STAR;
LocalTrsm
( RIGHT, UPPER, NORMAL, diag, F(1), U11_STAR_STAR, A01_VC_STAR );
A01 = A01_VC_STAR;
// A11 := inv(U11)' A11 inv(U11)
A11_STAR_STAR = A11;
LocalTwoSidedTrsm( UPPER, diag, A11_STAR_STAR, U11_STAR_STAR );
A11 = A11_STAR_STAR;
// A12 := A12 - A02' U01
LocalGemm( ADJOINT, NORMAL, F(1), A02, U01_MC_STAR, X12Adj_MR_STAR );
X12Adj_MR_MC.SumScatterFrom( X12Adj_MR_STAR );
Adjoint( X12Adj_MR_MC.LockedMatrix(), X12Local );
Axpy( F(-1), X12Local, A12.Matrix() );
// A12 := inv(U11)' A12
A12_STAR_VR = A12;
LocalTrsm
( LEFT, UPPER, ADJOINT, diag, F(1), U11_STAR_STAR, A12_STAR_VR );
A12 = A12_STAR_VR;
//--------------------------------------------------------------------//
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/*************/ /******************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
}
}
inline void
RowEchelon( DistMatrix<F>& A, DistMatrix<F>& B )
{
#ifndef RELEASE
CallStackEntry entry("RowEchelon");
if( A.Grid() != B.Grid() )
LogicError("{A,B} must be distributed over the same grid");
if( A.Height() != B.Height() )
LogicError("A and B must be the same height");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g), APan(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F>
BT(g), B0(g),
BB(g), B1(g),
B2(g);
// Temporary distributions
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,STAR,VR > A12_STAR_VR(g);
DistMatrix<F,STAR,MR > A12_STAR_MR(g);
DistMatrix<F,MC, STAR> A21_MC_STAR(g);
DistMatrix<F,STAR,VR > B1_STAR_VR(g);
DistMatrix<F,STAR,MR > B1_STAR_MR(g);
DistMatrix<Int,STAR,STAR> p1_STAR_STAR(g);
// In case B's columns are not aligned with A's
const bool BAligned = ( B.ColShift() == A.ColShift() );
DistMatrix<F,MC,STAR> A21_MC_STAR_B(g);
// Pivot composition
std::vector<Int> image, preimage;
// Start the algorithm
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
PartitionDown
( B, BT,
BB, 0 );
while( ATL.Height() < A.Height() && ATL.Width() < A.Width() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
RepartitionDown
( BT, B0,
/**/ /**/
B1,
BB, B2 );
View2x1
( APan, A12,
A22 );
A12_STAR_VR.AlignWith( A22 );
A12_STAR_MR.AlignWith( A22 );
A21_MC_STAR.AlignWith( A22 );
B1_STAR_VR.AlignWith( B1 );
B1_STAR_MR.AlignWith( B1 );
if( ! BAligned )
A21_MC_STAR_B.AlignWith( B2 );
p1_STAR_STAR.ResizeTo( A11.Height(), 1 );
//--------------------------------------------------------------------//
A11_STAR_STAR = A11;
A21_MC_STAR = A21;
lu::Panel( A11_STAR_STAR, A21_MC_STAR, p1_STAR_STAR, A00.Height() );
ComposePivots( p1_STAR_STAR, A00.Height(), image, preimage );
ApplyRowPivots( APan, image, preimage );
ApplyRowPivots( BB, image, preimage );
A12_STAR_VR = A12;
B1_STAR_VR = B1;
LocalTrsm
( LEFT, LOWER, NORMAL, UNIT, F(1), A11_STAR_STAR, A12_STAR_VR );
LocalTrsm( LEFT, LOWER, NORMAL, UNIT, F(1), A11_STAR_STAR, B1_STAR_VR );
A12_STAR_MR = A12_STAR_VR;
B1_STAR_MR = B1_STAR_VR;
LocalGemm( NORMAL, NORMAL, F(-1), A21_MC_STAR, A12_STAR_MR, F(1), A22 );
if( BAligned )
{
LocalGemm
( NORMAL, NORMAL, F(-1), A21_MC_STAR, B1_STAR_MR, F(1), B2 );
}
else
{
A21_MC_STAR_B = A21_MC_STAR;
LocalGemm
( NORMAL, NORMAL, F(-1), A21_MC_STAR_B, B1_STAR_MR, F(1), B2 );
}
A11 = A11_STAR_STAR;
A12 = A12_STAR_MR;
B1 = B1_STAR_MR;
//--------------------------------------------------------------------//
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlidePartitionDown
( BT, B0,
B1,
/**/ /**/
BB, B2 );
}
}
