inline void
SlideLockedPartitionRight
( DM& AL, DM& AR,
const DM& A0, const DM& A1, const DM& A2 )
{
DEBUG_ONLY(CallStackEntry cse("SlideLockedPartitionRight"))
LockedView1x2( AL, A0, A1 );
LockedView( AR, A2 );
}
inline void
SlideLockedPartitionLeft
( M& AL, M& AR,
const M& A0, const M& A1, const M& A2 )
{
DEBUG_ONLY(CallStackEntry cse("SlideLockedPartitionLeft"))
LockedView( AL, A0 );
LockedView1x2( AR, A1, A2 );
}
inline void
SlideLockedPartitionDownDiagonal
( DM& ATL, DM& ATR, const DM& A00, const DM& A01, const DM& A02,
const DM& A10, const DM& A11, const DM& A12,
DM& ABL, DM& ABR, const DM& A20, const DM& A21, const DM& A22 )
{
DEBUG_ONLY(CallStackEntry cse("SlideLockedPartitionDownDiagonal"))
LockedView2x2( ATL, A00, A01,
A10, A11 );
LockedView2x1( ATR, A02, A12 );
LockedView1x2( ABL, A20, A21 );
LockedView( ABR, A22 );
}
inline void
SymmLLC
( T alpha, const DistMatrix<T>& A, const DistMatrix<T>& B,
T beta, DistMatrix<T>& C )
{
#ifndef RELEASE
PushCallStack("internal::SymmLLC");
if( A.Grid() != B.Grid() || B.Grid() != C.Grid() )
throw std::logic_error
("{A,B,C} must be distributed over the same grid");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<T>
ATL(g), ATR(g), A00(g), A01(g), A02(g), AColPan(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g), ARowPan(g),
A20(g), A21(g), A22(g);
DistMatrix<T>
BT(g), B0(g),
BB(g), B1(g),
B2(g);
DistMatrix<T>
CT(g), C0(g), CAbove(g),
CB(g), C1(g), CBelow(g),
C2(g);
// Temporary distributions
DistMatrix<T,MC, STAR> AColPan_MC_STAR(g);
DistMatrix<T,STAR,MC > ARowPan_STAR_MC(g);
DistMatrix<T,MR, STAR> B1Trans_MR_STAR(g);
B1Trans_MR_STAR.AlignWith( C );
// Start the algorithm
Scale( beta, C );
LockedPartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDown
( B, BT,
BB, 0 );
PartitionDown
( C, CT,
CB, 0 );
while( CB.Height() > 0 )
{
LockedRepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDown
( BT, B0,
/**/ /**/
B1,
BB, B2 );
RepartitionDown
( CT, C0,
/**/ /**/
C1,
CB, C2 );
LockedView1x2( ARowPan, A10, A11 );
LockedView2x1
( AColPan, A11,
A21 );
View2x1
( CAbove, C0,
C1 );
View2x1
( CBelow, C1,
C2 );
AColPan_MC_STAR.AlignWith( CBelow );
ARowPan_STAR_MC.AlignWith( CAbove );
//--------------------------------------------------------------------//
AColPan_MC_STAR = AColPan;
ARowPan_STAR_MC = ARowPan;
MakeTrapezoidal( LEFT, LOWER, 0, AColPan_MC_STAR );
MakeTrapezoidal( RIGHT, LOWER, -1, ARowPan_STAR_MC );
B1Trans_MR_STAR.TransposeFrom( B1 );
LocalGemm
( NORMAL, TRANSPOSE,
alpha, AColPan_MC_STAR, B1Trans_MR_STAR, T(1), CBelow );
LocalGemm
( TRANSPOSE, TRANSPOSE,
alpha, ARowPan_STAR_MC, B1Trans_MR_STAR, T(1), CAbove );
//--------------------------------------------------------------------//
AColPan_MC_STAR.FreeAlignments();
ARowPan_STAR_MC.FreeAlignments();
SlideLockedPartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDown
( BT, B0,
B1,
/**/ /**/
BB, B2 );
SlidePartitionDown
( CT, C0,
C1,
/**/ /**/
CB, C2 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
LUHF
( Conjugation conjugation, Int offset,
const Matrix<F>& H, const Matrix<F>& t, Matrix<F>& A )
{
#ifndef RELEASE
CallStackEntry cse("apply_packed_reflectors::LUHF");
// TODO: Proper dimension checks
if( t.Height() != H.DiagonalLength(offset) )
LogicError("t must be the same length as H's offset diag");
#endif
Matrix<F>
HTL, HTR, H00, H01, H02, HPan, HPanCopy,
HBL, HBR, H10, H11, H12,
H20, H21, H22;
Matrix<F>
AT, A0,
AB, A1,
A2;
Matrix<F>
tT, t0,
tB, t1,
t2;
Matrix<F> SInv, Z;
LockedPartitionDownOffsetDiagonal
( offset,
H, HTL, HTR,
HBL, HBR, 0 );
LockedPartitionDown
( t, tT,
tB, 0 );
PartitionDown
( A, AT,
AB, 0 );
while( HTL.Height() < H.Height() && HTL.Width() < H.Width() )
{
LockedRepartitionDownDiagonal
( HTL, /**/ HTR, H00, /**/ H01, H02,
/*************/ /******************/
/**/ H10, /**/ H11, H12,
HBL, /**/ HBR, H20, /**/ H21, H22 );
LockedRepartitionDown
( tT, t0,
/**/ /**/
t1,
tB, t2 );
RepartitionDown
( AT, A0,
/**/ /**/
A1,
AB, A2, H11.Height() );
LockedView1x2( HPan, H11, H12 );
//--------------------------------------------------------------------//
HPanCopy = HPan;
MakeTriangular( UPPER, HPanCopy );
SetDiagonal( HPanCopy, F(1) );
Herk( LOWER, NORMAL, F(1), HPanCopy, SInv );
FixDiagonal( conjugation, t1, SInv );
Gemm( NORMAL, NORMAL, F(1), HPanCopy, AB, Z );
Trsm( LEFT, LOWER, NORMAL, NON_UNIT, F(1), SInv, Z );
Gemm( ADJOINT, NORMAL, F(-1), HPanCopy, Z, F(1), AB );
//--------------------------------------------------------------------//
SlideLockedPartitionDownDiagonal
( HTL, /**/ HTR, H00, H01, /**/ H02,
/**/ H10, H11, /**/ H12,
/*************/ /******************/
HBL, /**/ HBR, H20, H21, /**/ H22 );
SlideLockedPartitionDown
( tT, t0,
t1,
/**/ /**/
tB, t2 );
SlidePartitionDown
( AT, A0,
A1,
/**/ /**/
AB, A2 );
}
}
inline void
LUHF
( Conjugation conjugation, Int offset,
const DistMatrix<F>& H, const DistMatrix<F,MD,STAR>& t, DistMatrix<F>& A )
{
#ifndef RELEASE
CallStackEntry cse("apply_packed_reflectors::LUHF");
if( H.Grid() != t.Grid() || t.Grid() != A.Grid() )
LogicError("{H,t,A} must be distributed over the same grid");
// TODO: Proper dimension checks
if( t.Height() != H.DiagonalLength(offset) )
LogicError("t must be the same length as H's offset diag");
if( !t.AlignedWithDiagonal( H, offset ) )
LogicError("t must be aligned with H's offset diagonal");
#endif
const Grid& g = H.Grid();
DistMatrix<F>
HTL(g), HTR(g), H00(g), H01(g), H02(g), HPan(g), HPanCopy(g),
HBL(g), HBR(g), H10(g), H11(g), H12(g),
H20(g), H21(g), H22(g);
DistMatrix<F>
AT(g), A0(g),
AB(g), A1(g),
A2(g);
DistMatrix<F,MD,STAR>
tT(g), t0(g),
tB(g), t1(g),
t2(g);
DistMatrix<F,STAR,VR > HPan_STAR_VR(g);
DistMatrix<F,STAR,MC > HPan_STAR_MC(g);
DistMatrix<F,STAR,STAR> t1_STAR_STAR(g);
DistMatrix<F,STAR,STAR> SInv_STAR_STAR(g);
DistMatrix<F,STAR,MR > Z_STAR_MR(g);
DistMatrix<F,STAR,VR > Z_STAR_VR(g);
LockedPartitionDownOffsetDiagonal
( offset,
H, HTL, HTR,
HBL, HBR, 0 );
LockedPartitionDown
( t, tT,
tB, 0 );
PartitionDown
( A, AT,
AB, 0 );
while( HTL.Height() < H.Height() && HTL.Width() < H.Width() )
{
LockedRepartitionDownDiagonal
( HTL, /**/ HTR, H00, /**/ H01, H02,
/*************/ /******************/
/**/ H10, /**/ H11, H12,
HBL, /**/ HBR, H20, /**/ H21, H22 );
LockedRepartitionDown
( tT, t0,
/**/ /**/
t1,
tB, t2 );
RepartitionDown
( AT, A0,
/**/ /**/
A1,
AB, A2, H11.Height() );
LockedView1x2( HPan, H11, H12 );
HPan_STAR_MC.AlignWith( AB );
Z_STAR_MR.AlignWith( AB );
Z_STAR_VR.AlignWith( AB );
//--------------------------------------------------------------------//
HPanCopy = HPan;
MakeTriangular( UPPER, HPanCopy );
SetDiagonal( HPanCopy, F(1) );
HPan_STAR_VR = HPanCopy;
Zeros( SInv_STAR_STAR, HPan.Height(), HPan.Height() );
Herk
( LOWER, NORMAL,
F(1), HPan_STAR_VR.LockedMatrix(),
F(0), SInv_STAR_STAR.Matrix() );
SInv_STAR_STAR.SumOverGrid();
t1_STAR_STAR = t1;
FixDiagonal( conjugation, t1_STAR_STAR, SInv_STAR_STAR );
HPan_STAR_MC = HPan_STAR_VR;
LocalGemm( NORMAL, NORMAL, F(1), HPan_STAR_MC, AB, Z_STAR_MR );
Z_STAR_VR.SumScatterFrom( Z_STAR_MR );
LocalTrsm
( LEFT, LOWER, NORMAL, NON_UNIT, F(1), SInv_STAR_STAR, Z_STAR_VR );
Z_STAR_MR = Z_STAR_VR;
LocalGemm( ADJOINT, NORMAL, F(-1), HPan_STAR_MC, Z_STAR_MR, F(1), AB );
//--------------------------------------------------------------------//
SlideLockedPartitionDownDiagonal
( HTL, /**/ HTR, H00, H01, /**/ H02,
/**/ H10, H11, /**/ H12,
/*************/ /******************/
HBL, /**/ HBR, H20, H21, /**/ H22 );
SlideLockedPartitionDown
( tT, t0,
t1,
/**/ /**/
tB, t2 );
SlidePartitionDown
( AT, A0,
A1,
/**/ /**/
AB, A2 );
}
}
inline void
SymmRLC
( T alpha, const DistMatrix<T>& A, const DistMatrix<T>& B,
T beta, DistMatrix<T>& C )
{
#ifndef RELEASE
PushCallStack("internal::SymmRLC");
if( A.Grid() != B.Grid() || B.Grid() != C.Grid() )
throw std::logic_error
("{A,B,C} must be distributed over the same grid");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<T>
ATL(g), ATR(g), A00(g), A01(g), A02(g), AColPan(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g), ARowPan(g),
A20(g), A21(g), A22(g);
DistMatrix<T> BL(g), BR(g),
B0(g), B1(g), B2(g);
DistMatrix<T> CL(g), CR(g),
C0(g), C1(g), C2(g),
CLeft(g), CRight(g);
// Temporary distributions
DistMatrix<T,MC, STAR> B1_MC_STAR(g);
DistMatrix<T,VR, STAR> AColPan_VR_STAR(g);
DistMatrix<T,STAR,MR > AColPanTrans_STAR_MR(g);
DistMatrix<T,MR, STAR> ARowPanTrans_MR_STAR(g);
B1_MC_STAR.AlignWith( C );
// Start the algorithm
Scale( beta, C );
LockedPartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionRight( B, BL, BR, 0 );
PartitionRight( C, CL, CR, 0 );
while( CR.Width() > 0 )
{
LockedRepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionRight
( BL, /**/ BR,
B0, /**/ B1, B2 );
RepartitionRight
( CL, /**/ CR,
C0, /**/ C1, C2 );
LockedView1x2( ARowPan, A10, A11 );
LockedView2x1
( AColPan, A11,
A21 );
View1x2( CLeft, C0, C1 );
View1x2( CRight, C1, C2 );
AColPan_VR_STAR.AlignWith( CRight );
AColPanTrans_STAR_MR.AlignWith( CRight );
ARowPanTrans_MR_STAR.AlignWith( CLeft );
//--------------------------------------------------------------------//
B1_MC_STAR = B1;
ARowPanTrans_MR_STAR.TransposeFrom( ARowPan );
AColPan_VR_STAR = AColPan;
AColPanTrans_STAR_MR.TransposeFrom( AColPan_VR_STAR );
MakeTrapezoidal( RIGHT, UPPER, 0, ARowPanTrans_MR_STAR );
MakeTrapezoidal( LEFT, UPPER, 1, AColPanTrans_STAR_MR );
LocalGemm
( NORMAL, TRANSPOSE,
alpha, B1_MC_STAR, ARowPanTrans_MR_STAR, T(1), CLeft );
LocalGemm
( NORMAL, NORMAL,
alpha, B1_MC_STAR, AColPanTrans_STAR_MR, T(1), CRight );
//--------------------------------------------------------------------//
AColPan_VR_STAR.FreeAlignments();
AColPanTrans_STAR_MR.FreeAlignments();
ARowPanTrans_MR_STAR.FreeAlignments();
SlideLockedPartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionRight
( BL, /**/ BR,
B0, B1, /**/ B2 );
SlidePartitionRight
( CL, /**/ CR,
C0, C1, /**/ C2 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
