inline void
TwoSidedTrsmUVar4
( UnitOrNonUnit diag, DistMatrix<F>& A, const DistMatrix<F>& U )
{
#ifndef RELEASE
CallStackEntry entry("internal::TwoSidedTrsmUVar4");
if( A.Height() != A.Width() )
LogicError("A must be square");
if( U.Height() != U.Width() )
LogicError("Triangular matrices must be square");
if( A.Height() != U.Height() )
LogicError("A and U must be the same size");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F>
UTL(g), UTR(g), U00(g), U01(g), U02(g),
UBL(g), UBR(g), U10(g), U11(g), U12(g),
U20(g), U21(g), U22(g);
// Temporary distributions
DistMatrix<F,VC, STAR> A01_VC_STAR(g);
DistMatrix<F,STAR,MC > A01Trans_STAR_MC(g);
DistMatrix<F,STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F,STAR,VR > A12_STAR_VR(g);
DistMatrix<F,STAR,VC > A12_STAR_VC(g);
DistMatrix<F,STAR,MC > A12_STAR_MC(g);
DistMatrix<F,STAR,MR > A12_STAR_MR(g);
DistMatrix<F,STAR,STAR> U11_STAR_STAR(g);
DistMatrix<F,MR, STAR> U12Trans_MR_STAR(g);
DistMatrix<F,VR, STAR> U12Trans_VR_STAR(g);
DistMatrix<F,STAR,VR > U12_STAR_VR(g);
DistMatrix<F,STAR,VC > U12_STAR_VC(g);
DistMatrix<F,STAR,MC > U12_STAR_MC(g);
DistMatrix<F,STAR,VR > Y12_STAR_VR(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionDownDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
while( ATL.Height() < A.Height() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionDownDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
A01_VC_STAR.AlignWith( A02 );
A01Trans_STAR_MC.AlignWith( A02 );
A12_STAR_VR.AlignWith( A22 );
A12_STAR_VC.AlignWith( A22 );
A12_STAR_MC.AlignWith( A22 );
A12_STAR_MR.AlignWith( A22 );
U12Trans_MR_STAR.AlignWith( A02 );
U12Trans_VR_STAR.AlignWith( A02 );
U12_STAR_VR.AlignWith( A02 );
U12_STAR_VC.AlignWith( A22 );
U12_STAR_MC.AlignWith( A22 );
Y12_STAR_VR.AlignWith( A12 );
//--------------------------------------------------------------------//
// A01 := A01 inv(U11)
A01_VC_STAR = A01;
U11_STAR_STAR = U11;
LocalTrsm
( RIGHT, UPPER, NORMAL, diag, F(1), U11_STAR_STAR, A01_VC_STAR );
A01 = A01_VC_STAR;
// A11 := inv(U11)' A11 inv(U11)
A11_STAR_STAR = A11;
LocalTwoSidedTrsm( UPPER, diag, A11_STAR_STAR, U11_STAR_STAR );
A11 = A11_STAR_STAR;
// A02 := A02 - A01 U12
A01Trans_STAR_MC.TransposeFrom( A01_VC_STAR );
U12Trans_MR_STAR.TransposeFrom( U12 );
LocalGemm
( TRANSPOSE, TRANSPOSE,
F(-1), A01Trans_STAR_MC, U12Trans_MR_STAR, F(1), A02 );
// Y12 := A11 U12
U12Trans_VR_STAR = U12Trans_MR_STAR;
Zeros( U12_STAR_VR, A12.Height(), A12.Width() );
Transpose( U12Trans_VR_STAR.Matrix(), U12_STAR_VR.Matrix() );
Zeros( Y12_STAR_VR, A12.Height(), A12.Width() );
Hemm
( LEFT, UPPER,
F(1), A11_STAR_STAR.Matrix(), U12_STAR_VR.Matrix(),
F(0), Y12_STAR_VR.Matrix() );
// A12 := inv(U11)' A12
A12_STAR_VR = A12;
LocalTrsm
( LEFT, UPPER, ADJOINT, diag, F(1), U11_STAR_STAR, A12_STAR_VR );
// A12 := A12 - 1/2 Y12
Axpy( F(-1)/F(2), Y12_STAR_VR, A12_STAR_VR );
// A22 := A22 - (A12' U12 + U12' A12)
A12_STAR_MR = A12_STAR_VR;
A12_STAR_VC = A12_STAR_VR;
U12_STAR_VC = U12_STAR_VR;
A12_STAR_MC = A12_STAR_VC;
U12_STAR_MC = U12_STAR_VC;
LocalTrr2k
( UPPER, ADJOINT, TRANSPOSE, ADJOINT,
F(-1), A12_STAR_MC, U12Trans_MR_STAR,
U12_STAR_MC, A12_STAR_MR,
F(1), A22 );
// A12 := A12 - 1/2 Y12
Axpy( F(-1)/F(2), Y12_STAR_VR, A12_STAR_VR );
A12 = A12_STAR_VR;
//--------------------------------------------------------------------//
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionDownDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/**********************************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
}
}
inline void
UVar3( UnitOrNonUnit diag, DistMatrix<F>& U )
{
#ifndef RELEASE
CallStackEntry entry("triangular_inverse::UVar3");
if( U.Height() != U.Width() )
LogicError("Nonsquare matrices cannot be triangular");
#endif
const Grid& g = U.Grid();
// Matrix views
DistMatrix<F>
UTL(g), UTR(g), U00(g), U01(g), U02(g),
UBL(g), UBR(g), U10(g), U11(g), U12(g),
U20(g), U21(g), U22(g);
// Temporary distributions
DistMatrix<F,VC, STAR> U01_VC_STAR(g);
DistMatrix<F,STAR,STAR> U11_STAR_STAR(g);
DistMatrix<F,STAR,VR > U12_STAR_VR(g);
DistMatrix<F,STAR,MC > U01Trans_STAR_MC(g);
DistMatrix<F,MR, STAR> U12Trans_MR_STAR(g);
// Start the algorithm
PartitionUpDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
while( UBR.Height() < U.Height() )
{
RepartitionUpDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/*************/ /******************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
U01Trans_STAR_MC.AlignWith( U02 );
U12Trans_MR_STAR.AlignWith( U02 );
//--------------------------------------------------------------------//
U01_VC_STAR = U01;
U11_STAR_STAR = U11;
LocalTrsm
( RIGHT, UPPER, NORMAL, diag, F(-1), U11_STAR_STAR, U01_VC_STAR );
// We transpose before the communication to avoid cache-thrashing
// in the unpacking stage.
U12Trans_MR_STAR.TransposeFrom( U12 );
U01Trans_STAR_MC.TransposeFrom( U01_VC_STAR );
LocalGemm
( TRANSPOSE, TRANSPOSE,
F(1), U01Trans_STAR_MC, U12Trans_MR_STAR, F(1), U02 );
U01.TransposeFrom( U01Trans_STAR_MC );
U12_STAR_VR.TransposeFrom( U12Trans_MR_STAR );
LocalTrsm
( LEFT, UPPER, NORMAL, diag, F(1), U11_STAR_STAR, U12_STAR_VR );
LocalTriangularInverse( UPPER, diag, U11_STAR_STAR );
U11 = U11_STAR_STAR;
U12 = U12_STAR_VR;
//--------------------------------------------------------------------//
SlidePartitionUpDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
}
}
inline void
TrmmRUNC
( UnitOrNonUnit diag,
T alpha, const DistMatrix<T>& U,
DistMatrix<T>& X )
{
#ifndef RELEASE
CallStackEntry entry("internal::TrmmRUNC");
if( U.Grid() != X.Grid() )
throw std::logic_error
("U and X must be distributed over the same grid");
if( U.Height() != U.Width() || X.Width() != U.Height() )
{
std::ostringstream msg;
msg << "Nonconformal TrmmRUNC: \n"
<< " U ~ " << U.Height() << " x " << U.Width() << "\n"
<< " X ~ " << X.Height() << " x " << X.Width() << "\n";
throw std::logic_error( msg.str().c_str() );
}
#endif
const Grid& g = U.Grid();
// Matrix views
DistMatrix<T>
UTL(g), UTR(g), U00(g), U01(g), U02(g),
UBL(g), UBR(g), U10(g), U11(g), U12(g),
U20(g), U21(g), U22(g);
DistMatrix<T> XL(g), XR(g),
X0(g), X1(g), X2(g);
// Temporary distributions
DistMatrix<T,MR, STAR> U12Trans_MR_STAR(g);
DistMatrix<T,STAR,STAR> U11_STAR_STAR(g);
DistMatrix<T,VC, STAR> X1_VC_STAR(g);
DistMatrix<T,MC, STAR> X1_MC_STAR(g);
// Start the algorithm
Scale( alpha, X );
LockedPartitionUpDiagonal
( U, UTL, UTR,
UBL, UBR, 0 );
PartitionLeft( X, XL, XR, 0 );
while( XL.Width() > 0 )
{
LockedRepartitionUpDiagonal
( UTL, /**/ UTR, U00, U01, /**/ U02,
/**/ U10, U11, /**/ U12,
/*************/ /******************/
UBL, /**/ UBR, U20, U21, /**/ U22 );
RepartitionLeft
( XL, /**/ XR,
X0, X1, /**/ X2 );
X1_MC_STAR.AlignWith( X2 );
U12Trans_MR_STAR.AlignWith( X2 );
X1_VC_STAR.AlignWith( X1 );
//--------------------------------------------------------------------//
X1_MC_STAR = X1;
U12Trans_MR_STAR.TransposeFrom( U12 );
LocalGemm
( NORMAL, TRANSPOSE, T(1), X1_MC_STAR, U12Trans_MR_STAR, T(1), X2 );
U11_STAR_STAR = U11;
X1_VC_STAR = X1_MC_STAR;
LocalTrmm
( RIGHT, UPPER, NORMAL, diag, T(1), U11_STAR_STAR, X1_VC_STAR );
X1 = X1_VC_STAR;
//--------------------------------------------------------------------//
X1_MC_STAR.FreeAlignments();
U12Trans_MR_STAR.FreeAlignments();
X1_VC_STAR.FreeAlignments();
SlideLockedPartitionUpDiagonal
( UTL, /**/ UTR, U00, /**/ U01, U02,
/*************/ /******************/
/**/ U10, /**/ U11, U12,
UBL, /**/ UBR, U20, /**/ U21, U22 );
SlidePartitionLeft
( XL, /**/ XR,
X0, /**/ X1, X2 );
}
}
