inline void
Householder( Matrix<F>& A, Matrix<F>& t )
{
#ifndef RELEASE
CallStackEntry entry("rq::Householder");
#endif
t.ResizeTo( Min(A.Height(),A.Width()), 1 );
// Matrix views
Matrix<F>
ATL, ATR, A00, A01, A02, ATopPan,
ABL, ABR, A10, A11, A12, ABottomPan,
A20, A21, A22;
Matrix<F>
tT, t0,
tB, t1,
t2;
PartitionUpOffsetDiagonal
( A.Width()-A.Height(),
A, ATL, ATR,
ABL, ABR, 0 );
PartitionUp
( t, tT,
tB, 0 );
while( ABR.Height() < A.Height() && ABR.Width() < A.Width() )
{
RepartitionUpDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
RepartitionUp
( tT, t0,
t1,
/**/ /**/
tB, t2 );
View1x2( ATopPan, A00, A01 );
View1x2( ABottomPan, A10, A11 );
//--------------------------------------------------------------------//
PanelHouseholder( ABottomPan, t1 );
ApplyQ( RIGHT, ADJOINT, ABottomPan, t1, ATopPan );
//--------------------------------------------------------------------//
SlidePartitionUp
( tT, t0,
/**/ /**/
t1,
tB, t2 );
SlidePartitionUpDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
}
}
inline void
PanelLQ( Matrix<Real>& A )
{
#ifndef RELEASE
PushCallStack("internal::PanelLQ");
#endif
Matrix<Real>
ATL, ATR, A00, a01, A02, aTopRow, ABottomPan,
ABL, ABR, a10, alpha11, a12,
A20, a21, A22;
Matrix<Real> z;
PushBlocksizeStack( 1 );
PartitionDownLeftDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
while( ATL.Height() < A.Height() && ATL.Width() < A.Width() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ a01, A02,
/*************/ /**********************/
/**/ a10, /**/ alpha11, a12,
ABL, /**/ ABR, A20, /**/ a21, A22 );
View1x2( aTopRow, alpha11, a12 );
View1x2( ABottomPan, a21, A22 );
Zeros( ABottomPan.Height(), 1, z );
//--------------------------------------------------------------------//
const Real tau = Reflector( alpha11, a12 );
const Real alpha = alpha11.Get(0,0);
alpha11.Set(0,0,1);
Gemv( NORMAL, Real(1), ABottomPan, aTopRow, Real(0), z );
Ger( -tau, z, aTopRow, ABottomPan );
alpha11.Set(0,0,alpha);
//--------------------------------------------------------------------//
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, a01, /**/ A02,
/**/ a10, alpha11, /**/ a12,
/*************/ /**********************/
ABL, /**/ ABR, A20, a21, /**/ A22 );
}
PopBlocksizeStack();
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
LQ( DistMatrix<Real>& A )
{
#ifndef RELEASE
PushCallStack("LQ");
#endif
if( IsComplex<Real>::val )
throw std::logic_error("Called real routine with complex datatype");
const Grid& g = A.Grid();
// Matrix views
DistMatrix<Real>
ATL(g), ATR(g), A00(g), A01(g), A02(g), ATopPan(g), ABottomPan(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
PartitionDownLeftDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
while( ATL.Height() < A.Height() && ATL.Width() < A.Width() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
View1x2( ATopPan, A11, A12 );
View1x2( ABottomPan, A21, A22 );
//--------------------------------------------------------------------//
internal::PanelLQ( ATopPan );
ApplyPackedReflectors
( RIGHT, UPPER, HORIZONTAL, FORWARD, 0, ATopPan, ABottomPan );
//--------------------------------------------------------------------//
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
Householder( Matrix<Real>& A )
{
#ifndef RELEASE
PushCallStack("lq::Householder");
#endif
if( IsComplex<Real>::val )
throw std::logic_error("Called real routine with complex datatype");
// Matrix views
Matrix<Real>
ATL, ATR, A00, A01, A02, ATopPan, ABottomPan,
ABL, ABR, A10, A11, A12,
A20, A21, A22;
PartitionDownLeftDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
while( ATL.Height() < A.Height() && ATL.Width() < A.Width() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
View1x2( ATopPan, A11, A12 );
View1x2( ABottomPan, A21, A22 );
//--------------------------------------------------------------------//
PanelHouseholder( ATopPan );
ApplyPackedReflectors
( RIGHT, UPPER, HORIZONTAL, FORWARD, 0, ATopPan, ABottomPan );
//--------------------------------------------------------------------//
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
SlidePartitionRight
( DM& AL, DM& AR,
DM& A0, DM& A1, DM& A2 )
{
DEBUG_ONLY(CallStackEntry cse("SlidePartitionRight"))
View1x2( AL, A0, A1 );
View( AR, A2 );
}
inline void
SlidePartitionLeft
( M& AL, M& AR,
M& A0, M& A1, M& A2 )
{
DEBUG_ONLY(CallStackEntry cse("SlidePartitionLeft"))
View( AL, A0 );
View1x2( AR, A1, A2 );
}
inline void
SlidePartitionUpDiagonal
( M& ATL, M& ATR, M& A00, M& A01, M& A02,
M& A10, M& A11, M& A12,
M& ABL, M& ABR, M& A20, M& A21, M& A22 )
{
DEBUG_ONLY(CallStackEntry cse("SlidePartitionUpDiagonal"))
View( ATL, A00 );
View1x2( ATR, A01, A02 );
View2x1( ABL, A10, A20 );
View2x2( ABR, A11, A12,
A21, A22 );
}
inline void
SlidePartitionDownDiagonal
( DM& ATL, DM& ATR, DM& A00, DM& A01, DM& A02,
DM& A10, DM& A11, DM& A12,
DM& ABL, DM& ABR, DM& A20, DM& A21, DM& A22 )
{
DEBUG_ONLY(CallStackEntry cse("SlidePartitionDownDiagonal"))
View2x2( ATL, A00, A01,
A10, A11 );
View2x1( ATR, A02, A12 );
View1x2( ABL, A20, A21 );
View( ABR, A22 );
}
inline void
SymmRLC
( T alpha, const DistMatrix<T>& A, const DistMatrix<T>& B,
T beta, DistMatrix<T>& C )
{
#ifndef RELEASE
PushCallStack("internal::SymmRLC");
if( A.Grid() != B.Grid() || B.Grid() != C.Grid() )
throw std::logic_error
("{A,B,C} must be distributed over the same grid");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<T>
ATL(g), ATR(g), A00(g), A01(g), A02(g), AColPan(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g), ARowPan(g),
A20(g), A21(g), A22(g);
DistMatrix<T> BL(g), BR(g),
B0(g), B1(g), B2(g);
DistMatrix<T> CL(g), CR(g),
C0(g), C1(g), C2(g),
CLeft(g), CRight(g);
// Temporary distributions
DistMatrix<T,MC, STAR> B1_MC_STAR(g);
DistMatrix<T,VR, STAR> AColPan_VR_STAR(g);
DistMatrix<T,STAR,MR > AColPanTrans_STAR_MR(g);
DistMatrix<T,MR, STAR> ARowPanTrans_MR_STAR(g);
B1_MC_STAR.AlignWith( C );
// Start the algorithm
Scale( beta, C );
LockedPartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
LockedPartitionRight( B, BL, BR, 0 );
PartitionRight( C, CL, CR, 0 );
while( CR.Width() > 0 )
{
LockedRepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
LockedRepartitionRight
( BL, /**/ BR,
B0, /**/ B1, B2 );
RepartitionRight
( CL, /**/ CR,
C0, /**/ C1, C2 );
LockedView1x2( ARowPan, A10, A11 );
LockedView2x1
( AColPan, A11,
A21 );
View1x2( CLeft, C0, C1 );
View1x2( CRight, C1, C2 );
AColPan_VR_STAR.AlignWith( CRight );
AColPanTrans_STAR_MR.AlignWith( CRight );
ARowPanTrans_MR_STAR.AlignWith( CLeft );
//--------------------------------------------------------------------//
B1_MC_STAR = B1;
ARowPanTrans_MR_STAR.TransposeFrom( ARowPan );
AColPan_VR_STAR = AColPan;
AColPanTrans_STAR_MR.TransposeFrom( AColPan_VR_STAR );
MakeTrapezoidal( RIGHT, UPPER, 0, ARowPanTrans_MR_STAR );
MakeTrapezoidal( LEFT, UPPER, 1, AColPanTrans_STAR_MR );
LocalGemm
( NORMAL, TRANSPOSE,
alpha, B1_MC_STAR, ARowPanTrans_MR_STAR, T(1), CLeft );
LocalGemm
( NORMAL, NORMAL,
alpha, B1_MC_STAR, AColPanTrans_STAR_MR, T(1), CRight );
//--------------------------------------------------------------------//
AColPan_VR_STAR.FreeAlignments();
AColPanTrans_STAR_MR.FreeAlignments();
ARowPanTrans_MR_STAR.FreeAlignments();
SlideLockedPartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlideLockedPartitionRight
( BL, /**/ BR,
B0, B1, /**/ B2 );
SlidePartitionRight
( CL, /**/ CR,
C0, C1, /**/ C2 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void LUnb( DistMatrix<R>& A )
{
#ifndef RELEASE
PushCallStack("bidiag::LUnb");
if( A.Height() > A.Width() )
throw std::logic_error("A must be at least as wide as it is tall");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<R>
ATL(g), ATR(g), A00(g), a01(g), A02(g), alpha21T(g), a1R(g),
ABL(g), ABR(g), a10(g), alpha11(g), a12(g), a21B(g), A2R(g),
A20(g), a21(g), A22(g);
// Temporary matrices
DistMatrix<R,MC, STAR> a21_MC_STAR(g);
DistMatrix<R,STAR,MR > a1R_STAR_MR(g);
DistMatrix<R,MR, STAR> x12Trans_MR_STAR(g);
DistMatrix<R,MC, STAR> w21_MC_STAR(g);
PushBlocksizeStack( 1 );
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
while( ATL.Width() < A.Width() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ a01, A02,
/*************/ /**********************/
/**/ a10, /**/ alpha11, a12,
ABL, /**/ ABR, A20, /**/ a21, A22 );
View1x2( a1R, alpha11, a12 );
View1x2( A2R, a21, A22 );
a21_MC_STAR.AlignWith( A22 );
a1R_STAR_MR.AlignWith( A2R );
x12Trans_MR_STAR.AlignWith( A22 );
w21_MC_STAR.AlignWith( A2R );
Zeros( a12.Width(), 1, x12Trans_MR_STAR );
Zeros( a21.Height(), 1, w21_MC_STAR );
const bool thisIsMyRow = ( g.Row() == alpha11.ColAlignment() );
const bool thisIsMyCol = ( g.Col() == alpha11.RowAlignment() );
const bool nextIsMyRow = ( g.Row() == a21.ColAlignment() );
//--------------------------------------------------------------------//
// Find tauP, u, and epsilonP such that
// I - tauP | 1 | | 1, v^T | | alpha11 | = | epsilonP |
// | v | | a12^T | = | 0 |
const R tauP = Reflector( alpha11, a12 );
R epsilonP=0;
if( thisIsMyCol && thisIsMyRow )
epsilonP = alpha11.GetLocal(0,0);
// Set a1R^T = | 1 | and form w21 := A2R a1R^T = A2R | 1 |
// | v | | v |
alpha11.Set(0,0,R(1));
a1R_STAR_MR = a1R;
LocalGemv( NORMAL, R(1), A2R, a1R_STAR_MR, R(0), w21_MC_STAR );
w21_MC_STAR.SumOverRow();
// A2R := A2R - tauP w21 a1R
// = A2R - tauP A2R a1R^T a1R
// = A2R (I - tauP a1R^T a1R)
LocalGer( -tauP, w21_MC_STAR, a1R_STAR_MR, A2R );
// Put epsilonP back instead of the temporary value, 1
if( thisIsMyCol && thisIsMyRow )
alpha11.SetLocal(0,0,epsilonP);
if( A22.Height() != 0 )
{
// Expose the subvector we seek to zero, a21B
PartitionDown
( a21, alpha21T,
a21B );
// Find tauQ, u, and epsilonQ such that
// I - tauQ | 1 | | 1, u^T | | alpha21T | = | epsilonQ |
// | u | | a21B | = | 0 |
const R tauQ = Reflector( alpha21T, a21B );
R epsilonQ=0;
if( nextIsMyRow && thisIsMyCol )
epsilonQ = alpha21T.GetLocal(0,0);
// Set a21 = | 1 | and form x12^T = (a21^T A22)^T = A22^T a21
// | u |
alpha21T.Set(0,0,R(1));
a21_MC_STAR = a21;
LocalGemv
( TRANSPOSE, R(1), A22, a21_MC_STAR, R(0), x12Trans_MR_STAR );
x12Trans_MR_STAR.SumOverCol();
// A22 := A22 - tauQ a21 x12
// = A22 - tauQ a21 a21^T A22
// = (I - tauQ a21 a21^T) A22
LocalGer( -tauQ, a21_MC_STAR, x12Trans_MR_STAR, A22 );
// Put epsilonQ back instead of the temporary value, 1
if( nextIsMyRow && thisIsMyCol )
alpha21T.SetLocal(0,0,epsilonQ);
}
//--------------------------------------------------------------------//
a1R_STAR_MR.FreeAlignments();
a21_MC_STAR.FreeAlignments();
x12Trans_MR_STAR.FreeAlignments();
w21_MC_STAR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, a01, /**/ A02,
/**/ a10, alpha11, /**/ a12,
/*************/ /**********************/
ABL, /**/ ABR, A20, a21, /**/ A22 );
}
PopBlocksizeStack();
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
PanelHouseholder( DistMatrix<F>& A, DistMatrix<F,MD,STAR>& t )
{
#ifndef RELEASE
CallStackEntry entry("lq::PanelHouseholder");
if( A.Grid() != t.Grid() )
LogicError("{A,t} must be distributed over the same grid");
if( t.Height() != Min(A.Height(),A.Width()) || t.Width() != 1 )
LogicError
("t must be a vector of height equal to the minimum dimension of A");
if( !t.AlignedWithDiagonal( A, 0 ) )
LogicError("t must be aligned with A's main diagonal");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), a01(g), A02(g), aTopRow(g), ABottomPan(g),
ABL(g), ABR(g), a10(g), alpha11(g), a12(g),
A20(g), a21(g), A22(g);
DistMatrix<F,MD,STAR>
tT(g), t0(g),
tB(g), tau1(g),
t2(g);
// Temporary distributions
DistMatrix<F> aTopRowConj(g);
DistMatrix<F,STAR,MR > aTopRowConj_STAR_MR(g);
DistMatrix<F,MC, STAR> z_MC_STAR(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
PartitionDown
( t, tT,
tB, 0 );
while( ATL.Height() < A.Height() && ATL.Width() < A.Width() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ a01, A02,
/*************/ /**********************/
/**/ a10, /**/ alpha11, a12,
ABL, /**/ ABR, A20, /**/ a21, A22, 1 );
RepartitionDown
( tT, t0,
/**/ /****/
tau1,
tB, t2, 1 );
View1x2( aTopRow, alpha11, a12 );
View1x2( ABottomPan, a21, A22 );
aTopRowConj_STAR_MR.AlignWith( ABottomPan );
z_MC_STAR.AlignWith( ABottomPan );
//--------------------------------------------------------------------//
// Compute the Householder reflector
const F tau = Reflector( alpha11, a12 );
tau1.Set( 0, 0, tau );
// Apply the Householder reflector
const bool myDiagonalEntry = ( g.Row() == alpha11.ColAlignment() &&
g.Col() == alpha11.RowAlignment() );
F alpha = 0;
if( myDiagonalEntry )
{
alpha = alpha11.GetLocal(0,0);
alpha11.SetLocal(0,0,1);
}
Conjugate( aTopRow, aTopRowConj );
aTopRowConj_STAR_MR = aTopRowConj;
Zeros( z_MC_STAR, ABottomPan.Height(), 1 );
LocalGemv
( NORMAL, F(1), ABottomPan, aTopRowConj_STAR_MR, F(0), z_MC_STAR );
z_MC_STAR.SumOverRow();
Ger
( -Conj(tau),
z_MC_STAR.LockedMatrix(),
aTopRowConj_STAR_MR.LockedMatrix(),
ABottomPan.Matrix() );
if( myDiagonalEntry )
alpha11.SetLocal(0,0,alpha);
//--------------------------------------------------------------------//
SlidePartitionDown
( tT, t0,
tau1,
/**/ /****/
tB, t2 );
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, a01, /**/ A02,
/**/ a10, alpha11, /**/ a12,
/*************/ /**********************/
ABL, /**/ ABR, A20, a21, /**/ A22 );
}
}
inline void
PanelLQ( DistMatrix<Real>& A )
{
#ifndef RELEASE
PushCallStack("internal::PanelLQ");
#endif
const Grid& g = A.Grid();
// Matrix views
DistMatrix<Real>
ATL(g), ATR(g), A00(g), a01(g), A02(g), aTopRow(g), ABottomPan(g),
ABL(g), ABR(g), a10(g), alpha11(g), a12(g),
A20(g), a21(g), A22(g);
// Temporary distributions
DistMatrix<Real,STAR,MR> aTopRow_STAR_MR(g);
DistMatrix<Real,MC,STAR> z_MC_STAR(g);
PushBlocksizeStack( 1 );
PartitionDownLeftDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
while( ATL.Height() < A.Height() && ATL.Width() < A.Width() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ a01, A02,
/*************/ /**********************/
/**/ a10, /**/ alpha11, a12,
ABL, /**/ ABR, A20, /**/ a21, A22 );
View1x2( aTopRow, alpha11, a12 );
View1x2( ABottomPan, a21, A22 );
aTopRow_STAR_MR.AlignWith( ABottomPan );
z_MC_STAR.AlignWith( ABottomPan );
Zeros( ABottomPan.Height(), 1, z_MC_STAR );
//--------------------------------------------------------------------//
const Real tau = Reflector( alpha11, a12 );
const bool myDiagonalEntry = ( g.Row() == alpha11.ColAlignment() &&
g.Col() == alpha11.RowAlignment() );
Real alpha = 0;
if( myDiagonalEntry )
{
alpha = alpha11.GetLocal(0,0);
alpha11.SetLocal(0,0,1);
}
aTopRow_STAR_MR = aTopRow;
Gemv
( NORMAL,
Real(1), ABottomPan.LockedLocalMatrix(),
aTopRow_STAR_MR.LockedLocalMatrix(),
Real(0), z_MC_STAR.LocalMatrix() );
z_MC_STAR.SumOverRow();
Ger
( -tau,
z_MC_STAR.LockedLocalMatrix(),
aTopRow_STAR_MR.LockedLocalMatrix(),
ABottomPan.LocalMatrix() );
if( myDiagonalEntry )
alpha11.SetLocal(0,0,alpha);
//--------------------------------------------------------------------//
aTopRow_STAR_MR.FreeAlignments();
z_MC_STAR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, a01, /**/ A02,
/**/ a10, alpha11, /**/ a12,
/*************/ /**********************/
ABL, /**/ ABR, A20, a21, /**/ A22 );
}
PopBlocksizeStack();
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
PanelLQ
( DistMatrix<Complex<Real> >& A,
DistMatrix<Complex<Real>,MD,STAR>& t )
{
#ifndef RELEASE
PushCallStack("internal::PanelLQ");
if( A.Grid() != t.Grid() )
throw std::logic_error("{A,t} must be distributed over the same grid");
if( t.Height() != std::min(A.Height(),A.Width()) || t.Width() != 1 )
throw std::logic_error
("t must be a vector of height equal to the minimum dimension of A");
if( !t.AlignedWithDiagonal( A, 0 ) )
throw std::logic_error("t must be aligned with A's main diagonal");
#endif
typedef Complex<Real> C;
const Grid& g = A.Grid();
// Matrix views
DistMatrix<C>
ATL(g), ATR(g), A00(g), a01(g), A02(g), aTopRow(g), ABottomPan(g),
ABL(g), ABR(g), a10(g), alpha11(g), a12(g),
A20(g), a21(g), A22(g);
DistMatrix<C,MD,STAR>
tT(g), t0(g),
tB(g), tau1(g),
t2(g);
// Temporary distributions
DistMatrix<C> aTopRowConj(g);
DistMatrix<C,STAR,MR > aTopRowConj_STAR_MR(g);
DistMatrix<C,MC, STAR> z_MC_STAR(g);
PushBlocksizeStack( 1 );
PartitionDownLeftDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
PartitionDown
( t, tT,
tB, 0 );
while( ATL.Height() < A.Height() && ATL.Width() < A.Width() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ a01, A02,
/*************/ /**********************/
/**/ a10, /**/ alpha11, a12,
ABL, /**/ ABR, A20, /**/ a21, A22 );
RepartitionDown
( tT, t0,
/**/ /****/
tau1,
tB, t2 );
View1x2( aTopRow, alpha11, a12 );
View1x2( ABottomPan, a21, A22 );
aTopRowConj_STAR_MR.AlignWith( ABottomPan );
z_MC_STAR.AlignWith( ABottomPan );
Zeros( ABottomPan.Height(), 1, z_MC_STAR );
//--------------------------------------------------------------------//
const C tau = Reflector( alpha11, a12 );
tau1.Set( 0, 0, tau );
const bool myDiagonalEntry = ( g.Row() == alpha11.ColAlignment() &&
g.Col() == alpha11.RowAlignment() );
C alpha = 0;
if( myDiagonalEntry )
{
alpha = alpha11.GetLocal(0,0);
alpha11.SetLocal(0,0,1);
}
Conjugate( aTopRow, aTopRowConj );
aTopRowConj_STAR_MR = aTopRowConj;
Gemv
( NORMAL,
C(1), ABottomPan.LockedLocalMatrix(),
aTopRowConj_STAR_MR.LockedLocalMatrix(),
C(0), z_MC_STAR.LocalMatrix() );
z_MC_STAR.SumOverRow();
Ger
( -Conj(tau),
z_MC_STAR.LockedLocalMatrix(),
aTopRowConj_STAR_MR.LockedLocalMatrix(),
ABottomPan.LocalMatrix() );
if( myDiagonalEntry )
alpha11.SetLocal(0,0,alpha);
//--------------------------------------------------------------------//
aTopRowConj_STAR_MR.FreeAlignments();
z_MC_STAR.FreeAlignments();
SlidePartitionDown
( tT, t0,
tau1,
/**/ /****/
tB, t2 );
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, a01, /**/ A02,
/**/ a10, alpha11, /**/ a12,
/*************/ /**********************/
ABL, /**/ ABR, A20, a21, /**/ A22 );
}
PopBlocksizeStack();
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
PanelLQ
( Matrix<Complex<Real> >& A,
Matrix<Complex<Real> >& t )
{
#ifndef RELEASE
PushCallStack("internal::PanelLQ");
if( t.Height() != std::min(A.Height(),A.Width()) || t.Width() != 1 )
throw std::logic_error
("t must be a vector of height equal to the minimum dimension of A");
#endif
typedef Complex<Real> C;
Matrix<C>
ATL, ATR, A00, a01, A02, aTopRow, ABottomPan,
ABL, ABR, a10, alpha11, a12,
A20, a21, A22;
Matrix<C>
tT, t0,
tB, tau1,
t2;
Matrix<C> z, aTopRowConj;
PushBlocksizeStack( 1 );
PartitionDownLeftDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
PartitionDown
( t, tT,
tB, 0 );
while( ATL.Height() < A.Height() && ATL.Width() < A.Width() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ a01, A02,
/*************/ /**********************/
/**/ a10, /**/ alpha11, a12,
ABL, /**/ ABR, A20, /**/ a21, A22 );
RepartitionDown
( tT, t0,
/**/ /****/
tau1,
tB, t2 );
View1x2( aTopRow, alpha11, a12 );
View1x2( ABottomPan, a21, A22 );
Zeros( ABottomPan.Height(), 1, z );
//--------------------------------------------------------------------//
const C tau = Reflector( alpha11, a12 );
tau1.Set( 0, 0, tau );
const C alpha = alpha11.Get(0,0);
alpha11.Set(0,0,1);
Conjugate( aTopRow, aTopRowConj );
Gemv( NORMAL, C(1), ABottomPan, aTopRowConj, C(0), z );
Ger( -Conj(tau), z, aTopRowConj, ABottomPan );
alpha11.Set(0,0,alpha);
//--------------------------------------------------------------------//
SlidePartitionDown
( tT, t0,
tau1,
/**/ /****/
tB, t2 );
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, a01, /**/ A02,
/**/ a10, alpha11, /**/ a12,
/*************/ /**********************/
ABL, /**/ ABR, A20, a21, /**/ A22 );
}
PopBlocksizeStack();
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
Householder( DistMatrix<F>& A, DistMatrix<F,MD,STAR>& t )
{
#ifndef RELEASE
CallStackEntry entry("Householder");
if( A.Grid() != t.Grid() )
LogicError("{A,t} must be distributed over the same grid");
#endif
const Grid& g = A.Grid();
if( t.Viewing() )
{
if( !t.AlignedWithDiagonal( A ) )
LogicError("t was not aligned with A");
}
else
{
t.AlignWithDiagonal( A );
}
t.ResizeTo( Min(A.Height(),A.Width()), 1 );
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g), ATopPan(g), ABottomPan(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<F,MD,STAR>
tT(g), t0(g),
tB(g), t1(g),
t2(g);
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
PartitionDown
( t, tT,
tB, 0 );
while( ATL.Height() < A.Height() && ATL.Width() < A.Width() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
RepartitionDown
( tT, t0,
/**/ /**/
t1,
tB, t2 );
View1x2( ATopPan, A11, A12 );
View1x2( ABottomPan, A21, A22 );
//--------------------------------------------------------------------//
PanelHouseholder( ATopPan, t1 );
ApplyQ( RIGHT, ADJOINT, ATopPan, t1, ABottomPan );
//--------------------------------------------------------------------//
SlidePartitionDown
( tT, t0,
t1,
/**/ /**/
tB, t2 );
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
}
}
inline void
LQ( Matrix<Complex<Real> >& A,
Matrix<Complex<Real> >& t )
{
#ifndef RELEASE
PushCallStack("LQ");
#endif
typedef Complex<Real> C;
t.ResizeTo( std::min(A.Height(),A.Width()), 1 );
// Matrix views
Matrix<C>
ATL, ATR, A00, A01, A02, ATopPan, ABottomPan,
ABL, ABR, A10, A11, A12,
A20, A21, A22;
Matrix<C>
tT, t0,
tB, t1,
t2;
PartitionDownLeftDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
PartitionDown
( t, tT,
tB, 0 );
while( ATL.Height() < A.Height() && ATL.Width() < A.Width() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
RepartitionDown
( tT, t0,
/**/ /**/
t1,
tB, t2 );
View1x2( ATopPan, A11, A12 );
View1x2( ABottomPan, A21, A22 );
//--------------------------------------------------------------------//
internal::PanelLQ( ATopPan, t1 );
ApplyPackedReflectors
( RIGHT, UPPER, HORIZONTAL, FORWARD, CONJUGATED,
0, ATopPan, t1, ABottomPan );
//--------------------------------------------------------------------//
SlidePartitionDown
( tT, t0,
t1,
/**/ /**/
tB, t2 );
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
LU( DistMatrix<F>& A, DistMatrix<int,VC,STAR>& p )
{
#ifndef RELEASE
CallStackEntry entry("LU");
if( A.Grid() != p.Grid() )
throw std::logic_error("{A,p} must be distributed over the same grid");
if( p.Viewing() &&
(std::min(A.Height(),A.Width()) != p.Height() || p.Width() != 1) )
throw std::logic_error
("p must be a vector of the same height as the min dimension of A.");
#endif
const Grid& g = A.Grid();
if( !p.Viewing() )
p.ResizeTo( std::min(A.Height(),A.Width()), 1 );
// Matrix views
DistMatrix<F>
ATL(g), ATR(g), A00(g), A01(g), A02(g), AB(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<int,VC,STAR>
pT(g), p0(g),
pB(g), p1(g),
p2(g);
// Temporary distributions
DistMatrix<F, STAR,STAR> A11_STAR_STAR(g);
DistMatrix<F, MC, STAR> A21_MC_STAR(g);
DistMatrix<F, STAR,VR > A12_STAR_VR(g);
DistMatrix<F, STAR,MR > A12_STAR_MR(g);
DistMatrix<int,STAR,STAR> p1_STAR_STAR(g);
// Pivot composition
std::vector<int> image, preimage;
// Start the algorithm
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
PartitionDown
( p, pT,
pB, 0 );
while( ATL.Height() < A.Height() && ATL.Width() < A.Width() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
RepartitionDown
( pT, p0,
/**/ /**/
p1,
pB, p2 );
View1x2( AB, ABL, ABR );
const int pivotOffset = A01.Height();
A12_STAR_VR.AlignWith( A22 );
A12_STAR_MR.AlignWith( A22 );
A21_MC_STAR.AlignWith( A22 );
A11_STAR_STAR.ResizeTo( A11.Height(), A11.Width() );
p1_STAR_STAR.ResizeTo( p1.Height(), 1 );
//--------------------------------------------------------------------//
A21_MC_STAR = A21;
A11_STAR_STAR = A11;
lu::Panel( A11_STAR_STAR, A21_MC_STAR, p1_STAR_STAR, pivotOffset );
ComposePivots( p1_STAR_STAR, pivotOffset, image, preimage );
ApplyRowPivots( AB, image, preimage );
// Perhaps we should give up perfectly distributing this operation since
// it's total contribution is only O(n^2)
A12_STAR_VR = A12;
LocalTrsm
( LEFT, LOWER, NORMAL, UNIT, F(1), A11_STAR_STAR, A12_STAR_VR );
A12_STAR_MR = A12_STAR_VR;
LocalGemm( NORMAL, NORMAL, F(-1), A21_MC_STAR, A12_STAR_MR, F(1), A22 );
A11 = A11_STAR_STAR;
A12 = A12_STAR_MR;
A21 = A21_MC_STAR;
p1 = p1_STAR_STAR;
//--------------------------------------------------------------------//
A12_STAR_VR.FreeAlignments();
A12_STAR_MR.FreeAlignments();
A21_MC_STAR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
SlidePartitionDown
( pT, p0,
p1,
/**/ /**/
pB, p2 );
}
}
inline void
LQ( DistMatrix<Complex<Real> >& A,
DistMatrix<Complex<Real>,MD,STAR>& t )
{
#ifndef RELEASE
PushCallStack("LQ");
if( A.Grid() != t.Grid() )
throw std::logic_error("{A,t} must be distributed over the same grid");
#endif
typedef Complex<Real> C;
const Grid& g = A.Grid();
if( t.Viewing() )
{
if( !t.AlignedWithDiagonal( A ) )
throw std::logic_error("t was not aligned with A");
if( t.Height() != std::min(A.Height(),A.Width()) || t.Width() != 1 )
throw std::logic_error("t was not the appropriate shape");
}
else
{
t.AlignWithDiagonal( A );
t.ResizeTo( std::min(A.Height(),A.Width()), 1 );
}
// Matrix views
DistMatrix<C>
ATL(g), ATR(g), A00(g), A01(g), A02(g), ATopPan(g), ABottomPan(g),
ABL(g), ABR(g), A10(g), A11(g), A12(g),
A20(g), A21(g), A22(g);
DistMatrix<C,MD,STAR>
tT(g), t0(g),
tB(g), t1(g),
t2(g);
PartitionDownLeftDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
PartitionDown
( t, tT,
tB, 0 );
while( ATL.Height() < A.Height() && ATL.Width() < A.Width() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ A01, A02,
/*************/ /******************/
/**/ A10, /**/ A11, A12,
ABL, /**/ ABR, A20, /**/ A21, A22 );
RepartitionDown
( tT, t0,
/**/ /**/
t1,
tB, t2 );
View1x2( ATopPan, A11, A12 );
View1x2( ABottomPan, A21, A22 );
//--------------------------------------------------------------------//
internal::PanelLQ( ATopPan, t1 );
ApplyPackedReflectors
( RIGHT, UPPER, HORIZONTAL, FORWARD, CONJUGATED,
0, ATopPan, t1, ABottomPan );
//--------------------------------------------------------------------//
SlidePartitionDown
( tT, t0,
t1,
/**/ /**/
tB, t2 );
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, A01, /**/ A02,
/**/ A10, A11, /**/ A12,
/*************/ /******************/
ABL, /**/ ABR, A20, A21, /**/ A22 );
}
#ifndef RELEASE
PopCallStack();
#endif
}
inline void
PanelHouseholder( Matrix<F>& A, Matrix<F>& t )
{
#ifndef RELEASE
CallStackEntry entry("lq::PanelHouseholder");
if( t.Height() != Min(A.Height(),A.Width()) || t.Width() != 1 )
LogicError
("t must be a vector of height equal to the minimum dimension of A");
#endif
Matrix<F>
ATL, ATR, A00, a01, A02, aTopRow, ABottomPan,
ABL, ABR, a10, alpha11, a12,
A20, a21, A22;
Matrix<F>
tT, t0,
tB, tau1,
t2;
Matrix<F> z, aTopRowConj;
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
PartitionDown
( t, tT,
tB, 0 );
while( ATL.Height() < A.Height() && ATL.Width() < A.Width() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ a01, A02,
/*************/ /**********************/
/**/ a10, /**/ alpha11, a12,
ABL, /**/ ABR, A20, /**/ a21, A22, 1 );
RepartitionDown
( tT, t0,
/**/ /****/
tau1,
tB, t2, 1 );
View1x2( aTopRow, alpha11, a12 );
View1x2( ABottomPan, a21, A22 );
//--------------------------------------------------------------------//
// Compute the Householder reflector
const F tau = Reflector( alpha11, a12 );
tau1.Set( 0, 0, tau );
// Apply the Householder reflector
const F alpha = alpha11.Get(0,0);
alpha11.Set(0,0,1);
Conjugate( aTopRow, aTopRowConj );
Zeros( z, ABottomPan.Height(), 1 );
Gemv( NORMAL, F(1), ABottomPan, aTopRowConj, F(0), z );
Ger( -Conj(tau), z, aTopRowConj, ABottomPan );
alpha11.Set(0,0,alpha);
//--------------------------------------------------------------------//
SlidePartitionDown
( tT, t0,
tau1,
/**/ /****/
tB, t2 );
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, a01, /**/ A02,
/**/ a10, alpha11, /**/ a12,
/*************/ /**********************/
ABL, /**/ ABR, A20, a21, /**/ A22 );
}
}
inline void LUnb
( DistMatrix<Complex<R> >& A,
DistMatrix<Complex<R>,MD,STAR>& tP,
DistMatrix<Complex<R>,MD,STAR>& tQ )
{
#ifndef RELEASE
PushCallStack("bidiag::LUnb");
#endif
const int tPHeight = A.Height();
const int tQHeight = std::max(A.Height()-1,0);
#ifndef RELEASE
if( A.Grid() != tP.Grid() || tP.Grid() != tQ.Grid() )
throw std::logic_error("Process grids do not match");
if( A.Height() > A.Width() )
throw std::logic_error("A must be at least as wide as it is tall");
if( tP.Viewing() && (tP.Height() != tPHeight || tP.Width() != 1) )
throw std::logic_error("tP is the wrong height");
if( tQ.Viewing() && (tQ.Height() != tQHeight || tQ.Width() != 1) )
throw std::logic_error("tQ is the wrong height");
#endif
typedef Complex<R> C;
const Grid& g = A.Grid();
if( !tP.Viewing() )
tP.ResizeTo( tPHeight, 1 );
if( !tQ.Viewing() )
tQ.ResizeTo( tQHeight, 1 );
// Matrix views
DistMatrix<C>
ATL(g), ATR(g), A00(g), a01(g), A02(g), alpha21T(g), a1R(g),
ABL(g), ABR(g), a10(g), alpha11(g), a12(g), a21B(g), A2R(g),
A20(g), a21(g), A22(g);
// Temporary matrices
DistMatrix<C,MC, STAR> a21_MC_STAR(g);
DistMatrix<C,STAR,MR > a1R_STAR_MR(g);
DistMatrix<C,MR, STAR> x12Adj_MR_STAR(g);
DistMatrix<C,MC, STAR> w21_MC_STAR(g);
PushBlocksizeStack( 1 );
PartitionDownDiagonal
( A, ATL, ATR,
ABL, ABR, 0 );
while( ATL.Width() < A.Width() )
{
RepartitionDownDiagonal
( ATL, /**/ ATR, A00, /**/ a01, A02,
/*************/ /**********************/
/**/ a10, /**/ alpha11, a12,
ABL, /**/ ABR, A20, /**/ a21, A22 );
View1x2( a1R, alpha11, a12 );
View1x2( A2R, a21, A22 );
a21_MC_STAR.AlignWith( A22 );
a1R_STAR_MR.AlignWith( A2R );
x12Adj_MR_STAR.AlignWith( A22 );
w21_MC_STAR.AlignWith( A2R );
Zeros( a12.Width(), 1, x12Adj_MR_STAR );
Zeros( a21.Height(), 1, w21_MC_STAR );
const bool thisIsMyRow = ( g.Row() == alpha11.ColAlignment() );
const bool thisIsMyCol = ( g.Col() == alpha11.RowAlignment() );
const bool nextIsMyRow = ( g.Row() == a21.ColAlignment() );
//--------------------------------------------------------------------//
// Due to deficiencies in the BLAS ?gemv routines, this section is
// easier if we temporary conjugate a1R = | alpha11, a12 |
Conjugate( a1R );
// Find tauP, u, and epsilonP such that
// I - conj(tauP) | 1 | | 1, v^H | | alpha11 | = | epsilonP |
// | v | | a12^T | = | 0 |
const C tauP = Reflector( alpha11, a12 );
tP.Set(A00.Height(),0,tauP);
C epsilonP=0;
if( thisIsMyCol && thisIsMyRow )
epsilonP = alpha11.GetLocal(0,0);
// Set a1R^T = | 1 | and form w21 := A2R a1R^T = A2R | 1 |
// | v | | v |
alpha11.Set(0,0,C(1));
a1R_STAR_MR = a1R;
LocalGemv( NORMAL, C(1), A2R, a1R_STAR_MR, C(0), w21_MC_STAR );
w21_MC_STAR.SumOverRow();
// A2R := A2R - tauP w21 conj(a1R)
// = A2R - tauP A2R a1R^T conj(a1R)
// = A2R conj(I - conj(tauP) a1R^H a1R)
// which compensates for the fact that the reflector was generated
// on the conjugated a1R
LocalGer( -tauP, w21_MC_STAR, a1R_STAR_MR, A2R );
// Put epsilonP back instead of the temporary value, 1
if( thisIsMyCol && thisIsMyRow )
alpha11.SetLocal(0,0,epsilonP);
// Undo the temporary conjugation
Conjugate( a1R );
if( A22.Height() != 0 )
{
// Expose the subvector we seek to zero, a21B
PartitionDown
( a21, alpha21T,
a21B );
// Find tauQ, u, and epsilonQ such that
// I - conj(tauQ) | 1 | | 1, u^H | | alpha21T | = | epsilonQ |
// | u | | a21B | = | 0 |
const C tauQ = Reflector( alpha21T, a21B );
tQ.Set(A00.Height(),0,tauQ);
C epsilonQ=0;
if( nextIsMyRow && thisIsMyCol )
epsilonQ = alpha21T.GetLocal(0,0);
// Set a21 = | 1 | and form x12^H = (a21^H A22)^H = A22^H a21
// | u |
alpha21T.Set(0,0,C(1));
a21_MC_STAR = a21;
LocalGemv( ADJOINT, C(1), A22, a21_MC_STAR, C(0), x12Adj_MR_STAR );
x12Adj_MR_STAR.SumOverCol();
// A22 := A22 - conj(tauQ) a21 x12
// = A22 - conj(tauQ) a21 a21^H A22
// = (I - conj(tauQ) a21 a21^H) A22
LocalGer( -Conj(tauQ), a21_MC_STAR, x12Adj_MR_STAR, A22 );
// Put epsilonQ back instead of the temporary value, 1
if( nextIsMyRow && thisIsMyCol )
alpha21T.SetLocal(0,0,epsilonQ);
}
//--------------------------------------------------------------------//
a21_MC_STAR.FreeAlignments();
a1R_STAR_MR.FreeAlignments();
x12Adj_MR_STAR.FreeAlignments();
w21_MC_STAR.FreeAlignments();
SlidePartitionDownDiagonal
( ATL, /**/ ATR, A00, a01, /**/ A02,
/**/ a10, alpha11, /**/ a12,
/*************/ /**********************/
ABL, /**/ ABR, A20, a21, /**/ A22 );
}
PopBlocksizeStack();
#ifndef RELEASE
PopCallStack();
#endif
}