inline void TrsvUN( UnitOrNonUnit diag, const DistMatrix<F>& U, DistMatrix<F>& x ) { #ifndef RELEASE PushCallStack("internal::TrsvUN"); if( U.Grid() != x.Grid() ) throw std::logic_error("{U,x} must be distributed over the same grid"); if( U.Height() != U.Width() ) throw std::logic_error("U must be square"); if( x.Width() != 1 && x.Height() != 1 ) throw std::logic_error("x must be a vector"); const int xLength = ( x.Width() == 1 ? x.Height() : x.Width() ); if( U.Width() != xLength ) throw std::logic_error("Nonconformal TrsvUN"); #endif const Grid& g = U.Grid(); if( x.Width() == 1 ) { // Matrix views DistMatrix<F> U01(g), U11(g); DistMatrix<F> xT(g), x0(g), xB(g), x1(g), x2(g); // Temporary distributions DistMatrix<F,STAR,STAR> U11_STAR_STAR(g); DistMatrix<F,STAR,STAR> x1_STAR_STAR(g); DistMatrix<F,MR, STAR> x1_MR_STAR(g); DistMatrix<F,MC, STAR> z_MC_STAR(g); // Views of z[MC,* ], which will store updates to x DistMatrix<F,MC,STAR> z0_MC_STAR(g), z1_MC_STAR(g); z_MC_STAR.AlignWith( U ); Zeros( x.Height(), 1, z_MC_STAR ); // Start the algorithm PartitionUp ( x, xT, xB, 0 ); while( xT.Height() > 0 ) { RepartitionUp ( xT, x0, x1, /**/ /**/ xB, x2 ); const int n0 = x0.Height(); const int n1 = x1.Height(); LockedView( U01, U, 0, n0, n0, n1 ); LockedView( U11, U, n0, n0, n1, n1 ); View( z0_MC_STAR, z_MC_STAR, 0, 0, n0, 1 ); View( z1_MC_STAR, z_MC_STAR, n0, 0, n1, 1 ); x1_MR_STAR.AlignWith( U01 ); //----------------------------------------------------------------// if( x2.Height() != 0 ) x1.SumScatterUpdate( F(1), z1_MC_STAR ); x1_STAR_STAR = x1; U11_STAR_STAR = U11; Trsv ( UPPER, NORMAL, diag, U11_STAR_STAR.LockedLocalMatrix(), x1_STAR_STAR.LocalMatrix() ); x1 = x1_STAR_STAR; x1_MR_STAR = x1_STAR_STAR; Gemv ( NORMAL, F(-1), U01.LockedLocalMatrix(), x1_MR_STAR.LockedLocalMatrix(), F(1), z0_MC_STAR.LocalMatrix() ); //----------------------------------------------------------------// x1_MR_STAR.FreeAlignments(); SlidePartitionUp ( xT, x0, /**/ /**/ x1, xB, x2 ); } } else { // Matrix views DistMatrix<F> U01(g), U11(g); DistMatrix<F> xL(g), xR(g), x0(g), x1(g), x2(g); // Temporary distributions DistMatrix<F,STAR,STAR> U11_STAR_STAR(g); DistMatrix<F,STAR,STAR> x1_STAR_STAR(g); DistMatrix<F,STAR,MR > x1_STAR_MR(g); DistMatrix<F,MC, MR > z1(g); DistMatrix<F,MR, MC > z1_MR_MC(g); DistMatrix<F,STAR,MC > z_STAR_MC(g); // Views of z[* ,MC] DistMatrix<F,STAR,MC> z0_STAR_MC(g), z1_STAR_MC(g); z_STAR_MC.AlignWith( U ); Zeros( 1, x.Width(), z_STAR_MC ); // Start the algorithm PartitionLeft( x, xL, xR, 0 ); while( xL.Width() > 0 ) { RepartitionLeft ( xL, /**/ xR, x0, x1, /**/ x2 ); const int n0 = x0.Width(); const int n1 = x1.Width(); LockedView( U01, U, 0, n0, n0, n1 ); LockedView( U11, U, n0, n0, n1, n1 ); View( z0_STAR_MC, z_STAR_MC, 0, 0, 1, n0 ); View( z1_STAR_MC, z_STAR_MC, 0, n0, 1, n1 ); x1_STAR_MR.AlignWith( U01 ); z1.AlignWith( x1 ); //----------------------------------------------------------------// if( x2.Width() != 0 ) { z1_MR_MC.SumScatterFrom( z1_STAR_MC ); z1 = z1_MR_MC; Axpy( F(1), z1, x1 ); } x1_STAR_STAR = x1; U11_STAR_STAR = U11; Trsv ( UPPER, NORMAL, diag, U11_STAR_STAR.LockedLocalMatrix(), x1_STAR_STAR.LocalMatrix() ); x1 = x1_STAR_STAR; x1_STAR_MR = x1_STAR_STAR; Gemv ( NORMAL, F(-1), U01.LockedLocalMatrix(), x1_STAR_MR.LockedLocalMatrix(), F(1), z0_STAR_MC.LocalMatrix() ); //----------------------------------------------------------------// x1_STAR_MR.FreeAlignments(); z1.FreeAlignments(); SlidePartitionLeft ( xL, /**/ xR, x0, /**/ x1, x2 ); } } #ifndef RELEASE PopCallStack(); #endif }
inline void internal::TrsvLN ( UnitOrNonUnit diag, const DistMatrix<F,MC,MR>& L, DistMatrix<F,MC,MR>& x ) { #ifndef RELEASE PushCallStack("internal::TrsvLN"); if( L.Grid() != x.Grid() ) throw std::logic_error("{L,x} must be distributed over the same grid"); if( L.Height() != L.Width() ) throw std::logic_error("L must be square"); if( x.Width() != 1 && x.Height() != 1 ) throw std::logic_error("x must be a vector"); const int xLength = ( x.Width() == 1 ? x.Height() : x.Width() ); if( L.Width() != xLength ) throw std::logic_error("Nonconformal TrsvLN"); #endif const Grid& g = L.Grid(); if( x.Width() == 1 ) { // Matrix views DistMatrix<F,MC,MR> LTL(g), LTR(g), L00(g), L01(g), L02(g), LBL(g), LBR(g), L10(g), L11(g), L12(g), L20(g), L21(g), L22(g); DistMatrix<F,MC,MR> xT(g), x0(g), xB(g), x1(g), x2(g); // Temporary distributions DistMatrix<F,STAR,STAR> L11_STAR_STAR(g); DistMatrix<F,STAR,STAR> x1_STAR_STAR(g); DistMatrix<F,MR, STAR> x1_MR_STAR(g); DistMatrix<F,MC, STAR> z2_MC_STAR(g); // Start the algorithm LockedPartitionDownDiagonal ( L, LTL, LTR, LBL, LBR, 0 ); PartitionDown ( x, xT, xB, 0 ); while( xB.Height() > 0 ) { LockedRepartitionDownDiagonal ( LTL, /**/ LTR, L00, /**/ L01, L02, /*************/ /******************/ /**/ L10, /**/ L11, L12, LBL, /**/ LBR, L20, /**/ L21, L22 ); RepartitionDown ( xT, x0, /**/ /**/ x1, xB, x2 ); x1_MR_STAR.AlignWith( L21 ); z2_MC_STAR.AlignWith( L21 ); z2_MC_STAR.ResizeTo( x2.Height(), 1 ); //----------------------------------------------------------------// x1_STAR_STAR = x1; L11_STAR_STAR = L11; Trsv ( LOWER, NORMAL, diag, L11_STAR_STAR.LockedLocalMatrix(), x1_STAR_STAR.LocalMatrix() ); x1 = x1_STAR_STAR; x1_MR_STAR = x1_STAR_STAR; Gemv ( NORMAL, (F)-1, L21.LockedLocalMatrix(), x1_MR_STAR.LockedLocalMatrix(), (F)0, z2_MC_STAR.LocalMatrix() ); x2.SumScatterUpdate( (F)1, z2_MC_STAR ); //----------------------------------------------------------------// x1_MR_STAR.FreeAlignments(); z2_MC_STAR.FreeAlignments(); SlideLockedPartitionDownDiagonal ( LTL, /**/ LTR, L00, L01, /**/ L02, /**/ L10, L11, /**/ L12, /*************/ /******************/ LBL, /**/ LBR, L20, L21, /**/ L22 ); SlidePartitionDown ( xT, x0, x1, /**/ /**/ xB, x2 ); } } else { // Matrix views DistMatrix<F,MC,MR> LTL(g), LTR(g), L00(g), L01(g), L02(g), LBL(g), LBR(g), L10(g), L11(g), L12(g), L20(g), L21(g), L22(g); DistMatrix<F,MC,MR> xL(g), xR(g), x0(g), x1(g), x2(g); // Temporary distributions DistMatrix<F,STAR,STAR> L11_STAR_STAR(g); DistMatrix<F,STAR,STAR> x1_STAR_STAR(g); DistMatrix<F,STAR,MR > x1_STAR_MR(g); DistMatrix<F,STAR,MC > z2_STAR_MC(g); DistMatrix<F,MR, MC > z2_MR_MC(g); DistMatrix<F,MC, MR > z2(g); // Start the algorithm LockedPartitionDownDiagonal ( L, LTL, LTR, LBL, LBR, 0 ); PartitionRight( x, xL, xR, 0 ); while( xR.Width() > 0 ) { LockedRepartitionDownDiagonal ( LTL, /**/ LTR, L00, /**/ L01, L02, /*************/ /******************/ /**/ L10, /**/ L11, L12, LBL, /**/ LBR, L20, /**/ L21, L22 ); RepartitionRight ( xL, /**/ xR, x0, /**/ x1, x2 ); x1_STAR_MR.AlignWith( L21 ); z2_STAR_MC.AlignWith( L21 ); z2.AlignWith( x2 ); z2_STAR_MC.ResizeTo( 1, x2.Width() ); //----------------------------------------------------------------// x1_STAR_STAR = x1; L11_STAR_STAR = L11; Trsv ( LOWER, NORMAL, diag, L11_STAR_STAR.LockedLocalMatrix(), x1_STAR_STAR.LocalMatrix() ); x1 = x1_STAR_STAR; x1_STAR_MR = x1_STAR_STAR; Gemv ( NORMAL, (F)-1, L21.LockedLocalMatrix(), x1_STAR_MR.LockedLocalMatrix(), (F)0, z2_STAR_MC.LocalMatrix() ); z2_MR_MC.SumScatterFrom( z2_STAR_MC ); z2 = z2_MR_MC; Axpy( (F)1, z2, x2 ); //----------------------------------------------------------------// x1_STAR_MR.FreeAlignments(); z2_STAR_MC.FreeAlignments(); z2.FreeAlignments(); SlideLockedPartitionDownDiagonal ( LTL, /**/ LTR, L00, L01, /**/ L02, /**/ L10, L11, /**/ L12, /*************/ /******************/ LBL, /**/ LBR, L20, L21, /**/ L22 ); SlidePartitionRight ( xL, /**/ xR, x0, x1, /**/ x2 ); } } #ifndef RELEASE PopCallStack(); #endif }
inline void internal::TrsvUN ( UnitOrNonUnit diag, const DistMatrix<F,MC,MR>& U, DistMatrix<F,MC,MR>& x ) { #ifndef RELEASE PushCallStack("internal::TrsvUN"); if( U.Grid() != x.Grid() ) throw std::logic_error("{U,x} must be distributed over the same grid"); if( U.Height() != U.Width() ) throw std::logic_error("U must be square"); if( x.Width() != 1 && x.Height() != 1 ) throw std::logic_error("x must be a vector"); const int xLength = ( x.Width() == 1 ? x.Height() : x.Width() ); if( U.Width() != xLength ) throw std::logic_error("Nonconformal TrsvUN"); #endif const Grid& g = U.Grid(); if( x.Width() == 1 ) { // Matrix views DistMatrix<F,MC,MR> UTL(g), UTR(g), U00(g), U01(g), U02(g), UBL(g), UBR(g), U10(g), U11(g), U12(g), U20(g), U21(g), U22(g); DistMatrix<F,MC,MR> xT(g), x0(g), xB(g), x1(g), x2(g); // Temporary distributions DistMatrix<F,STAR,STAR> U11_STAR_STAR(g); DistMatrix<F,STAR,STAR> x1_STAR_STAR(g); DistMatrix<F,MR, STAR> x1_MR_STAR(g); DistMatrix<F,MC, STAR> z0_MC_STAR(g); // Start the algorithm LockedPartitionUpDiagonal ( U, UTL, UTR, UBL, UBR, 0 ); PartitionUp ( x, xT, xB, 0 ); while( xT.Height() > 0 ) { LockedRepartitionUpDiagonal ( UTL, /**/ UTR, U00, U01, /**/ U02, /**/ U10, U11, /**/ U12, /*************/ /******************/ UBL, /**/ UBR, U20, U21, /**/ U22 ); RepartitionUp ( xT, x0, x1, /**/ /**/ xB, x2 ); x1_MR_STAR.AlignWith( U01 ); z0_MC_STAR.AlignWith( U01 ); z0_MC_STAR.ResizeTo( x0.Height(), 1 ); //----------------------------------------------------------------// x1_STAR_STAR = x1; U11_STAR_STAR = U11; Trsv ( UPPER, NORMAL, diag, U11_STAR_STAR.LockedLocalMatrix(), x1_STAR_STAR.LocalMatrix() ); x1 = x1_STAR_STAR; x1_MR_STAR = x1_STAR_STAR; Gemv ( NORMAL, (F)-1, U01.LockedLocalMatrix(), x1_MR_STAR.LockedLocalMatrix(), (F)0, z0_MC_STAR.LocalMatrix() ); x0.SumScatterUpdate( (F)1, z0_MC_STAR ); //----------------------------------------------------------------// x1_MR_STAR.FreeAlignments(); z0_MC_STAR.FreeAlignments(); SlideLockedPartitionUpDiagonal ( UTL, /**/ UTR, U00, /**/ U01, U02, /*************/ /******************/ /**/ U10, /**/ U11, U12, UBL, /**/ UBR, U20, /**/ U21, U22 ); SlidePartitionUp ( xT, x0, /**/ /**/ x1, xB, x2 ); } } else { // Matrix views DistMatrix<F,MC,MR> UTL(g), UTR(g), U00(g), U01(g), U02(g), UBL(g), UBR(g), U10(g), U11(g), U12(g), U20(g), U21(g), U22(g); DistMatrix<F,MC,MR> xL(g), xR(g), x0(g), x1(g), x2(g); // Temporary distributions DistMatrix<F,STAR,STAR> U11_STAR_STAR(g); DistMatrix<F,STAR,STAR> x1_STAR_STAR(g); DistMatrix<F,STAR,MR > x1_STAR_MR(g); DistMatrix<F,STAR,MC > z0_STAR_MC(g); DistMatrix<F,MR, MC > z0_MR_MC(g); DistMatrix<F,MC, MR > z0(g); // Start the algorithm LockedPartitionUpDiagonal ( U, UTL, UTR, UBL, UBR, 0 ); PartitionLeft( x, xL, xR, 0 ); while( xL.Width() > 0 ) { LockedRepartitionUpDiagonal ( UTL, /**/ UTR, U00, U01, /**/ U02, /**/ U10, U11, /**/ U12, /*************/ /******************/ UBL, /**/ UBR, U20, U21, /**/ U22 ); RepartitionLeft ( xL, /**/ xR, x0, x1, /**/ x2 ); x1_STAR_MR.AlignWith( U01 ); z0_STAR_MC.AlignWith( U01 ); z0.AlignWith( x0 ); z0_STAR_MC.ResizeTo( 1, x0.Width() ); //----------------------------------------------------------------// x1_STAR_STAR = x1; U11_STAR_STAR = U11; Trsv ( UPPER, NORMAL, diag, U11_STAR_STAR.LockedLocalMatrix(), x1_STAR_STAR.LocalMatrix() ); x1 = x1_STAR_STAR; x1_STAR_MR = x1_STAR_STAR; Gemv ( NORMAL, (F)-1, U01.LockedLocalMatrix(), x1_STAR_MR.LockedLocalMatrix(), (F)0, z0_STAR_MC.LocalMatrix() ); z0_MR_MC.SumScatterFrom( z0_STAR_MC ); z0 = z0_MR_MC; Axpy( (F)1, z0, x0 ); //----------------------------------------------------------------// x1_STAR_MR.FreeAlignments(); z0_STAR_MC.FreeAlignments(); z0.FreeAlignments(); SlideLockedPartitionUpDiagonal ( UTL, /**/ UTR, U00, /**/ U01, U02, /*************/ /******************/ /**/ U10, /**/ U11, U12, UBL, /**/ UBR, U20, /**/ U21, U22 ); SlidePartitionLeft ( xL, /**/ xR, x0, /**/ x1, x2 ); } } #ifndef RELEASE PopCallStack(); #endif }
inline void LocalSymvRowAccumulateU ( T alpha, const DistMatrix<T>& A, const DistMatrix<T,STAR,MC>& x_STAR_MC, const DistMatrix<T,STAR,MR>& x_STAR_MR, DistMatrix<T,STAR,MC>& z_STAR_MC, DistMatrix<T,STAR,MR>& z_STAR_MR ) { #ifndef RELEASE PushCallStack("internal::LocalSymvRowAccumulateU"); if( A.Grid() != x_STAR_MC.Grid() || x_STAR_MC.Grid() != x_STAR_MR.Grid() || x_STAR_MR.Grid() != z_STAR_MC.Grid() || z_STAR_MC.Grid() != z_STAR_MR.Grid() ) throw std::logic_error ("{A,x,z} must be distributed over the same grid"); if( x_STAR_MC.Height() != 1 || x_STAR_MR.Height() != 1 || z_STAR_MC.Height() != 1 || z_STAR_MR.Height() != 1 ) throw std::logic_error("Expected x and z to be row vectors"); if( A.Height() != A.Width() || A.Height() != x_STAR_MC.Width() || A.Height() != x_STAR_MR.Width() || A.Height() != z_STAR_MC.Width() || A.Height() != z_STAR_MR.Width() ) { std::ostringstream msg; msg << "Nonconformal LocalSymvRowAccumulateU: \n" << " A ~ " << A.Height() << " x " << A.Width() << "\n" << " x[* ,MC] ~ " << x_STAR_MC.Height() << " x " << x_STAR_MC.Width() << "\n" << " x[* ,MR] ~ " << x_STAR_MR.Height() << " x " << x_STAR_MR.Width() << "\n" << " z[* ,MC] ~ " << z_STAR_MC.Height() << " x " << z_STAR_MC.Width() << "\n" << " z[* ,MR] ~ " << z_STAR_MR.Height() << " x " << z_STAR_MR.Width() << "\n"; throw std::logic_error( msg.str() ); } if( x_STAR_MC.RowAlignment() != A.ColAlignment() || x_STAR_MR.RowAlignment() != A.RowAlignment() || z_STAR_MC.RowAlignment() != A.ColAlignment() || z_STAR_MR.RowAlignment() != A.RowAlignment() ) throw std::logic_error("Partial matrix distributions are misaligned"); #endif const Grid& g = A.Grid(); // Matrix views DistMatrix<T> A11(g), A12(g); DistMatrix<T> D11(g); DistMatrix<T,STAR,MC> x1_STAR_MC(g); DistMatrix<T,STAR,MR> xL_STAR_MR(g), xR_STAR_MR(g), x0_STAR_MR(g), x1_STAR_MR(g), x2_STAR_MR(g); DistMatrix<T,STAR,MC> z1_STAR_MC(g); DistMatrix<T,STAR,MR> z1_STAR_MR(g), z2_STAR_MR(g); // We want our local gemvs to be of width blocksize, so we will // temporarily change to max(r,c) times the current blocksize const int ratio = std::max( g.Height(), g.Width() ); PushBlocksizeStack( ratio*LocalSymvBlocksize<T>() ); LockedPartitionRight( x_STAR_MR, xL_STAR_MR, xR_STAR_MR, 0 ); while( xL_STAR_MR.Width() < x_STAR_MR.Width() ) { LockedRepartitionRight ( xL_STAR_MR, /**/ xR_STAR_MR, x0_STAR_MR, /**/ x1_STAR_MR, x2_STAR_MR ); const int n0 = x0_STAR_MR.Width(); const int n1 = x1_STAR_MR.Width(); const int n2 = x2_STAR_MR.Width(); LockedView( A11, A, n0, n0, n1, n1 ); LockedView( A12, A, n0, n0+n1, n1, n2 ); LockedView( x1_STAR_MC, x_STAR_MC, 0, n0, 1, n1 ); View( z1_STAR_MC, z_STAR_MC, 0, n0, 1, n1 ); View( z1_STAR_MR, z_STAR_MR, 0, n0, 1, n1 ); View( z2_STAR_MR, z_STAR_MR, 0, n0+n1, 1, n2 ); D11.AlignWith( A11 ); //--------------------------------------------------------------------// // TODO: These diagonal block updates can be greatly improved D11 = A11; MakeTrapezoidal( LEFT, UPPER, 0, D11 ); Gemv ( NORMAL, alpha, D11.LockedLocalMatrix(), x1_STAR_MR.LockedLocalMatrix(), T(1), z1_STAR_MC.LocalMatrix() ); MakeTrapezoidal( LEFT, UPPER, 1, D11 ); Gemv ( TRANSPOSE, alpha, D11.LockedLocalMatrix(), x1_STAR_MC.LockedLocalMatrix(), T(1), z1_STAR_MR.LocalMatrix() ); Gemv ( NORMAL, alpha, A12.LockedLocalMatrix(), x2_STAR_MR.LockedLocalMatrix(), T(1), z1_STAR_MC.LocalMatrix() ); Gemv ( TRANSPOSE, alpha, A12.LockedLocalMatrix(), x1_STAR_MC.LockedLocalMatrix(), T(1), z2_STAR_MR.LocalMatrix() ); //--------------------------------------------------------------------// D11.FreeAlignments(); SlideLockedPartitionRight ( xL_STAR_MR, /**/ xR_STAR_MR, x0_STAR_MR, x1_STAR_MR, /**/ x2_STAR_MR ); } PopBlocksizeStack(); #ifndef RELEASE PopCallStack(); #endif }