void Jordan( AbstractBlockDistMatrix<T>& J, Int n, T lambda )
{
DEBUG_ONLY(CSE cse("Jordan"))
Zeros( J, n, n );
FillDiagonal( J, lambda );
FillDiagonal( J, T(1), 1 );
}
void IPM
( const Matrix<Real>& A,
const Matrix<Real>& d,
Real lambda,
Matrix<Real>& x,
const qp::affine::Ctrl<Real>& ctrl )
{
EL_DEBUG_CSE
const Int m = A.Height();
const Int n = A.Width();
const Range<Int> wInd(0,n), betaInd(n,n+1), zInd(n+1,n+m+1);
Matrix<Real> Q, c, AHat, b, G, h;
// Q := | I 0 0 |
// | 0 0 0 |
// | 0 0 0 |
// ==============
Zeros( Q, n+m+1, n+m+1 );
auto Qww = Q( wInd, wInd );
FillDiagonal( Qww, Real(1) );
// c := [0;0;lambda]
// =================
Zeros( c, n+m+1, 1 );
auto cz = c( zInd, ALL );
Fill( cz, lambda );
// AHat = []
// =========
Zeros( AHat, 0, n+m+1 );
// b = []
// ======
Zeros( b, 0, 1 );
// G := |-diag(d) A, -d, -I|
// | 0, 0, -I|
// =========================
Zeros( G, 2*m, n+m+1 );
auto G0w = G( IR(0,m), wInd );
auto G0beta = G( IR(0,m), betaInd );
auto G0z = G( IR(0,m), zInd );
auto G1z = G( IR(m,2*m), zInd );
G0w = A; G0w *= -1; DiagonalScale( LEFT, NORMAL, d, G0w );
G0beta = d; G0beta *= -1;
FillDiagonal( G0z, Real(-1) );
FillDiagonal( G1z, Real(-1) );
// h := [-ones(m,1); zeros(m,1)]
// =============================
Zeros( h, 2*m, 1 );
auto h0 = h( IR(0,m), ALL );
Fill( h0, Real(-1) );
// Solve the affine QP
// ===================
Matrix<Real> y, z, s;
QP( Q, AHat, G, b, c, h, x, y, z, s, ctrl );
}
void OneTwoOne( AbstractDistMatrix<T>& A, Int n )
{
DEBUG_ONLY(CSE cse("OneTwoOne"))
Zeros( A, n, n );
FillDiagonal( A, T(1), -1 );
FillDiagonal( A, T(2), 0 );
FillDiagonal( A, T(1), 1 );
}
void OneTwoOne( AbstractDistMatrix<T>& A, Int n )
{
EL_DEBUG_CSE
Zeros( A, n, n );
FillDiagonal( A, T(1), -1 );
FillDiagonal( A, T(2), 0 );
FillDiagonal( A, T(1), 1 );
}
void IPM
( const Matrix<Real>& A,
const Matrix<Real>& b,
Real lambda,
Matrix<Real>& x,
const qp::affine::Ctrl<Real>& ctrl )
{
DEBUG_CSE
const Int m = A.Height();
const Int n = A.Width();
const Range<Int> uInd(0,n), vInd(n,2*n), rInd(2*n,2*n+m);
Matrix<Real> Q, c, AHat, G, h;
// Q := | 0 0 0 |
// | 0 0 0 |
// | 0 0 I |
// ==============
Zeros( Q, 2*n+m, 2*n+m );
auto Qrr = Q( rInd, rInd );
FillDiagonal( Qrr, Real(1) );
// c := lambda*[1;1;0]
// ===================
Zeros( c, 2*n+m, 1 );
auto cuv = c( IR(0,2*n), ALL );
Fill( cuv, lambda );
// \hat A := [A, -A, I]
// ====================
Zeros( AHat, m, 2*n+m );
auto AHatu = AHat( IR(0,m), uInd );
auto AHatv = AHat( IR(0,m), vInd );
auto AHatr = AHat( IR(0,m), rInd );
AHatu = A;
AHatv -= A;
FillDiagonal( AHatr, Real(1) );
// G := | -I 0 0 |
// | 0 -I 0 |
// ================
Zeros( G, 2*n, 2*n+m );
FillDiagonal( G, Real(-1) );
// h := 0
// ======
Zeros( h, 2*n, 1 );
// Solve the affine QP
// ===================
Matrix<Real> xHat, y, z, s;
QP( Q, AHat, G, b, c, h, xHat, y, z, s, ctrl );
// x := u - v
// ==========
x = xHat( uInd, ALL );
x -= xHat( vInd, ALL );
}
void LAV
( const AbstractDistMatrix<Real>& A,
const AbstractDistMatrix<Real>& b,
AbstractDistMatrix<Real>& xPre,
const lp::affine::Ctrl<Real>& ctrl )
{
EL_DEBUG_CSE
DistMatrixWriteProxy<Real,Real,MC,MR> xProx( xPre );
auto& x = xProx.Get();
const Int m = A.Height();
const Int n = A.Width();
const Grid& g = A.Grid();
const Range<Int> xInd(0,n), uInd(n,n+m), vInd(n+m,n+2*m);
DistMatrix<Real> c(g), AHat(g), G(g), h(g);
// c := [0;1;1]
// ============
Zeros( c, n+2*m, 1 );
auto cuv = c( IR(n,n+2*m), ALL );
Fill( cuv, Real(1) );
// \hat A := [A, I, -I]
// ====================
Zeros( AHat, m, n+2*m );
auto AHatx = AHat( IR(0,m), xInd );
auto AHatu = AHat( IR(0,m), uInd );
auto AHatv = AHat( IR(0,m), vInd );
AHatx = A;
FillDiagonal( AHatu, Real( 1) );
FillDiagonal( AHatv, Real(-1) );
// G := | 0 -I 0 |
// | 0 0 -I |
// ================
Zeros( G, 2*m, n+2*m );
auto Guv = G( IR(0,2*m), IR(n,n+2*m) );
FillDiagonal( Guv, Real(-1) );
// h := | 0 |
// | 0 |
// ==========
Zeros( h, 2*m, 1 );
// Solve the affine linear program
// ===============================
DistMatrix<Real> xHat(g), y(g), z(g), s(g);
LP( AHat, G, b, c, h, xHat, y, z, s, ctrl );
// Extract x
// =========
x = xHat( xInd, ALL );
}
void JordanCholesky( AbstractDistMatrix<T>& A, Int n )
{
DEBUG_ONLY(CSE cse("JordanCholesky"))
Zeros( A, n, n );
// Set the main diagonal equal to five everywhere but the top-left entry
FillDiagonal( A, T(5) );
if( n > 0 )
A.Set( 0, 0, T(1) );
// Set the rest of the tridiagonal to 2
FillDiagonal( A, T(2), 1 );
FillDiagonal( A, T(2), -1 );
}
void LAV
( const Matrix<Real>& A,
const Matrix<Real>& b,
Matrix<Real>& x,
const lp::affine::Ctrl<Real>& ctrl )
{
EL_DEBUG_CSE
const Int m = A.Height();
const Int n = A.Width();
const Range<Int> xInd(0,n), uInd(n,n+m), vInd(n+m,n+2*m);
Matrix<Real> c, AHat, G, h;
// c := [0;1;1]
// ============
Zeros( c, n+2*m, 1 );
auto cuv = c( IR(n,n+2*m), ALL );
Fill( cuv, Real(1) );
// \hat A := [A, I, -I]
// ====================
Zeros( AHat, m, n+2*m );
auto AHatx = AHat( IR(0,m), xInd );
auto AHatu = AHat( IR(0,m), uInd );
auto AHatv = AHat( IR(0,m), vInd );
AHatx = A;
FillDiagonal( AHatu, Real( 1) );
FillDiagonal( AHatv, Real(-1) );
// G := | 0 -I 0 |
// | 0 0 -I |
// ================
Zeros( G, 2*m, n+2*m );
auto Guv = G( IR(0,2*m), IR(n,n+2*m) );
FillDiagonal( Guv, Real(-1) );
// h := | 0 |
// | 0 |
// ==========
Zeros( h, 2*m, 1 );
// Solve the affine linear program
// ===============================
Matrix<Real> xHat, y, z, s;
LP( AHat, G, b, c, h, xHat, y, z, s, ctrl );
// Extract x
// ==========
x = xHat( xInd, ALL );
}
void Lauchli( Matrix<T>& A, Int n, T mu )
{
DEBUG_ONLY(CallStackEntry cse("Lauchli"))
Zeros( A, n+1, n );
// Set the first row to all ones
auto a0 = A( IR(0,1), IR(0,n) );
Fill( a0, T(1) );
// Set the subdiagonal to mu
FillDiagonal( A, mu, -1 );
}
void Lauchli( AbstractDistMatrix<T>& A, Int n, T mu )
{
DEBUG_ONLY(CallStackEntry cse("Lauchli"))
Zeros( A, n+1, n );
// Set the first row to all ones
unique_ptr<AbstractDistMatrix<T>> a0( A.Construct(A.Grid(),A.Root()) );
View( *a0, A, IR(0,1), IR(0,n) );
Fill( *a0, T(1) );
// Set the subdiagonal to mu
FillDiagonal( A, mu, -1 );
}
void GEPPGrowth( Matrix<T>& A, Int n )
{
DEBUG_ONLY(CSE cse("GEPPGrowth"))
Identity( A, n, n );
if( n <= 1 )
return;
// Set the last column to all ones
auto aLast = A( IR(0,n), IR(n-1,n) );
Fill( aLast, T(1) );
// Set the subdiagonals to -1
for( Int j=1; j<n; ++j )
FillDiagonal( A, T(-1), -j );
}
void GEPPGrowth( ElementalMatrix<T>& A, Int n )
{
DEBUG_ONLY(CSE cse("GEPPGrowth"))
Identity( A, n, n );
if( n <= 1 )
return;
// Set the last column to all ones
unique_ptr<ElementalMatrix<T>> aLast( A.Construct(A.Grid(),A.Root()) );
View( *aLast, A, IR(0,n), IR(n-1,n) );
Fill( *aLast, T(1) );
// Set the subdiagonals to -1
for( Int j=1; j<n; ++j )
FillDiagonal( A, T(-1), -j );
}
void Ridge
( Orientation orientation,
const Matrix<Field>& A,
const Matrix<Field>& B,
Base<Field> gamma,
Matrix<Field>& X,
RidgeAlg alg )
{
EL_DEBUG_CSE
const bool normal = ( orientation==NORMAL );
const Int m = ( normal ? A.Height() : A.Width() );
const Int n = ( normal ? A.Width() : A.Height() );
if( orientation == TRANSPOSE && IsComplex<Field>::value )
LogicError("Transpose version of complex Ridge not yet supported");
if( m >= n )
{
Matrix<Field> Z;
if( alg == RIDGE_CHOLESKY )
{
if( orientation == NORMAL )
Herk( LOWER, ADJOINT, Base<Field>(1), A, Z );
else
Herk( LOWER, NORMAL, Base<Field>(1), A, Z );
ShiftDiagonal( Z, Field(gamma*gamma) );
Cholesky( LOWER, Z );
if( orientation == NORMAL )
Gemm( ADJOINT, NORMAL, Field(1), A, B, X );
else
Gemm( NORMAL, NORMAL, Field(1), A, B, X );
cholesky::SolveAfter( LOWER, NORMAL, Z, X );
}
else if( alg == RIDGE_QR )
{
Zeros( Z, m+n, n );
auto ZT = Z( IR(0,m), IR(0,n) );
auto ZB = Z( IR(m,m+n), IR(0,n) );
if( orientation == NORMAL )
ZT = A;
else
Adjoint( A, ZT );
FillDiagonal( ZB, Field(gamma) );
// NOTE: This QR factorization could exploit the upper-triangular
// structure of the diagonal matrix ZB
qr::ExplicitTriang( Z );
if( orientation == NORMAL )
Gemm( ADJOINT, NORMAL, Field(1), A, B, X );
else
Gemm( NORMAL, NORMAL, Field(1), A, B, X );
cholesky::SolveAfter( LOWER, NORMAL, Z, X );
}
else
{
Matrix<Field> U, V;
Matrix<Base<Field>> s;
if( orientation == NORMAL )
{
SVDCtrl<Base<Field>> ctrl;
ctrl.overwrite = false;
SVD( A, U, s, V, ctrl );
}
else
{
Matrix<Field> AAdj;
Adjoint( A, AAdj );
SVDCtrl<Base<Field>> ctrl;
ctrl.overwrite = true;
SVD( AAdj, U, s, V, ctrl );
}
auto sigmaMap =
[=]( const Base<Field>& sigma )
{ return sigma / (sigma*sigma + gamma*gamma); };
EntrywiseMap( s, MakeFunction(sigmaMap) );
Gemm( ADJOINT, NORMAL, Field(1), U, B, X );
DiagonalScale( LEFT, NORMAL, s, X );
U = X;
Gemm( NORMAL, NORMAL, Field(1), V, U, X );
}
}
else
{
LogicError("This case not yet supported");
}
}
void MakeIdentity( AbstractBlockDistMatrix<T>& I )
{
DEBUG_ONLY(CallStackEntry cse("MakeIdentity"))
Zero( I );
FillDiagonal( I, T(1) );
}
void IPM
( const AbstractDistMatrix<Real>& APre,
const AbstractDistMatrix<Real>& b,
Real lambda,
AbstractDistMatrix<Real>& xPre,
const qp::affine::Ctrl<Real>& ctrl )
{
EL_DEBUG_CSE
DistMatrixReadProxy<Real,Real,MC,MR> AProx( APre );
const auto& A = AProx.GetLocked();
DistMatrixWriteProxy<Real,Real,MC,MR> xProx( xPre );
auto& x = xProx.Get();
const Int m = A.Height();
const Int n = A.Width();
const Grid& g = A.Grid();
const Range<Int> uInd(0,n), vInd(n,2*n), rInd(2*n,2*n+m);
DistMatrix<Real> Q(g), c(g), AHat(g), G(g), h(g);
// Q := | 0 0 0 |
// | 0 0 0 |
// | 0 0 I |
// ==============
Zeros( Q, 2*n+m, 2*n+m );
auto Qrr = Q( rInd, rInd );
FillDiagonal( Qrr, Real(1) );
// c := lambda*[1;1;0]
// ===================
Zeros( c, 2*n+m, 1 );
auto cuv = c( IR(0,2*n), ALL );
Fill( cuv, lambda );
// \hat A := [A, -A, I]
// ====================
Zeros( AHat, m, 2*n+m );
auto AHatu = AHat( IR(0,m), uInd );
auto AHatv = AHat( IR(0,m), vInd );
auto AHatr = AHat( IR(0,m), rInd );
AHatu = A;
AHatv -= A;
FillDiagonal( AHatr, Real(1) );
// G := | -I 0 0 |
// | 0 -I 0 |
// ================
Zeros( G, 2*n, 2*n+m );
FillDiagonal( G, Real(-1) );
// h := 0
// ======
Zeros( h, 2*n, 1 );
// Solve the affine QP
// ===================
DistMatrix<Real> xHat(g), y(g), z(g), s(g);
QP( Q, AHat, G, b, c, h, xHat, y, z, s, ctrl );
// x := u - v
// ==========
x = xHat( uInd, ALL );
x -= xHat( vInd, ALL );
}
Int ADMM
( const Matrix<Real>& A,
const Matrix<Real>& b,
const Matrix<Real>& c,
Matrix<Real>& z,
const ADMMCtrl<Real>& ctrl )
{
EL_DEBUG_CSE
// Cache a custom partially-pivoted LU factorization of
// | rho*I A^H | = | B11 B12 |
// | A 0 | | B21 B22 |
// by (justifiably) avoiding pivoting in the first n steps of
// the factorization, so that
// [I,rho*I] = lu(rho*I).
// The factorization would then proceed with
// B21 := B21 U11^{-1} = A (rho*I)^{-1} = A/rho
// B12 := L11^{-1} B12 = I A^H = A^H.
// The Schur complement would then be
// B22 := B22 - B21 B12 = 0 - (A*A^H)/rho.
// We then factor said matrix with LU with partial pivoting and
// swap the necessary rows of B21 in order to implicitly commute
// the row pivots with the Gauss transforms in the manner standard
// for GEPP. Unless A A' is singular, pivoting should not be needed,
// as Cholesky factorization of the negative matrix should be valid.
//
// The result is the factorization
// | I 0 | | rho*I A^H | = | I 0 | | rho*I U12 |,
// | 0 P22 | | A 0 | | L21 L22 | | 0 U22 |
// where [L22,U22] are stored within B22.
Matrix<Real> U12, L21, B22, bPiv;
Adjoint( A, U12 );
L21 = A;
L21 *= 1/ctrl.rho;
Herk( LOWER, NORMAL, -1/ctrl.rho, A, B22 );
MakeHermitian( LOWER, B22 );
// TODO: Replace with sparse-direct Cholesky version?
Permutation P2;
LU( B22, P2 );
P2.PermuteRows( L21 );
bPiv = b;
P2.PermuteRows( bPiv );
// Possibly form the inverse of L22 U22
Matrix<Real> X22;
if( ctrl.inv )
{
X22 = B22;
MakeTrapezoidal( LOWER, X22 );
FillDiagonal( X22, Real(1) );
TriangularInverse( LOWER, UNIT, X22 );
Trsm( LEFT, UPPER, NORMAL, NON_UNIT, Real(1), B22, X22 );
}
Int numIter=0;
const Int m = A.Height();
const Int n = A.Width();
Matrix<Real> g, xTmp, y, t;
Zeros( g, m+n, 1 );
PartitionDown( g, xTmp, y, n );
Matrix<Real> x, u, zOld, xHat;
Zeros( z, n, 1 );
Zeros( u, n, 1 );
Zeros( t, n, 1 );
while( numIter < ctrl.maxIter )
{
zOld = z;
// Find x from
// | rho*I A^H | | x | = | rho*(z-u)-c |
// | A 0 | | y | | b |
// via our cached custom factorization:
//
// |x| = inv(U) inv(L) P' |rho*(z-u)-c|
// |y| |b |
// = |rho*I U12|^{-1} |I 0 | |I 0 | |rho*(z-u)-c|
// = |0 U22| |L21 L22| |0 P22'| |b |
// = " " |rho*(z-u)-c|
// | P22' b |
xTmp = z;
xTmp -= u;
xTmp *= ctrl.rho;
xTmp -= c;
y = bPiv;
Gemv( NORMAL, Real(-1), L21, xTmp, Real(1), y );
if( ctrl.inv )
{
Gemv( NORMAL, Real(1), X22, y, t );
y = t;
}
else
{
Trsv( LOWER, NORMAL, UNIT, B22, y );
Trsv( UPPER, NORMAL, NON_UNIT, B22, y );
}
Gemv( NORMAL, Real(-1), U12, y, Real(1), xTmp );
xTmp *= 1/ctrl.rho;
// xHat := alpha*x + (1-alpha)*zOld
xHat = xTmp;
xHat *= ctrl.alpha;
Axpy( 1-ctrl.alpha, zOld, xHat );
// z := pos(xHat+u)
z = xHat;
z += u;
LowerClip( z, Real(0) );
// u := u + (xHat-z)
u += xHat;
u -= z;
const Real objective = Dot( c, xTmp );
// rNorm := || x - z ||_2
t = xTmp;
t -= z;
const Real rNorm = FrobeniusNorm( t );
// sNorm := |rho| || z - zOld ||_2
t = z;
t -= zOld;
const Real sNorm = Abs(ctrl.rho)*FrobeniusNorm( t );
const Real epsPri = Sqrt(Real(n))*ctrl.absTol +
ctrl.relTol*Max(FrobeniusNorm(xTmp),FrobeniusNorm(z));
const Real epsDual = Sqrt(Real(n))*ctrl.absTol +
ctrl.relTol*Abs(ctrl.rho)*FrobeniusNorm(u);
if( ctrl.print )
{
t = xTmp;
LowerClip( t, Real(0) );
t -= xTmp;
const Real clipDist = FrobeniusNorm( t );
cout << numIter << ": "
<< "||x-z||_2=" << rNorm << ", "
<< "epsPri=" << epsPri << ", "
<< "|rho| ||z-zOld||_2=" << sNorm << ", "
<< "epsDual=" << epsDual << ", "
<< "||x-Pos(x)||_2=" << clipDist << ", "
<< "c'x=" << objective << endl;
}
if( rNorm < epsPri && sNorm < epsDual )
break;
++numIter;
}
if( ctrl.maxIter == numIter )
cout << "ADMM failed to converge" << endl;
x = xTmp;
return numIter;
}
