/**
@see magmablas_zlaswp_q
@ingroup magma_zaux2
********************************************************************/
extern "C" void
magmablas_zlaswp(
magma_int_t n,
magmaDoubleComplex_ptr dAT, magma_int_t ldda,
magma_int_t k1, magma_int_t k2,
const magma_int_t *ipiv, magma_int_t inci )
{
magmablas_zlaswp_q( n, dAT, ldda, k1, k2, ipiv, inci, magmablasGetQueue() );
}
/**
Purpose
-------
ZGETRF_m computes an LU factorization of a general M-by-N matrix A
using partial pivoting with row interchanges. This version does not
require work space on the GPU passed as input. GPU memory is allocated
in the routine. The matrix may exceed the GPU memory.
The factorization has the form
A = P * L * U
where P is a permutation matrix, L is lower triangular with unit
diagonal elements (lower trapezoidal if m > n), and U is upper
triangular (upper trapezoidal if m < n).
This is the right-looking Level 3 BLAS version of the algorithm.
Note: The factorization of big panel is done calling multiple-gpu-interface.
Pivots are applied on GPU within the big panel.
Arguments
---------
@param[in]
ngpu INTEGER
Number of GPUs to use. ngpu > 0.
@param[in]
m INTEGER
The number of rows of the matrix A. M >= 0.
@param[in]
n INTEGER
The number of columns of the matrix A. N >= 0.
@param[in,out]
A COMPLEX_16 array, dimension (LDA,N)
On entry, the M-by-N matrix to be factored.
On exit, the factors L and U from the factorization
A = P*L*U; the unit diagonal elements of L are not stored.
\n
Higher performance is achieved if A is in pinned memory, e.g.
allocated using magma_malloc_pinned.
@param[in]
lda INTEGER
The leading dimension of the array A. LDA >= max(1,M).
@param[out]
ipiv INTEGER array, dimension (min(M,N))
The pivot indices; for 1 <= i <= min(M,N), row i of the
matrix was interchanged with row IPIV(i).
@param[out]
info INTEGER
- = 0: successful exit
- < 0: if INFO = -i, the i-th argument had an illegal value
or another error occured, such as memory allocation failed.
- > 0: if INFO = i, U(i,i) is exactly zero. The factorization
has been completed, but the factor U is exactly
singular, and division by zero will occur if it is used
to solve a system of equations.
@ingroup magma_zgesv_comp
********************************************************************/
extern "C" magma_int_t
magma_zgetrf_m(
magma_int_t ngpu,
magma_int_t m, magma_int_t n,
magmaDoubleComplex *A, magma_int_t lda, magma_int_t *ipiv,
magma_int_t *info)
{
#define A(i,j) (A + (j)*lda + (i))
#define dAT(d,i,j) (dAT[d] + (i)*nb*ldn_local + (j)*nb)
#define dPT(d,i,j) (dPT[d] + (i)*nb*nb + (j)*nb*maxm)
magma_timer_t time=0, time_total=0, time_alloc=0, time_set=0, time_get=0, time_comp=0;
timer_start( time_total );
real_Double_t flops;
magmaDoubleComplex c_one = MAGMA_Z_ONE;
magmaDoubleComplex c_neg_one = MAGMA_Z_NEG_ONE;
magmaDoubleComplex *dAT[MagmaMaxGPUs], *dA[MagmaMaxGPUs], *dPT[MagmaMaxGPUs];
magma_int_t iinfo = 0, nb, nbi, maxm, n_local[MagmaMaxGPUs], ldn_local;
magma_int_t N, M, NB, NBk, I, d, ngpu0 = ngpu;
magma_int_t ii, jj, h, offset, ib, rows;
magma_queue_t stream[MagmaMaxGPUs][2];
magma_event_t event[MagmaMaxGPUs][2];
*info = 0;
if (m < 0)
*info = -1;
else if (n < 0)
*info = -2;
else if (lda < max(1,m))
*info = -4;
if (*info != 0) {
magma_xerbla( __func__, -(*info) );
return *info;
}
/* Quick return if possible */
if (m == 0 || n == 0)
return *info;
magma_device_t orig_dev;
magma_getdevice( &orig_dev );
magma_queue_t orig_stream;
magmablasGetKernelStream( &orig_stream );
/* initialize nb */
nb = magma_get_zgetrf_nb(m);
maxm = ((m + 31)/32)*32;
/* figure out NB */
size_t freeMem, totalMem;
cudaMemGetInfo( &freeMem, &totalMem );
freeMem /= sizeof(magmaDoubleComplex);
/* number of columns in the big panel */
h = 1+(2+ngpu0);
NB = (magma_int_t)(0.8*freeMem/maxm-h*nb);
const char* ngr_nb_char = getenv("MAGMA_NGR_NB");
if ( ngr_nb_char != NULL )
NB = max( nb, min( NB, atoi(ngr_nb_char) ) );
//NB = 5*max(nb,32);
if ( ngpu0 > ceil((double)NB/nb) ) {
ngpu = (int)ceil((double)NB/nb);
h = 1+(2+ngpu);
NB = (magma_int_t)(0.8*freeMem/maxm-h*nb);
} else {
ngpu = ngpu0;
}
if ( ngpu*NB >= n ) {
#ifdef CHECK_ZGETRF_OOC
printf( " * still fit in GPU memory.\n" );
#endif
NB = n;
} else {
#ifdef CHECK_ZGETRF_OOC
printf( " * don't fit in GPU memory.\n" );
#endif
NB = ngpu*NB;
NB = max( nb, (NB / nb) * nb); /* making sure it's devisable by nb (x64) */
}
#ifdef CHECK_ZGETRF_OOC
if ( NB != n ) printf( " * running in out-core mode (n=%d, NB=%d, nb=%d, freeMem=%.2e).\n", n, NB, nb, (double)freeMem );
else printf( " * running in in-core mode (n=%d, NB=%d, nb=%d, freeMem=%.2e).\n", n, NB, nb, (double)freeMem );
#endif
if ( (nb <= 1) || (nb >= min(m,n)) ) {
/* Use CPU code for scalar of one tile. */
lapackf77_zgetrf(&m, &n, A, &lda, ipiv, info);
} else {
/* Use hybrid blocked code. */
/* allocate memory on GPU to store the big panel */
timer_start( time_alloc );
n_local[0] = (NB/nb)/ngpu;
if ( NB%(nb*ngpu) != 0 )
n_local[0]++;
n_local[0] *= nb;
ldn_local = ((n_local[0]+31)/32)*32;
for( d=0; d < ngpu; d++ ) {
magma_setdevice(d);
if (MAGMA_SUCCESS != magma_zmalloc( &dA[d], (ldn_local+h*nb)*maxm )) {
*info = MAGMA_ERR_DEVICE_ALLOC;
return *info;
}
dPT[d] = dA[d] + nb*maxm; /* for storing the previous panel from CPU */
dAT[d] = dA[d] + h*nb*maxm; /* for storing the big panel */
magma_queue_create( &stream[d][0] );
magma_queue_create( &stream[d][1] );
magma_event_create( &event[d][0] );
magma_event_create( &event[d][1] );
}
//magma_setdevice(0);
timer_stop( time_alloc );
for( I=0; I < n; I += NB ) {
M = m;
N = min( NB, n-I ); /* number of columns in this big panel */
//s = min( max(m-I,0), N )/nb; /* number of small block-columns in this big panel */
maxm = ((M + 31)/32)*32;
if ( ngpu0 > ceil((double)N/nb) ) {
ngpu = (int)ceil((double)N/nb);
} else {
ngpu = ngpu0;
}
for( d=0; d < ngpu; d++ ) {
n_local[d] = ((N/nb)/ngpu)*nb;
if (d < (N/nb)%ngpu)
n_local[d] += nb;
else if (d == (N/nb)%ngpu)
n_local[d] += N%nb;
}
ldn_local = ((n_local[0]+31)/32)*32;
/* upload the next big panel into GPU, transpose (A->A'), and pivot it */
timer_start( time );
magmablas_zsetmatrix_transpose_mgpu(ngpu, stream, A(0,I), lda,
dAT, ldn_local, dA, maxm, M, N, nb);
for( d=0; d < ngpu; d++ ) {
magma_setdevice(d);
magma_queue_sync( stream[d][0] );
magma_queue_sync( stream[d][1] );
magmablasSetKernelStream(NULL);
}
time_set += timer_stop( time );
timer_start( time );
/* == --------------------------------------------------------------- == */
/* == loop around the previous big-panels to update the new big-panel == */
for( offset = 0; offset < min(m,I); offset += NB ) {
NBk = min( m-offset, NB );
/* start sending the first tile from the previous big-panels to gpus */
for( d=0; d < ngpu; d++ ) {
magma_setdevice(d);
nbi = min( nb, NBk );
magma_zsetmatrix_async( (M-offset), nbi,
A(offset,offset), lda,
dA[d], (maxm-offset), stream[d][0] );
/* make sure the previous update finished */
magmablasSetKernelStream(stream[d][0]);
//magma_queue_sync( stream[d][1] );
magma_queue_wait_event( stream[d][0], event[d][0] );
/* transpose */
magmablas_ztranspose( M-offset, nbi, dA[d], maxm-offset, dPT(d,0,0), nb );
}
/* applying the pivot from the previous big-panel */
for( d=0; d < ngpu; d++ ) {
magma_setdevice(d);
magmablas_zlaswp_q( ldn_local, dAT(d,0,0), ldn_local, offset+1, offset+NBk, ipiv, 1, stream[d][1] );
}
/* == going through each block-column of previous big-panels == */
for( jj=0, ib=offset/nb; jj < NBk; jj += nb, ib++ ) {
ii = offset+jj;
rows = maxm - ii;
nbi = min( nb, NBk-jj );
for( d=0; d < ngpu; d++ ) {
magma_setdevice(d);
/* wait for a block-column on GPU */
magma_queue_sync( stream[d][0] );
/* start sending next column */
if ( jj+nb < NBk ) {
magma_zsetmatrix_async( (M-ii-nb), min(nb,NBk-jj-nb),
A(ii+nb,ii+nb), lda,
dA[d], (rows-nb), stream[d][0] );
/* make sure the previous update finished */
magmablasSetKernelStream(stream[d][0]);
//magma_queue_sync( stream[d][1] );
magma_queue_wait_event( stream[d][0], event[d][(1+jj/nb)%2] );
/* transpose next column */
magmablas_ztranspose( M-ii-nb, nb, dA[d], rows-nb, dPT(d,0,(1+jj/nb)%2), nb );
}
/* update with the block column */
magmablasSetKernelStream(stream[d][1]);
magma_ztrsm( MagmaRight, MagmaUpper, MagmaNoTrans, MagmaUnit,
n_local[d], nbi, c_one, dPT(d,0,(jj/nb)%2), nb, dAT(d,ib,0), ldn_local );
if ( M > ii+nb ) {
magma_zgemm( MagmaNoTrans, MagmaNoTrans,
n_local[d], M-(ii+nb), nbi, c_neg_one, dAT(d,ib,0), ldn_local,
dPT(d,1,(jj/nb)%2), nb, c_one, dAT(d,ib+1,0), ldn_local );
}
magma_event_record( event[d][(jj/nb)%2], stream[d][1] );
} /* end of for each block-columns in a big-panel */
}
} /* end of for each previous big-panels */
for( d=0; d < ngpu; d++ ) {
magma_setdevice(d);
magma_queue_sync( stream[d][0] );
magma_queue_sync( stream[d][1] );
magmablasSetKernelStream(NULL);
}
/* calling magma-gpu interface to panel-factorize the big panel */
if ( M > I ) {
magma_zgetrf2_mgpu(ngpu, M-I, N, nb, I, dAT, ldn_local, ipiv+I, dA, A(0,I), lda,
stream, &iinfo);
if ( iinfo < 0 ) {
*info = iinfo;
break;
} else if ( iinfo != 0 ) {
*info = iinfo + I * NB;
//break;
}
/* adjust pivots */
for( ii=I; ii < min(I+N,m); ii++ )
ipiv[ii] += I;
}
time_comp += timer_stop( time );
/* download the current big panel to CPU */
timer_start( time );
magmablas_zgetmatrix_transpose_mgpu(ngpu, stream, dAT, ldn_local, A(0,I), lda, dA, maxm, M, N, nb);
for( d=0; d < ngpu; d++ ) {
magma_setdevice(d);
magma_queue_sync( stream[d][0] );
magma_queue_sync( stream[d][1] );
magmablasSetKernelStream(NULL);
}
time_get += timer_stop( time );
} /* end of for */
timer_stop( time_total );
flops = FLOPS_ZGETRF( m, n ) / 1e9;
timer_printf(" memory-allocation time: %e\n", time_alloc );
timer_printf(" NB=%d nb=%d\n", (int) NB, (int) nb );
timer_printf(" memcopy and transpose %e seconds\n", time_set );
timer_printf(" total time %e seconds\n", time_total );
timer_printf(" Performance %f GFlop/s, %f seconds without htod and dtoh\n", flops / (time_comp), time_comp );
timer_printf(" Performance %f GFlop/s, %f seconds with htod\n", flops / (time_comp + time_set), time_comp + time_set );
timer_printf(" Performance %f GFlop/s, %f seconds with dtoh\n", flops / (time_comp + time_get), time_comp + time_get );
timer_printf(" Performance %f GFlop/s, %f seconds without memory-allocation\n", flops / (time_total - time_alloc), time_total - time_alloc );
for( d=0; d < ngpu0; d++ ) {
magma_setdevice(d);
magma_free( dA[d] );
magma_event_destroy( event[d][0] );
magma_event_destroy( event[d][1] );
magma_queue_destroy( stream[d][0] );
magma_queue_destroy( stream[d][1] );
}
magma_setdevice( orig_dev );
magmablasSetKernelStream( orig_stream );
}
if ( *info >= 0 )
magma_zgetrf_piv(m, n, NB, A, lda, ipiv, info);
return *info;
} /* magma_zgetrf_m */
/**
Purpose
-------
ZGETRF computes an LU factorization of a general M-by-N matrix A
using partial pivoting with row interchanges.
The factorization has the form
A = P * L * U
where P is a permutation matrix, L is lower triangular with unit
diagonal elements (lower trapezoidal if m > n), and U is upper
triangular (upper trapezoidal if m < n).
This is the right-looking Level 3 BLAS version of the algorithm.
Use two buffer to send panels.
Arguments
---------
@param[in]
ngpu INTEGER
Number of GPUs to use. ngpu > 0.
@param[in]
m INTEGER
The number of rows of the matrix A. M >= 0.
@param[in]
n INTEGER
The number of columns of the matrix A. N >= 0.
@param[in,out]
d_lAT COMPLEX_16 array of pointers on the GPU, dimension (ngpu).
On entry, the M-by-N matrix A distributed over GPUs
(d_lAT[d] points to the local matrix on d-th GPU).
It uses a 1D block column cyclic format (with the block size
nb), and each local matrix is stored by row.
On exit, the factors L and U from the factorization
A = P*L*U; the unit diagonal elements of L are not stored.
@param[in]
lddat INTEGER
The leading dimension of the array d_lAT[d]. LDDA >= max(1,M).
@param[out]
ipiv INTEGER array, dimension (min(M,N))
The pivot indices; for 1 <= i <= min(M,N), row i of the
matrix was interchanged with row IPIV(i).
@param (workspace) on device
d_lAP COMPLEX_16 array of pointers on the GPU, dimension (ngpu).
d_lAP[d] is the workspace on d-th GPU. Each local workspace
must be of size (3+ngpu)*nb*maxm, where maxm is m rounded
up to a multiple of 32 and nb is the block size.
@param (workspace)
W COMPLEX_16 array, dimension (ngpu*nb*maxm).
It is used to store panel on CPU.
@param[in]
ldw INTEGER
The leading dimension of the workspace w.
@param[in]
queues magma_queue_t
queues[d] points to the streams for the d-th GPU to execute
in. Each GPU require two streams.
@param[out]
info INTEGER
- = 0: successful exit
- < 0: if INFO = -i, the i-th argument had an illegal value
or another error occured, such as memory allocation failed.
- > 0: if INFO = i, U(i,i) is exactly zero. The factorization
has been completed, but the factor U is exactly
singular, and division by zero will occur if it is used
to solve a system of equations.
@ingroup magma_zgesv_comp
********************************************************************/
extern "C" magma_int_t
magma_zgetrf2_mgpu(
magma_int_t ngpu,
magma_int_t m, magma_int_t n, magma_int_t nb, magma_int_t offset,
magmaDoubleComplex_ptr d_lAT[], magma_int_t lddat, magma_int_t *ipiv,
magmaDoubleComplex_ptr d_lAP[],
magmaDoubleComplex *W, magma_int_t ldw,
magma_queue_t queues[][2],
magma_int_t *info)
{
#define dAT(id,i,j) (d_lAT[(id)] + ((offset)+(i)*nb)*lddat + (j)*nb)
#define W(j) (W + ((j)%ngpu)*nb*ldw)
magmaDoubleComplex c_one = MAGMA_Z_ONE;
magmaDoubleComplex c_neg_one = MAGMA_Z_NEG_ONE;
magma_int_t block_size = 32;
magma_int_t iinfo, n_local[MagmaMaxGPUs];
magma_int_t maxm, mindim;
magma_int_t i, j, d, dd, rows, cols, s, ldpan[MagmaMaxGPUs];
magma_int_t id, j_local, j_local2, nb0, nb1, h = 2+ngpu;
magmaDoubleComplex *d_panel[MagmaMaxGPUs], *panel_local[MagmaMaxGPUs];
/* Check arguments */
*info = 0;
if (m < 0)
*info = -2;
else if (n < 0)
*info = -3;
else if (ngpu*lddat < max(1,n))
*info = -5;
if (*info != 0) {
magma_xerbla( __func__, -(*info) );
return *info;
}
/* Quick return if possible */
if (m == 0 || n == 0)
return *info;
/* Function Body */
mindim = min(m, n);
if ( ngpu > ceil((double)n/nb) ) {
*info = -1;
return *info;
}
magma_device_t orig_dev;
magma_getdevice( &orig_dev );
magma_queue_t orig_stream;
magmablasGetKernelStream( &orig_stream );
/* Use hybrid blocked code. */
maxm = ((m + block_size-1)/block_size)*block_size;
/* some initializations */
for (d=0; d < ngpu; d++) {
magma_setdevice(d);
n_local[d] = ((n/nb)/ngpu)*nb;
if (d < (n/nb) % ngpu)
n_local[d] += nb;
else if (d == (n/nb) % ngpu)
n_local[d] += n % nb;
/* workspaces */
d_panel[d] = &(d_lAP[d][h*nb*maxm]); /* temporary panel storage */
}
trace_init( 1, ngpu, 2, (CUstream_st**)queues );
/* start sending the panel to cpu */
nb0 = min(mindim, nb);
magma_setdevice(0);
magmablasSetKernelStream(queues[0][1]);
trace_gpu_start( 0, 1, "comm", "get" );
magmablas_ztranspose( nb0, m, dAT(0,0,0), lddat, d_lAP[0], maxm );
magma_zgetmatrix_async( m, nb0,
d_lAP[0], maxm,
W(0), ldw, queues[0][1] );
trace_gpu_end( 0, 1 );
/* ------------------------------------------------------------------------------------- */
magma_timer_t time=0;
timer_start( time );
s = mindim / nb;
for( j=0; j < s; j++ ) {
/* Set the GPU number that holds the current panel */
id = j % ngpu;
magma_setdevice(id);
/* Set the local index where the current panel is */
j_local = j/ngpu;
cols = maxm - j*nb;
rows = m - j*nb;
/* synchronize j-th panel from id-th gpu into work */
magma_queue_sync( queues[id][1] );
/* j-th panel factorization */
trace_cpu_start( 0, "getrf", "getrf" );
lapackf77_zgetrf( &rows, &nb, W(j), &ldw, ipiv+j*nb, &iinfo);
if ( (*info == 0) && (iinfo > 0) ) {
*info = iinfo + j*nb;
}
trace_cpu_end( 0 );
/* start sending the panel to all the gpus */
d = (j+1) % ngpu;
for( dd=0; dd < ngpu; dd++ ) {
magma_setdevice(d);
trace_gpu_start( 0, 1, "comm", "set" );
magma_zsetmatrix_async( rows, nb,
W(j), ldw,
&d_lAP[d][(j%h)*nb*maxm], cols,
queues[d][1] );
trace_gpu_end( 0, 1 );
d = (d+1) % ngpu;
}
/* apply the pivoting */
d = (j+1) % ngpu;
for( dd=0; dd < ngpu; dd++ ) {
magma_setdevice(d);
trace_gpu_start( d, 1, "pivot", "pivot" );
if ( dd == 0 ) {
for( i=j*nb; i < j*nb + nb; ++i ) {
ipiv[i] += j*nb;
}
}
magmablas_zlaswp_q( lddat, dAT(d,0,0), lddat, j*nb + 1, j*nb + nb, ipiv, 1, queues[d][0] );
trace_gpu_end( d, 1 );
d = (d+1) % ngpu;
}
/* update the trailing-matrix/look-ahead */
d = (j+1) % ngpu;
for( dd=0; dd < ngpu; dd++ ) {
magma_setdevice(d);
/* storage for panel */
if ( d == id ) {
/* the panel belond to this gpu */
panel_local[d] = dAT(d,j,j_local);
ldpan[d] = lddat;
/* next column */
j_local2 = j_local+1;
} else {
/* the panel belong to another gpu */
panel_local[d] = d_panel[d];
ldpan[d] = nb;
/* next column */
j_local2 = j_local;
if ( d < id ) j_local2 ++;
}
/* the size of the next column */
if ( s > (j+1) ) {
nb0 = nb;
} else {
nb0 = n_local[d]-nb*(s/ngpu);
if ( d < s % ngpu ) nb0 -= nb;
}
if ( d == (j+1) % ngpu) {
/* owns the next column, look-ahead the column */
nb1 = nb0;
magmablasSetKernelStream(queues[d][1]);
/* make sure all the pivoting has been applied */
magma_queue_sync(queues[d][0]);
trace_gpu_start( d, 1, "gemm", "gemm" );
/* transpose panel on GPU */
magmablas_ztranspose( rows, nb, &d_lAP[d][(j%h)*nb*maxm], cols, panel_local[d], ldpan[d] );
/* synch for remaining update */
magma_queue_sync(queues[d][1]);
} else {
/* update the entire trailing matrix */
nb1 = n_local[d] - j_local2*nb;
magmablasSetKernelStream(queues[d][0]);
/* synchronization to make sure panel arrived on gpu */
magma_queue_sync(queues[d][1]);
trace_gpu_start( d, 0, "gemm", "gemm" );
/* transpose panel on GPU */
magmablas_ztranspose( rows, nb, &d_lAP[d][(j%h)*nb*maxm], cols, panel_local[d], ldpan[d] );
}
/* gpu updating the trailing matrix */
magma_ztrsm( MagmaRight, MagmaUpper, MagmaNoTrans, MagmaUnit,
nb1, nb, c_one,
panel_local[d], ldpan[d],
dAT(d, j, j_local2), lddat);
magma_zgemm( MagmaNoTrans, MagmaNoTrans,
nb1, m-(j+1)*nb, nb,
c_neg_one, dAT(d, j, j_local2), lddat,
&(panel_local[d][nb*ldpan[d]]), ldpan[d],
c_one, dAT(d, j+1, j_local2), lddat );
if ( d == (j+1) % ngpu ) {
/* Set the local index where the current panel is */
int loff = j+1;
int j_local = (j+1)/ngpu;
int ldda = maxm - (j+1)*nb;
int cols = m - (j+1)*nb;
nb0 = min(nb, mindim - (j+1)*nb); /* size of the diagonal block */
trace_gpu_end( d, 1 );
if ( nb0 > 0 ) {
/* transpose the panel for sending it to cpu */
trace_gpu_start( d, 1, "comm", "get" );
magmablas_ztranspose( nb0, m-(j+1)*nb, dAT(d,loff,j_local), lddat, &d_lAP[d][((j+1)%h)*nb*maxm], ldda );
/* send the panel to cpu */
magma_zgetmatrix_async( cols, nb0,
&d_lAP[d][((j+1)%h)*nb*maxm], ldda,
W(j+1), ldw, queues[d][1] );
trace_gpu_end( d, 1 );
}
} else {
trace_gpu_end( d, 0 );
}
d = (d+1) % ngpu;
}
/* update the remaining matrix by gpu owning the next panel */
if ( (j+1) < s ) {
int j_local = (j+1)/ngpu;
int rows = m - (j+1)*nb;
d = (j+1) % ngpu;
magma_setdevice(d);
magmablasSetKernelStream(queues[d][0]);
trace_gpu_start( d, 0, "gemm", "gemm" );
magma_ztrsm( MagmaRight, MagmaUpper, MagmaNoTrans, MagmaUnit,
n_local[d] - (j_local+1)*nb, nb,
c_one, panel_local[d], ldpan[d],
dAT(d,j,j_local+1), lddat );
magma_zgemm( MagmaNoTrans, MagmaNoTrans,
n_local[d]-(j_local+1)*nb, rows, nb,
c_neg_one, dAT(d,j,j_local+1), lddat,
&(panel_local[d][nb*ldpan[d]]), ldpan[d],
c_one, dAT(d,j+1, j_local+1), lddat );
trace_gpu_end( d, 0 );
}
} /* end of for j=1..s */
/* ------------------------------------------------------------------------------ */
/* Set the GPU number that holds the last panel */
id = s % ngpu;
/* Set the local index where the last panel is */
j_local = s/ngpu;
/* size of the last diagonal-block */
nb0 = min(m - s*nb, n - s*nb);
rows = m - s*nb;
cols = maxm - s*nb;
if ( nb0 > 0 ) {
magma_setdevice(id);
/* wait for the last panel on cpu */
magma_queue_sync( queues[id][1] );
/* factor on cpu */
lapackf77_zgetrf( &rows, &nb0, W(s), &ldw, ipiv+s*nb, &iinfo);
if ( (*info == 0) && (iinfo > 0) )
*info = iinfo + s*nb;
/* send the factor to gpus */
for( d=0; d < ngpu; d++ ) {
magma_setdevice(d);
j_local2 = j_local;
if ( d < id ) j_local2 ++;
if ( d == id || n_local[d] > j_local2*nb ) {
magma_zsetmatrix_async( rows, nb0,
W(s), ldw,
&d_lAP[d][(s%h)*nb*maxm],
cols, queues[d][1] );
}
}
for( d=0; d < ngpu; d++ ) {
magma_setdevice(d);
if ( d == 0 ) {
for( i=s*nb; i < s*nb + nb0; ++i ) {
ipiv[i] += s*nb;
}
}
magmablas_zlaswp_q( lddat, dAT(d,0,0), lddat, s*nb + 1, s*nb + nb0, ipiv, 1, queues[d][0] );
}
for( d=0; d < ngpu; d++ ) {
magma_setdevice(d);
magmablasSetKernelStream(queues[d][1]);
/* wait for the pivoting to be done */
magma_queue_sync( queues[d][0] );
j_local2 = j_local;
if ( d < id ) j_local2++;
if ( d == id ) {
/* the panel belond to this gpu */
panel_local[d] = dAT(d,s,j_local);
/* next column */
nb1 = n_local[d] - j_local*nb-nb0;
magmablas_ztranspose( rows, nb0, &d_lAP[d][(s%h)*nb*maxm], cols, panel_local[d], lddat );
if ( nb1 > 0 ) {
magma_ztrsm( MagmaRight, MagmaUpper, MagmaNoTrans, MagmaUnit,
nb1, nb0, c_one,
panel_local[d], lddat,
dAT(d,s,j_local)+nb0, lddat);
}
} else if ( n_local[d] > j_local2*nb ) {
/* the panel belong to another gpu */
panel_local[d] = d_panel[d];
/* next column */
nb1 = n_local[d] - j_local2*nb;
magmablas_ztranspose( rows, nb0, &d_lAP[d][(s%h)*nb*maxm], cols, panel_local[d], nb );
magma_ztrsm( MagmaRight, MagmaUpper, MagmaNoTrans, MagmaUnit,
nb1, nb0, c_one,
panel_local[d], nb,
dAT(d,s,j_local2), lddat);
}
}
} /* if ( nb0 > 0 ) */
/* clean up */
trace_finalize( "zgetrf_mgpu.svg","trace.css" );
for( d=0; d < ngpu; d++ ) {
magma_setdevice(d);
magma_queue_sync( queues[d][0] );
magma_queue_sync( queues[d][1] );
}
magma_setdevice( orig_dev );
magmablasSetKernelStream( orig_stream );
timer_start( time );
timer_printf("\n Performance %f GFlop/s\n", FLOPS_ZGETRF(m,n) / 1e9 / time );
return *info;
} /* magma_zgetrf2_mgpu */
