void bli_l3_thrinfo_create_root
(
dim_t id,
thrcomm_t* gl_comm,
cntx_t* cntx,
cntl_t* cntl,
thrinfo_t** thread
)
{
// Query the global communicator for the total number of threads to use.
dim_t n_threads = bli_thrcomm_num_threads( gl_comm );
// Use the thread id passed in as the global communicator id.
dim_t gl_comm_id = id;
// Use the blocksize id of the current (root) control tree node to
// query the top-most ways of parallelism to obtain.
bszid_t bszid = bli_cntl_bszid( cntl );
dim_t xx_way = bli_cntx_way_for_bszid( bszid, cntx );
// Determine the work id for this thrinfo_t node.
dim_t work_id = gl_comm_id / ( n_threads / xx_way );
// Create the root thrinfo_t node.
*thread = bli_thrinfo_create
(
gl_comm,
gl_comm_id,
xx_way,
work_id,
TRUE,
NULL
);
}
void bli_gemm_blk_var2
(
obj_t* a,
obj_t* b,
obj_t* c,
cntx_t* cntx,
rntm_t* rntm,
cntl_t* cntl,
thrinfo_t* thread
)
{
obj_t b1, c1;
dir_t direct;
dim_t i;
dim_t b_alg;
dim_t my_start, my_end;
// Determine the direction in which to partition (forwards or backwards).
direct = bli_l3_direct( a, b, c, cntl );
// Prune any zero region that exists along the partitioning dimension.
bli_l3_prune_unref_mparts_n( a, b, c, cntl );
// Determine the current thread's subpartition range.
bli_thread_range_ndim
(
direct, thread, a, b, c, cntl, cntx,
&my_start, &my_end
);
// Partition along the n dimension.
for ( i = my_start; i < my_end; i += b_alg )
{
// Determine the current algorithmic blocksize.
b_alg = bli_determine_blocksize( direct, i, my_end, b,
bli_cntl_bszid( cntl ), cntx );
// Acquire partitions for B1 and C1.
bli_acquire_mpart_ndim( direct, BLIS_SUBPART1,
i, b_alg, b, &b1 );
bli_acquire_mpart_ndim( direct, BLIS_SUBPART1,
i, b_alg, c, &c1 );
// Perform gemm subproblem.
bli_gemm_int
(
&BLIS_ONE,
a,
&b1,
&BLIS_ONE,
&c1,
cntx,
rntm,
bli_cntl_sub_node( cntl ),
bli_thrinfo_sub_node( thread )
);
}
}
siz_t bli_thread_get_range_ndim
(
dir_t direct,
thrinfo_t* thr,
obj_t* a,
obj_t* b,
obj_t* c,
cntl_t* cntl,
cntx_t* cntx,
dim_t* start,
dim_t* end
)
{
bszid_t bszid = bli_cntl_bszid( cntl );
opid_t family = bli_cntx_get_family( cntx );
// This is part of trsm's current implementation, whereby right side
// cases are implemented in left-side micro-kernels, which requires
// we swap the usage of the register blocksizes for the purposes of
// packing A and B.
if ( family == BLIS_TRSM )
{
if ( bli_obj_root_is_triangular( *b ) ) bszid = BLIS_MR;
else bszid = BLIS_NR;
}
blksz_t* bmult = bli_cntx_get_bmult( bszid, cntx );
obj_t* x;
bool_t use_weighted;
// Use the operation family to choose the one of the two matrices
// being partitioned that potentially has structure, and also to
// decide whether or not we need to use weighted range partitioning.
// NOTE: It's important that we use non-weighted range partitioning
// for hemm and symm (ie: the gemm family) because the weighted
// function will mistakenly skip over unstored regions of the
// structured matrix, even though they represent part of that matrix
// that will be dense and full (after packing).
if ( family == BLIS_GEMM ) { x = b; use_weighted = FALSE; }
else if ( family == BLIS_HERK ) { x = c; use_weighted = TRUE; }
else if ( family == BLIS_TRMM ) { x = b; use_weighted = TRUE; }
else /*family == BLIS_TRSM*/ { x = b; use_weighted = FALSE; }
if ( use_weighted )
{
if ( direct == BLIS_FWD )
return bli_thread_get_range_weighted_l2r( thr, x, bmult, start, end );
else
return bli_thread_get_range_weighted_r2l( thr, x, bmult, start, end );
}
else
{
if ( direct == BLIS_FWD )
return bli_thread_get_range_l2r( thr, x, bmult, start, end );
else
return bli_thread_get_range_r2l( thr, x, bmult, start, end );
}
}
void blx_gemm_blk_var3
(
obj_t* a,
obj_t* b,
obj_t* c,
cntx_t* cntx,
rntm_t* rntm,
cntl_t* cntl,
thrinfo_t* thread
)
{
obj_t a1, b1;
dim_t i;
dim_t b_alg;
dim_t k_trans;
// Query dimension in partitioning direction.
k_trans = bli_obj_width_after_trans( a );
// Partition along the k dimension.
for ( i = 0; i < k_trans; i += b_alg )
{
// Determine the current algorithmic blocksize.
b_alg = blx_determine_blocksize_f( i, k_trans, c,
bli_cntl_bszid( cntl ), cntx );
// Acquire partitions for A1 and B1.
bli_acquire_mpart_ndim( BLIS_FWD, BLIS_SUBPART1, i, b_alg, a, &a1 );
bli_acquire_mpart_mdim( BLIS_FWD, BLIS_SUBPART1, i, b_alg, b, &b1 );
// Perform gemm subproblem.
blx_gemm_int
(
&a1, &b1, c, cntx, rntm,
bli_cntl_sub_node( cntl ),
bli_thrinfo_sub_node( thread )
);
bli_thread_obarrier( bli_thrinfo_sub_node( thread ) );
// This variant executes multiple rank-k updates. Therefore, if the
// internal beta scalar on matrix C is non-zero, we must use it
// only for the first iteration (and then BLIS_ONE for all others).
// And since c is a locally aliased obj_t, we can simply overwrite
// the internal beta scalar with BLIS_ONE once it has been used in
// the first iteration.
if ( i == 0 ) bli_obj_scalar_reset( c );
}
}
void bli_ger_blk_var2( obj_t* alpha,
obj_t* x,
obj_t* y,
obj_t* a,
cntx_t* cntx,
ger_t* cntl )
{
obj_t a1, a1_pack;
obj_t y1, y1_pack;
dim_t i;
dim_t b_alg;
dim_t n_trans;
// Initialize objects for packing.
bli_obj_init_pack( &a1_pack );
bli_obj_init_pack( &y1_pack );
// Query dimension in partitioning direction.
n_trans = bli_obj_width_after_trans( *a );
// Partition along the n dimension.
for ( i = 0; i < n_trans; i += b_alg )
{
// Determine the current algorithmic blocksize.
b_alg = bli_determine_blocksize_f( i, n_trans, a,
bli_cntl_bszid( cntl ), cntx );
// Acquire partitions for A1 and y1.
bli_acquire_mpart_l2r( BLIS_SUBPART1,
i, b_alg, a, &a1 );
bli_acquire_vpart_f2b( BLIS_SUBPART1,
i, b_alg, y, &y1 );
// Initialize objects for packing A1 and y1 (if needed).
bli_packm_init( &a1, &a1_pack,
cntx, bli_cntl_sub_packm_a( cntl ) );
bli_packv_init( &y1, &y1_pack,
cntx, bli_cntl_sub_packv_y( cntl ) );
// Copy/pack A1, y1 (if needed).
bli_packm_int( &a1, &a1_pack,
cntx, bli_cntl_sub_packm_a( cntl ),
&BLIS_PACKM_SINGLE_THREADED );
bli_packv_int( &y1, &y1_pack,
cntx, bli_cntl_sub_packv_y( cntl ) );
// A1 = A1 + alpha * x * y1;
bli_ger_int( BLIS_NO_CONJUGATE,
BLIS_NO_CONJUGATE,
alpha,
x,
&y1_pack,
&a1_pack,
cntx,
bli_cntl_sub_ger( cntl ) );
// Copy/unpack A1 (if A1 was packed).
bli_unpackm_int( &a1_pack, &a1,
cntx, bli_cntl_sub_unpackm_a( cntl ),
&BLIS_PACKM_SINGLE_THREADED );
}
// If any packing buffers were acquired within packm, release them back
// to the memory manager.
bli_packm_release( &a1_pack, bli_cntl_sub_packm_a( cntl ) );
bli_packv_release( &y1_pack, bli_cntl_sub_packv_y( cntl ) );
}
void bli_gemv_blk_var2( obj_t* alpha,
obj_t* a,
obj_t* x,
obj_t* beta,
obj_t* y,
cntx_t* cntx,
gemv_t* cntl )
{
obj_t a1, a1_pack;
obj_t x1, x1_pack;
dim_t n_trans;
dim_t i;
dim_t b_alg;
// Initialize objects for packing.
bli_obj_init_pack( &a1_pack );
bli_obj_init_pack( &x1_pack );
// Query dimension in partitioning direction.
n_trans = bli_obj_width_after_trans( a );
// y = beta * y;
bli_scalv_int( beta,
y,
cntx, bli_cntl_sub_scalv( cntl ) );
// Partition along the "k" dimension (n dimension of A).
for ( i = 0; i < n_trans; i += b_alg )
{
// Determine the current algorithmic blocksize.
b_alg = bli_determine_blocksize_f( i, n_trans, a,
bli_cntl_bszid( cntl ), cntx );
// Acquire partitions for A1 and x1.
bli_acquire_mpart_l2r( BLIS_SUBPART1,
i, b_alg, a, &a1 );
bli_acquire_vpart_f2b( BLIS_SUBPART1,
i, b_alg, x, &x1 );
// Initialize objects for packing A1 and x1 (if needed).
bli_packm_init( &a1, &a1_pack,
cntx, bli_cntl_sub_packm_a( cntl ) );
bli_packv_init( &x1, &x1_pack,
cntx, bli_cntl_sub_packv_x( cntl ) );
// Copy/pack A1, x1 (if needed).
bli_packm_int( &a1, &a1_pack,
cntx, bli_cntl_sub_packm_a( cntl ),
&BLIS_PACKM_SINGLE_THREADED );
bli_packv_int( &x1, &x1_pack,
cntx, bli_cntl_sub_packv_x( cntl ) );
// y = y + alpha * A1 * x1;
bli_gemv_int( BLIS_NO_TRANSPOSE,
BLIS_NO_CONJUGATE,
alpha,
&a1_pack,
&x1_pack,
&BLIS_ONE,
y,
cntx,
bli_cntl_sub_gemv( cntl ) );
}
// If any packing buffers were acquired within packm, release them back
// to the memory manager.
bli_packm_release( &a1_pack, bli_cntl_sub_packm_a( cntl ) );
bli_packv_release( &x1_pack, bli_cntl_sub_packv_x( cntl ) );
}
void bli_hemv_blk_var1( conj_t conjh,
obj_t* alpha,
obj_t* a,
obj_t* x,
obj_t* beta,
obj_t* y,
cntx_t* cntx,
hemv_t* cntl )
{
obj_t a11, a11_pack;
obj_t a10;
obj_t x1, x1_pack;
obj_t x0;
obj_t y1, y1_pack;
obj_t y0;
dim_t mn;
dim_t ij;
dim_t b_alg;
// Even though this blocked algorithm is expressed only in terms of the
// lower triangular case, the upper triangular case is still supported:
// when bli_acquire_mpart_tl2br() is passed a matrix that is stored in
// in the upper triangle, and the requested subpartition resides in the
// lower triangle (as is the case for this algorithm), the routine fills
// the request as if the caller had actually requested the corresponding
// "mirror" subpartition in the upper triangle, except that it marks the
// subpartition for transposition (and conjugation).
// Initialize objects for packing.
bli_obj_init_pack( &a11_pack );
bli_obj_init_pack( &x1_pack );
bli_obj_init_pack( &y1_pack );
// Query dimension.
mn = bli_obj_length( a );
// y = beta * y;
bli_scalv_int( beta,
y,
cntx, bli_cntl_sub_scalv( cntl ) );
// Partition diagonally.
for ( ij = 0; ij < mn; ij += b_alg )
{
// Determine the current algorithmic blocksize.
b_alg = bli_determine_blocksize_f( ij, mn, a,
bli_cntl_bszid( cntl ), cntx );
// Acquire partitions for A11, A10, x1, x0, y1, and y0.
bli_acquire_mpart_tl2br( BLIS_SUBPART11,
ij, b_alg, a, &a11 );
bli_acquire_mpart_tl2br( BLIS_SUBPART10,
ij, b_alg, a, &a10 );
bli_acquire_vpart_f2b( BLIS_SUBPART1,
ij, b_alg, x, &x1 );
bli_acquire_vpart_f2b( BLIS_SUBPART0,
ij, b_alg, x, &x0 );
bli_acquire_vpart_f2b( BLIS_SUBPART1,
ij, b_alg, y, &y1 );
bli_acquire_vpart_f2b( BLIS_SUBPART0,
ij, b_alg, y, &y0 );
// Initialize objects for packing A11, x1, and y1 (if needed).
bli_packm_init( &a11, &a11_pack,
cntx, bli_cntl_sub_packm_a11( cntl ) );
bli_packv_init( &x1, &x1_pack,
cntx, bli_cntl_sub_packv_x1( cntl ) );
bli_packv_init( &y1, &y1_pack,
cntx, bli_cntl_sub_packv_y1( cntl ) );
// Copy/pack A11, x1, y1 (if needed).
bli_packm_int( &a11, &a11_pack,
cntx, bli_cntl_sub_packm_a11( cntl ),
&BLIS_PACKM_SINGLE_THREADED );
bli_packv_int( &x1, &x1_pack,
cntx, bli_cntl_sub_packv_x1( cntl ) );
bli_packv_int( &y1, &y1_pack,
cntx, bli_cntl_sub_packv_y1( cntl ) );
// y0 = y0 + alpha * A10' * x1;
bli_gemv_int( bli_apply_conj( conjh, BLIS_TRANSPOSE ),
BLIS_NO_CONJUGATE,
alpha,
&a10,
&x1_pack,
&BLIS_ONE,
&y0,
cntx,
bli_cntl_sub_gemv_t_rp( cntl ) );
// y1 = y1 + alpha * A11 * x1;
bli_hemv_int( conjh,
alpha,
&a11_pack,
&x1_pack,
&BLIS_ONE,
&y1_pack,
cntx,
bli_cntl_sub_hemv( cntl ) );
// y1 = y1 + alpha * A10 * x0;
bli_gemv_int( BLIS_NO_TRANSPOSE,
BLIS_NO_CONJUGATE,
alpha,
&a10,
&x0,
&BLIS_ONE,
&y1_pack,
cntx,
bli_cntl_sub_gemv_n_rp( cntl ) );
// Copy/unpack y1 (if y1 was packed).
bli_unpackv_int( &y1_pack, &y1,
cntx, bli_cntl_sub_unpackv_y1( cntl ) );
}
// If any packing buffers were acquired within packm, release them back
// to the memory manager.
bli_packm_release( &a11_pack, bli_cntl_sub_packm_a11( cntl ) );
bli_packv_release( &x1_pack, bli_cntl_sub_packv_x1( cntl ) );
bli_packv_release( &y1_pack, bli_cntl_sub_packv_y1( cntl ) );
}
void bli_trsv_u_blk_var1( obj_t* alpha,
obj_t* a,
obj_t* x,
cntx_t* cntx,
trsv_t* cntl )
{
obj_t a11, a11_pack;
obj_t a12;
obj_t x1, x1_pack;
obj_t x2;
dim_t mn;
dim_t ij;
dim_t b_alg;
// Initialize objects for packing.
bli_obj_init_pack( &a11_pack );
bli_obj_init_pack( &x1_pack );
// Query dimension.
mn = bli_obj_length( a );
// x = alpha * x;
bli_scalv_int( alpha,
x,
cntx, bli_cntl_sub_scalv( cntl ) );
// Partition diagonally.
for ( ij = 0; ij < mn; ij += b_alg )
{
// Determine the current algorithmic blocksize.
b_alg = bli_determine_blocksize_b( ij, mn, a,
bli_cntl_bszid( cntl ), cntx );
// Acquire partitions for A11, A12, x1, and x2.
bli_acquire_mpart_br2tl( BLIS_SUBPART11,
ij, b_alg, a, &a11 );
bli_acquire_mpart_br2tl( BLIS_SUBPART12,
ij, b_alg, a, &a12 );
bli_acquire_vpart_b2f( BLIS_SUBPART1,
ij, b_alg, x, &x1 );
bli_acquire_vpart_b2f( BLIS_SUBPART2,
ij, b_alg, x, &x2 );
// Initialize objects for packing A11 and x1 (if needed).
bli_packm_init( &a11, &a11_pack,
cntx, bli_cntl_sub_packm_a11( cntl ) );
bli_packv_init( &x1, &x1_pack,
cntx, bli_cntl_sub_packv_x1( cntl ) );
// Copy/pack A11, x1 (if needed).
bli_packm_int( &a11, &a11_pack,
cntx, bli_cntl_sub_packm_a11( cntl ),
&BLIS_PACKM_SINGLE_THREADED );
bli_packv_int( &x1, &x1_pack,
cntx, bli_cntl_sub_packv_x1( cntl ) );
// x1 = x1 - A12 * x2;
bli_gemv_int( BLIS_NO_TRANSPOSE,
BLIS_NO_CONJUGATE,
&BLIS_MINUS_ONE,
&a12,
&x2,
&BLIS_ONE,
&x1_pack,
cntx,
bli_cntl_sub_gemv_rp( cntl ) );
// x1 = x1 / tril( A11 );
bli_trsv_int( &BLIS_ONE,
&a11_pack,
&x1_pack,
cntx,
bli_cntl_sub_trsv( cntl ) );
// Copy/unpack x1 (if x1 was packed).
bli_unpackv_int( &x1_pack, &x1,
cntx, bli_cntl_sub_unpackv_x1( cntl ) );
}
// If any packing buffers were acquired within packm, release them back
// to the memory manager.
bli_packm_release( &a11_pack, bli_cntl_sub_packm_a11( cntl ) );
bli_packv_release( &x1_pack, bli_cntl_sub_packv_x1( cntl ) );
}
void bli_trsm_blk_var3
(
obj_t* a,
obj_t* b,
obj_t* c,
cntx_t* cntx,
cntl_t* cntl,
thrinfo_t* thread
)
{
obj_t a1, b1;
dir_t direct;
dim_t i;
dim_t b_alg;
dim_t k_trans;
// Determine the direction in which to partition (forwards or backwards).
direct = bli_l3_direct( a, b, c, cntl );
// Prune any zero region that exists along the partitioning dimension.
bli_l3_prune_unref_mparts_k( a, b, c, cntl );
// Query dimension in partitioning direction.
k_trans = bli_obj_width_after_trans( *a );
// Partition along the k dimension.
for ( i = 0; i < k_trans; i += b_alg )
{
// Determine the current algorithmic blocksize.
b_alg = bli_trsm_determine_kc( direct, i, k_trans, a, b,
bli_cntl_bszid( cntl ), cntx );
// Acquire partitions for A1 and B1.
bli_acquire_mpart_ndim( direct, BLIS_SUBPART1,
i, b_alg, a, &a1 );
bli_acquire_mpart_mdim( direct, BLIS_SUBPART1,
i, b_alg, b, &b1 );
// Perform trsm subproblem.
bli_trsm_int
(
&BLIS_ONE,
&a1,
&b1,
&BLIS_ONE,
c,
cntx,
bli_cntl_sub_node( cntl ),
bli_thrinfo_sub_node( thread )
);
//bli_thread_ibarrier( thread );
bli_thread_obarrier( bli_thrinfo_sub_node( thread ) );
// This variant executes multiple rank-k updates. Therefore, if the
// internal alpha scalars on A/B and C are non-zero, we must ensure
// that they are only used in the first iteration.
if ( i == 0 )
{
bli_obj_scalar_reset( a ); bli_obj_scalar_reset( b );
bli_obj_scalar_reset( c );
}
}
}
void bli_trmm_blk_var2b( obj_t* a,
obj_t* b,
obj_t* c,
cntx_t* cntx,
gemm_t* cntl,
thrinfo_t* thread )
{
obj_t a_pack_s;
obj_t b1_pack_s, c1_pack_s;
obj_t b1, c1;
obj_t* a_pack = NULL;
obj_t* b1_pack = NULL;
obj_t* c1_pack = NULL;
dim_t i;
dim_t b_alg;
// Prune any zero region that exists along the partitioning dimension.
bli_trmm_prune_unref_mparts_n( a, b, c );
if( bli_thread_am_ochief( thread ) ) {
// Initialize object for packing A
bli_obj_init_pack( &a_pack_s );
bli_packm_init( a, &a_pack_s,
cntx, bli_cntl_sub_packm_a( cntl ) );
// Scale C by beta (if instructed).
bli_scalm_int( &BLIS_ONE,
c,
cntx, bli_cntl_sub_scalm( cntl ) );
}
a_pack = bli_thread_obroadcast( thread, &a_pack_s );
// Initialize pack objects for B and C that are passed into packm_init().
if( bli_thread_am_ichief( thread ) ) {
bli_obj_init_pack( &b1_pack_s );
bli_obj_init_pack( &c1_pack_s );
}
b1_pack = bli_thread_ibroadcast( thread, &b1_pack_s );
c1_pack = bli_thread_ibroadcast( thread, &c1_pack_s );
// Pack A (if instructed).
bli_packm_int( a, a_pack,
cntx, bli_cntl_sub_packm_a( cntl ),
bli_thrinfo_sub_opackm( thread ) );
dim_t my_start, my_end;
bli_thread_get_range_weighted_r2l( thread, b,
bli_cntx_get_bmult( bli_cntl_bszid( cntl ), cntx ),
&my_start, &my_end );
// Partition along the n dimension.
for ( i = my_start; i < my_end; i += b_alg )
{
// Determine the current algorithmic blocksize.
b_alg = bli_determine_blocksize_b( i, my_end, b,
bli_cntl_bszid( cntl ), cntx );
// Acquire partitions for B1 and C1.
bli_acquire_mpart_r2l( BLIS_SUBPART1,
i, b_alg, b, &b1 );
bli_acquire_mpart_r2l( BLIS_SUBPART1,
i, b_alg, c, &c1 );
// Initialize objects for packing A1 and B1.
if( bli_thread_am_ichief( thread ) ) {
bli_packm_init( &b1, b1_pack,
cntx, bli_cntl_sub_packm_b( cntl ) );
bli_packm_init( &c1, c1_pack,
cntx, bli_cntl_sub_packm_c( cntl ) );
}
bli_thread_ibarrier( thread );
// Pack B1 (if instructed).
bli_packm_int( &b1, b1_pack,
cntx, bli_cntl_sub_packm_b( cntl ),
bli_thrinfo_sub_ipackm( thread ) );
// Pack C1 (if instructed).
bli_packm_int( &c1, c1_pack,
cntx, bli_cntl_sub_packm_c( cntl ),
bli_thrinfo_sub_ipackm( thread ) );
// Perform trmm subproblem.
bli_trmm_int( &BLIS_ONE,
a_pack,
b1_pack,
&BLIS_ONE,
c1_pack,
cntx,
bli_cntl_sub_gemm( cntl ),
bli_thrinfo_sub_self( thread ) );
bli_thread_ibarrier( thread );
// Unpack C1 (if C1 was packed).
bli_unpackm_int( c1_pack, &c1,
cntx, bli_cntl_sub_unpackm_c( cntl ),
bli_thrinfo_sub_ipackm( thread ) );
}
// If any packing buffers were acquired within packm, release them back
// to the memory manager.
bli_thread_obarrier( thread );
if( bli_thread_am_ochief( thread ) )
bli_packm_release( a_pack, bli_cntl_sub_packm_a( cntl ) );
if( bli_thread_am_ichief( thread ) ) {
bli_packm_release( b1_pack, bli_cntl_sub_packm_b( cntl ) );
bli_packm_release( c1_pack, bli_cntl_sub_packm_c( cntl ) );
}
}
