void bli_trsm_blk_var3b( obj_t* a,
obj_t* b,
obj_t* c,
trsm_t* cntl,
trsm_thrinfo_t* thread )
{
obj_t c_pack_s;
obj_t a1_pack_s, b1_pack_s;
obj_t a1, b1;
obj_t* a1_pack = NULL;
obj_t* b1_pack = NULL;
obj_t* c_pack = NULL;
dim_t i;
dim_t b_alg;
dim_t k_trans;
// Prune any zero region that exists along the partitioning dimension.
bli_trsm_prune_unref_mparts_k( a, b, c );
// Initialize pack objects for C that are passed into packm_init().
if( thread_am_ochief( thread ) ) {
bli_obj_init_pack( &c_pack_s );
// Initialize object for packing C.
bli_packm_init( c, &c_pack_s,
cntl_sub_packm_c( cntl ) );
// Scale C by beta (if instructed).
bli_scalm_int( &BLIS_ONE,
c,
cntl_sub_scalm( cntl ) );
}
c_pack = thread_obroadcast( thread, &c_pack_s );
if( thread_am_ichief( thread ) ) {
bli_obj_init_pack( &a1_pack_s );
bli_obj_init_pack( &b1_pack_s );
}
a1_pack = thread_ibroadcast( thread, &a1_pack_s );
b1_pack = thread_ibroadcast( thread, &b1_pack_s );
// Pack C (if instructed).
bli_packm_int( c, c_pack,
cntl_sub_packm_c( cntl ),
trsm_thread_sub_opackm( thread ) );
// Query dimension in partitioning direction.
k_trans = bli_obj_width_after_trans( *a );
// Partition along the k dimension.
for ( i = 0; i < k_trans; i += b_alg )
{
// Determine the current algorithmic blocksize.
// NOTE: We call a trsm-specific function to determine the kc
// blocksize so that we can implement the "nudging" of kc to be
// a multiple of mr, as needed.
b_alg = bli_trsm_determine_kc_b( i, k_trans, b,
cntl_blocksize( cntl ) );
// Acquire partitions for A1 and B1.
bli_acquire_mpart_r2l( BLIS_SUBPART1,
i, b_alg, a, &a1 );
bli_acquire_mpart_b2t( BLIS_SUBPART1,
i, b_alg, b, &b1 );
// Initialize objects for packing A1 and B1.
if( thread_am_ichief( thread ) ) {
bli_packm_init( &a1, a1_pack,
cntl_sub_packm_a( cntl ) );
bli_packm_init( &b1, b1_pack,
cntl_sub_packm_b( cntl ) );
}
thread_ibarrier( thread );
// Pack A1 (if instructed).
bli_packm_int( &a1, a1_pack,
cntl_sub_packm_a( cntl ),
trsm_thread_sub_ipackm( thread ) );
// Pack B1 (if instructed).
bli_packm_int( &b1, b1_pack,
cntl_sub_packm_b( cntl ),
trsm_thread_sub_ipackm( thread ) );
// Perform trsm subproblem.
bli_trsm_int( &BLIS_ONE,
a1_pack,
b1_pack,
&BLIS_ONE,
c_pack,
cntl_sub_trsm( cntl ),
trsm_thread_sub_trsm( thread ) );
// This variant executes multiple rank-k updates. Therefore, if the
// internal alpha scalars on A/B and C are non-zero, we must ensure
// that they are only used in the first iteration.
thread_ibarrier( thread );
if ( i == 0 && thread_am_ichief( thread ) ) {
bli_obj_scalar_reset( a );
bli_obj_scalar_reset( b );
bli_obj_scalar_reset( c_pack );
}
}
thread_obarrier( thread );
// Unpack C (if C was packed).
bli_unpackm_int( c_pack, c,
cntl_sub_unpackm_c( cntl ),
trsm_thread_sub_opackm( thread ) );
// If any packing buffers were acquired within packm, release them back
// to the memory manager.
if( thread_am_ochief( thread ) ) {
bli_packm_release( c_pack, cntl_sub_packm_c( cntl ) );
}
if( thread_am_ichief( thread ) ) {
bli_packm_release( a1_pack, cntl_sub_packm_a( cntl ) );
bli_packm_release( b1_pack, cntl_sub_packm_b( cntl ) );
}
}
void bli_trsm_blk_var1b( obj_t* a,
obj_t* b,
obj_t* c,
trsm_t* cntl,
trsm_thrinfo_t* thread )
{
obj_t b_pack_s;
obj_t a1_pack_s;
obj_t a1, c1;
obj_t* b_pack = NULL;
obj_t* a1_pack = NULL;
dim_t i;
dim_t b_alg;
// Prune any zero region that exists along the partitioning dimension.
bli_trsm_prune_unref_mparts_m( a, b, c );
// Initialize object for packing B.
if( thread_am_ochief( thread ) ) {
bli_obj_init_pack( &b_pack_s );
bli_packm_init( b, &b_pack_s,
cntl_sub_packm_b( cntl ) );
}
b_pack = thread_obroadcast( thread, &b_pack_s );
// Initialize object for packing B.
if( thread_am_ichief( thread ) ) {
bli_obj_init_pack( &a1_pack_s );
}
a1_pack = thread_ibroadcast( thread, &a1_pack_s );
// Pack B1 (if instructed).
bli_packm_int( b, b_pack,
cntl_sub_packm_b( cntl ),
trsm_thread_sub_opackm( thread ) );
dim_t my_start, my_end;
num_t dt = bli_obj_execution_datatype( *a );
dim_t bf = ( bli_obj_root_is_triangular( *a ) ?
bli_info_get_default_mr( BLIS_TRSM, dt ) :
bli_info_get_default_nr( BLIS_TRSM, dt ) );
bli_get_range_b2t( thread, a, bf,
&my_start, &my_end );
// Partition along the remaining portion of the m dimension.
for ( i = my_start; i < my_end; i += b_alg )
{
// Determine the current algorithmic blocksize.
b_alg = bli_determine_blocksize_b( i, my_end, a,
cntl_blocksize( cntl ) );
// Acquire partitions for A1 and C1.
bli_acquire_mpart_b2t( BLIS_SUBPART1,
i, b_alg, a, &a1 );
bli_acquire_mpart_b2t( BLIS_SUBPART1,
i, b_alg, c, &c1 );
// Initialize object for packing A1.
if( thread_am_ichief( thread ) ) {
bli_packm_init( &a1, a1_pack,
cntl_sub_packm_a( cntl ) );
}
thread_ibarrier( thread );
// Pack A1 (if instructed).
bli_packm_int( &a1, a1_pack,
cntl_sub_packm_a( cntl ),
trsm_thread_sub_ipackm( thread ) );
// Perform trsm subproblem.
bli_trsm_int( &BLIS_ONE,
a1_pack,
b_pack,
&BLIS_ONE,
&c1,
cntl_sub_trsm( cntl ),
trsm_thread_sub_trsm( thread ) );
thread_ibarrier( thread );
}
// If any packing buffers were acquired within packm, release them back
// to the memory manager.
thread_obarrier( thread );
if( thread_am_ochief( thread ) )
bli_packm_release( b_pack, cntl_sub_packm_b( cntl ) );
if( thread_am_ichief( thread ) )
bli_packm_release( a1_pack, cntl_sub_packm_a( cntl ) );
}
void bli_trsm_u_blk_var4( obj_t* alpha,
obj_t* a,
obj_t* b,
obj_t* beta,
obj_t* c,
trsm_t* cntl )
{
obj_t a1, a1_pack;
obj_t b_pack;
obj_t c1;
dim_t i;
dim_t bm_alg;
dim_t m_trans;
// Initialize all pack objects that are passed into packm_init().
bli_obj_init_pack( &a1_pack );
bli_obj_init_pack( &b_pack );
// Query dimension in partitioning direction.
m_trans = bli_obj_length_after_trans( *a );
// Initialize object for packing B.
bli_packm_init( b, &b_pack,
cntl_sub_packm_b( cntl ) );
// Find the offset to the first non-zero block of A.
for ( i = 0; i < m_trans; i += bm_alg )
{
// Determine the current algorithmic blocksize.
bm_alg = bli_determine_blocksize_b( i, m_trans, a,
cntl_blocksize( cntl ) );
// Acquire partitions for A1 and C1.
bli_acquire_mpart_b2t( BLIS_SUBPART1,
i, bm_alg, a, &a1 );
if ( !bli_obj_is_zeros( a1 ) ) break;
}
// Fuse the first iteration with incremental packing and computation.
{
obj_t b_inc, b_pack_inc;
obj_t c1_inc;
dim_t j;
dim_t bn_inc;
dim_t n_trans;
// Query dimension in partitioning direction.
n_trans = bli_obj_width( b_pack );
// Determine the current algorithmic blocksize.
bm_alg = bli_determine_blocksize_b( i, m_trans, a,
cntl_blocksize( cntl ) );
// Acquire partitions for A1 and C1.
bli_acquire_mpart_b2t( BLIS_SUBPART1,
i, bm_alg, a, &a1 );
bli_acquire_mpart_b2t( BLIS_SUBPART1,
i, bm_alg, c, &c1 );
// Initialize objects for packing A1 and C1.
bli_packm_init( &a1, &a1_pack, cntl_sub_packm_a( cntl ) );
// Pack A1 and scale by alpha (if instructed).
bli_packm_int( alpha, &a1, &a1_pack, cntl_sub_packm_a( cntl ) );
// Partition along the n dimension.
for ( j = 0; j < n_trans; j += bn_inc )
{
// Determine the current incremental packing blocksize.
bn_inc = bli_determine_blocksize_f( j, n_trans, b,
cntl_blocksize_aux( cntl ) );
// Acquire partitions.
bli_acquire_mpart_l2r( BLIS_SUBPART1,
j, bn_inc, b, &b_inc );
bli_acquire_mpart_l2r( BLIS_SUBPART1,
j, bn_inc, &b_pack, &b_pack_inc );
bli_acquire_mpart_l2r( BLIS_SUBPART1,
j, bn_inc, &c1, &c1_inc );
// Pack B1 and scale by alpha (if instructed).
bli_packm_int( alpha, &b_inc, &b_pack_inc, cntl_sub_packm_b( cntl ) );
// Perform trsm subproblem.
bli_trsm_int( BLIS_LEFT,
alpha,
&a1_pack,
&b_pack_inc,
beta,
&c1_inc,
cntl_sub_trsm( cntl ) );
}
}
// Partition along the remaining portion of the m dimension.
for ( i = i + bm_alg; i < m_trans; i += bm_alg )
{
// Determine the current algorithmic blocksize.
bm_alg = bli_determine_blocksize_b( i, m_trans, a,
cntl_blocksize( cntl ) );
// Acquire partitions for A1 and C1.
bli_acquire_mpart_b2t( BLIS_SUBPART1,
i, bm_alg, a, &a1 );
bli_acquire_mpart_b2t( BLIS_SUBPART1,
i, bm_alg, c, &c1 );
// Initialize object for packing A1.
bli_packm_init( &a1, &a1_pack,
cntl_sub_packm_a( cntl ) );
// Pack A1 and scale by alpha (if instructed).
bli_packm_int( alpha,
&a1, &a1_pack,
cntl_sub_packm_a( cntl ) );
if ( bli_obj_intersects_diag( a1_pack ) )
bli_trsm_int( BLIS_LEFT,
alpha,
&a1_pack,
&b_pack,
beta,
&c1,
cntl_sub_trsm( cntl ) );
else
bli_gemm_int( &BLIS_MINUS_ONE,
&a1_pack,
&b_pack,
&BLIS_ONE,
&c1,
cntl_sub_gemm( cntl ) );
}
// If any packing buffers were acquired within packm, release them back
// to the memory manager.
bli_obj_release_pack( &a1_pack );
bli_obj_release_pack( &b_pack );
}
void bli_trmm_blk_var3b( obj_t* a,
obj_t* b,
obj_t* c,
gemm_t* cntl,
trmm_thrinfo_t* thread )
{
obj_t c_pack_s;
obj_t a1_pack_s, b1_pack_s;
obj_t a1, b1;
obj_t* a1_pack = NULL;
obj_t* b1_pack = NULL;
obj_t* c_pack = NULL;
dim_t i;
dim_t b_alg;
dim_t k_trans;
if( thread_am_ochief( thread ) ){
// Initialize object for packing C
bli_obj_init_pack( &c_pack_s );
bli_packm_init( c, &c_pack_s,
cntl_sub_packm_c( cntl ) );
// Scale C by beta (if instructed).
bli_scalm_int( &BLIS_ONE,
c,
cntl_sub_scalm( cntl ) );
}
c_pack = thread_obroadcast( thread, &c_pack_s );
// Initialize pack objects for A and B that are passed into packm_init().
if( thread_am_ichief( thread ) ){
bli_obj_init_pack( &a1_pack_s );
bli_obj_init_pack( &b1_pack_s );
}
a1_pack = thread_ibroadcast( thread, &a1_pack_s );
b1_pack = thread_ibroadcast( thread, &b1_pack_s );
// Pack C (if instructed).
bli_packm_int( c, c_pack,
cntl_sub_packm_c( cntl ),
trmm_thread_sub_opackm( thread ) );
// Query dimension in partitioning direction.
k_trans = bli_obj_width_after_trans( *a );
// Partition along the k dimension.
for ( i = 0; i < k_trans; i += b_alg )
{
// Determine the current algorithmic blocksize.
// NOTE: We call a trmm-specific function to determine the kc
// blocksize so that we can implement the "nudging" of kc to be
// a multiple of mr or nr, as needed.
b_alg = bli_trmm_determine_kc_b( i, k_trans, a, b,
cntl_blocksize( cntl ) );
// Acquire partitions for A1 and B1.
bli_acquire_mpart_r2l( BLIS_SUBPART1,
i, b_alg, a, &a1 );
bli_acquire_mpart_b2t( BLIS_SUBPART1,
i, b_alg, b, &b1 );
// Initialize objects for packing A1 and B1.
if( thread_am_ichief( thread ) ) {
bli_packm_init( &a1, a1_pack,
cntl_sub_packm_a( cntl ) );
bli_packm_init( &b1, b1_pack,
cntl_sub_packm_b( cntl ) );
}
thread_ibarrier( thread );
// Pack A1 (if instructed).
bli_packm_int( &a1, a1_pack,
cntl_sub_packm_a( cntl ),
trmm_thread_sub_ipackm( thread ) );
// Pack B1 (if instructed).
bli_packm_int( &b1, b1_pack,
cntl_sub_packm_b( cntl ),
trmm_thread_sub_ipackm( thread ) );
// Perform trmm subproblem.
bli_trmm_int( &BLIS_ONE,
a1_pack,
b1_pack,
&BLIS_ONE,
c_pack,
cntl_sub_gemm( cntl ),
trmm_thread_sub_trmm( thread ) );
thread_ibarrier( thread );
}
thread_obarrier( thread );
// Unpack C (if C was packed).
bli_unpackm_int( c_pack, c,
cntl_sub_unpackm_c( cntl ),
trmm_thread_sub_opackm( thread ) );
// If any packing buffers were acquired within packm, release them back
// to the memory manager.
if( thread_am_ochief( thread ) ){
bli_packm_release( c_pack, cntl_sub_packm_c( cntl ) );
}
if( thread_am_ichief( thread ) ){
bli_packm_release( a1_pack, cntl_sub_packm_a( cntl ) );
bli_packm_release( b1_pack, cntl_sub_packm_b( cntl ) );
}
}
void bli_trmm_blk_var3b( obj_t* a,
obj_t* b,
obj_t* c,
trmm_t* cntl )
{
obj_t a1, a1_pack;
obj_t b1, b1_pack;
obj_t c_pack;
dim_t i;
dim_t b_alg;
dim_t k_trans;
// Initialize all pack objects that are passed into packm_init().
bli_obj_init_pack( &a1_pack );
bli_obj_init_pack( &b1_pack );
bli_obj_init_pack( &c_pack );
// Query dimension in partitioning direction.
k_trans = bli_obj_width_after_trans( *a );
// Scale C by beta (if instructed).
bli_scalm_int( &BLIS_ONE,
c,
cntl_sub_scalm( cntl ) );
// Initialize object for packing C.
bli_packm_init( c, &c_pack,
cntl_sub_packm_c( cntl ) );
// Pack C (if instructed).
bli_packm_int( c, &c_pack,
cntl_sub_packm_c( cntl ) );
// Partition along the k dimension.
for ( i = 0; i < k_trans; i += b_alg )
{
// Determine the current algorithmic blocksize.
b_alg = bli_determine_blocksize_b( i, k_trans, b,
cntl_blocksize( cntl ) );
// Acquire partitions for A1 and B1.
bli_acquire_mpart_r2l( BLIS_SUBPART1,
i, b_alg, a, &a1 );
bli_acquire_mpart_b2t( BLIS_SUBPART1,
i, b_alg, b, &b1 );
// Initialize objects for packing A1 and B1.
bli_packm_init( &a1, &a1_pack,
cntl_sub_packm_a( cntl ) );
bli_packm_init( &b1, &b1_pack,
cntl_sub_packm_b( cntl ) );
// Pack A1 (if instructed).
bli_packm_int( &a1, &a1_pack,
cntl_sub_packm_a( cntl ) );
// Pack B1 (if instructed).
bli_packm_int( &b1, &b1_pack,
cntl_sub_packm_b( cntl ) );
// Perform trmm subproblem.
bli_trmm_int( &BLIS_ONE,
&a1_pack,
&b1_pack,
&BLIS_ONE,
&c_pack,
cntl_sub_trmm( cntl ) );
}
// Unpack C (if C was packed).
bli_unpackm_int( &c_pack, c,
cntl_sub_unpackm_c( cntl ) );
// If any packing buffers were acquired within packm, release them back
// to the memory manager.
bli_obj_release_pack( &a1_pack );
bli_obj_release_pack( &b1_pack );
bli_obj_release_pack( &c_pack );
}
void bli_trsm_blk_var1b( obj_t* a,
obj_t* b,
obj_t* c,
trsm_t* cntl )
{
obj_t a1, a1_pack;
obj_t b_pack;
obj_t c1;
dim_t i;
dim_t b_alg;
dim_t m_trans;
dim_t offA;
// Initialize all pack objects that are passed into packm_init().
bli_obj_init_pack( &a1_pack );
bli_obj_init_pack( &b_pack );
// Set the default length of and offset to the non-zero part of A.
m_trans = bli_obj_length_after_trans( *a );
offA = 0;
// If A is upper triangular, we have to adjust where the non-zero part of
// A begins.
if ( bli_obj_is_upper( *a ) )
offA = m_trans - bli_abs( bli_obj_diag_offset_after_trans( *a ) ) -
bli_obj_width_after_trans( *a );
// Initialize object for packing B.
bli_packm_init( b, &b_pack,
cntl_sub_packm_b( cntl ) );
// Pack B1 (if instructed).
bli_packm_int( b, &b_pack,
cntl_sub_packm_b( cntl ) );
// Partition along the remaining portion of the m dimension.
for ( i = offA; i < m_trans; i += b_alg )
{
// Determine the current algorithmic blocksize.
b_alg = bli_determine_blocksize_b( i, m_trans, a,
cntl_blocksize( cntl ) );
// Acquire partitions for A1 and C1.
bli_acquire_mpart_b2t( BLIS_SUBPART1,
i, b_alg, a, &a1 );
bli_acquire_mpart_b2t( BLIS_SUBPART1,
i, b_alg, c, &c1 );
//if ( bli_obj_is_zeros( a1 ) ) continue;
// Initialize object for packing A1.
bli_packm_init( &a1, &a1_pack,
cntl_sub_packm_a( cntl ) );
// Pack A1 (if instructed).
bli_packm_int( &a1, &a1_pack,
cntl_sub_packm_a( cntl ) );
// Perform trsm subproblem.
bli_trsm_int( &BLIS_ONE,
&a1_pack,
&b_pack,
&BLIS_ONE,
&c1,
cntl_sub_trsm( cntl ) );
}
// If any packing buffers were acquired within packm, release them back
// to the memory manager.
bli_obj_release_pack( &a1_pack );
bli_obj_release_pack( &b_pack );
}