siz_t bli_thread_get_range_ndim
(
dir_t direct,
thrinfo_t* thr,
obj_t* a,
obj_t* b,
obj_t* c,
cntl_t* cntl,
cntx_t* cntx,
dim_t* start,
dim_t* end
)
{
bszid_t bszid = bli_cntl_bszid( cntl );
opid_t family = bli_cntx_get_family( cntx );
// This is part of trsm's current implementation, whereby right side
// cases are implemented in left-side micro-kernels, which requires
// we swap the usage of the register blocksizes for the purposes of
// packing A and B.
if ( family == BLIS_TRSM )
{
if ( bli_obj_root_is_triangular( *b ) ) bszid = BLIS_MR;
else bszid = BLIS_NR;
}
blksz_t* bmult = bli_cntx_get_bmult( bszid, cntx );
obj_t* x;
bool_t use_weighted;
// Use the operation family to choose the one of the two matrices
// being partitioned that potentially has structure, and also to
// decide whether or not we need to use weighted range partitioning.
// NOTE: It's important that we use non-weighted range partitioning
// for hemm and symm (ie: the gemm family) because the weighted
// function will mistakenly skip over unstored regions of the
// structured matrix, even though they represent part of that matrix
// that will be dense and full (after packing).
if ( family == BLIS_GEMM ) { x = b; use_weighted = FALSE; }
else if ( family == BLIS_HERK ) { x = c; use_weighted = TRUE; }
else if ( family == BLIS_TRMM ) { x = b; use_weighted = TRUE; }
else /*family == BLIS_TRSM*/ { x = b; use_weighted = FALSE; }
if ( use_weighted )
{
if ( direct == BLIS_FWD )
return bli_thread_get_range_weighted_l2r( thr, x, bmult, start, end );
else
return bli_thread_get_range_weighted_r2l( thr, x, bmult, start, end );
}
else
{
if ( direct == BLIS_FWD )
return bli_thread_get_range_l2r( thr, x, bmult, start, end );
else
return bli_thread_get_range_r2l( thr, x, bmult, start, end );
}
}
void bli_trsm_blk_var1b( obj_t* a,
obj_t* b,
obj_t* c,
trsm_t* cntl,
trsm_thrinfo_t* thread )
{
obj_t b_pack_s;
obj_t a1_pack_s;
obj_t a1, c1;
obj_t* b_pack = NULL;
obj_t* a1_pack = NULL;
dim_t i;
dim_t b_alg;
// Prune any zero region that exists along the partitioning dimension.
bli_trsm_prune_unref_mparts_m( a, b, c );
// Initialize object for packing B.
if( thread_am_ochief( thread ) ) {
bli_obj_init_pack( &b_pack_s );
bli_packm_init( b, &b_pack_s,
cntl_sub_packm_b( cntl ) );
}
b_pack = thread_obroadcast( thread, &b_pack_s );
// Initialize object for packing B.
if( thread_am_ichief( thread ) ) {
bli_obj_init_pack( &a1_pack_s );
}
a1_pack = thread_ibroadcast( thread, &a1_pack_s );
// Pack B1 (if instructed).
bli_packm_int( b, b_pack,
cntl_sub_packm_b( cntl ),
trsm_thread_sub_opackm( thread ) );
dim_t my_start, my_end;
num_t dt = bli_obj_execution_datatype( *a );
dim_t bf = ( bli_obj_root_is_triangular( *a ) ?
bli_info_get_default_mr( BLIS_TRSM, dt ) :
bli_info_get_default_nr( BLIS_TRSM, dt ) );
bli_get_range_b2t( thread, a, bf,
&my_start, &my_end );
// Partition along the remaining portion of the m dimension.
for ( i = my_start; i < my_end; i += b_alg )
{
// Determine the current algorithmic blocksize.
b_alg = bli_determine_blocksize_b( i, my_end, a,
cntl_blocksize( cntl ) );
// Acquire partitions for A1 and C1.
bli_acquire_mpart_b2t( BLIS_SUBPART1,
i, b_alg, a, &a1 );
bli_acquire_mpart_b2t( BLIS_SUBPART1,
i, b_alg, c, &c1 );
// Initialize object for packing A1.
if( thread_am_ichief( thread ) ) {
bli_packm_init( &a1, a1_pack,
cntl_sub_packm_a( cntl ) );
}
thread_ibarrier( thread );
// Pack A1 (if instructed).
bli_packm_int( &a1, a1_pack,
cntl_sub_packm_a( cntl ),
trsm_thread_sub_ipackm( thread ) );
// Perform trsm subproblem.
bli_trsm_int( &BLIS_ONE,
a1_pack,
b_pack,
&BLIS_ONE,
&c1,
cntl_sub_trsm( cntl ),
trsm_thread_sub_trsm( thread ) );
thread_ibarrier( thread );
}
// If any packing buffers were acquired within packm, release them back
// to the memory manager.
thread_obarrier( thread );
if( thread_am_ochief( thread ) )
bli_packm_release( b_pack, cntl_sub_packm_b( cntl ) );
if( thread_am_ichief( thread ) )
bli_packm_release( a1_pack, cntl_sub_packm_a( cntl ) );
}
void bli_trmm_int( obj_t* alpha,
obj_t* a,
obj_t* b,
obj_t* beta,
obj_t* c,
trmm_t* cntl )
{
obj_t a_local;
obj_t b_local;
obj_t c_local;
bool_t side, uplo;
varnum_t n;
impl_t i;
FUNCPTR_T f;
// Check parameters.
if ( bli_error_checking_is_enabled() )
bli_trmm_int_check( alpha, a, b, beta, c, cntl );
// If C has a zero dimension, return early.
if ( bli_obj_has_zero_dim( *c ) ) return;
// If A or B has a zero dimension, scale C by beta and return early.
if ( bli_obj_has_zero_dim( *a ) ||
bli_obj_has_zero_dim( *b ) )
{
bli_scalm( beta, c );
return;
}
// Alias A and B in case we need to update attached scalars.
bli_obj_alias_to( *a, a_local );
bli_obj_alias_to( *b, b_local );
// Alias C in case we need to induce a transposition.
bli_obj_alias_to( *c, c_local );
// If we are about to call a leaf-level implementation, and matrix C
// still needs a transposition, then we must induce one by swapping the
// strides and dimensions. Note that this transposition would normally
// be handled explicitly in the packing of C, but if C is not being
// packed, this is our last chance to handle the transposition.
if ( cntl_is_leaf( cntl ) && bli_obj_has_trans( *c ) )
{
bli_obj_induce_trans( c_local );
bli_obj_set_onlytrans( BLIS_NO_TRANSPOSE, c_local );
}
// If alpha is non-unit, typecast and apply it to the scalar attached
// to B.
if ( !bli_obj_equals( alpha, &BLIS_ONE ) )
{
bli_obj_scalar_apply_scalar( alpha, &b_local );
}
// If beta is non-unit, typecast and apply it to the scalar attached
// to C.
if ( !bli_obj_equals( beta, &BLIS_ONE ) )
{
bli_obj_scalar_apply_scalar( beta, &c_local );
}
// Set two bools: one based on the implied side parameter (the structure
// of the root object) and one based on the uplo field of the triangular
// matrix's root object (whether that is matrix A or matrix B).
if ( bli_obj_root_is_triangular( *a ) )
{
side = 0;
if ( bli_obj_root_is_lower( *a ) ) uplo = 0;
else uplo = 1;
}
else // if ( bli_obj_root_is_triangular( *b ) )
{
side = 1;
// Set a bool based on the uplo field of A's root object.
if ( bli_obj_root_is_lower( *b ) ) uplo = 0;
else uplo = 1;
}
// Extract the variant number and implementation type.
n = cntl_var_num( cntl );
i = cntl_impl_type( cntl );
// Index into the variant array to extract the correct function pointer.
f = vars[side][uplo][n][i];
// Invoke the variant.
f( &a_local,
&b_local,
&c_local,
cntl );
}
void bli_trsm_int
(
obj_t* alpha,
obj_t* a,
obj_t* b,
obj_t* beta,
obj_t* c,
cntx_t* cntx,
rntm_t* rntm,
cntl_t* cntl,
thrinfo_t* thread
)
{
obj_t a_local;
obj_t b_local;
obj_t c_local;
trsm_var_oft f;
// Check parameters.
if ( bli_error_checking_is_enabled() )
bli_gemm_basic_check( alpha, a, b, beta, c, cntx );
// If C has a zero dimension, return early.
if ( bli_obj_has_zero_dim( c ) ) return;
// If A or B has a zero dimension, scale C by beta and return early.
if ( bli_obj_has_zero_dim( a ) ||
bli_obj_has_zero_dim( b ) )
{
if ( bli_thread_am_ochief( thread ) )
bli_scalm( beta, c );
bli_thread_obarrier( thread );
return;
}
// Alias A and B in case we need to update attached scalars.
bli_obj_alias_to( a, &a_local );
bli_obj_alias_to( b, &b_local );
// Alias C in case we need to induce a transposition.
bli_obj_alias_to( c, &c_local );
// If we are about to call a leaf-level implementation, and matrix C
// still needs a transposition, then we must induce one by swapping the
// strides and dimensions. Note that this transposition would normally
// be handled explicitly in the packing of C, but if C is not being
// packed, this is our last chance to handle the transposition.
if ( bli_cntl_is_leaf( cntl ) && bli_obj_has_trans( c ) )
{
bli_obj_induce_trans( &c_local );
bli_obj_set_onlytrans( BLIS_NO_TRANSPOSE, &c_local );
}
// If beta is non-unit, apply it to the scalar attached to C.
if ( !bli_obj_equals( beta, &BLIS_ONE ) )
{
bli_obj_scalar_apply_scalar( beta, &c_local );
}
// Set two bools: one based on the implied side parameter (the structure
// of the root object) and one based on the uplo field of the triangular
// matrix's root object (whether that is matrix A or matrix B).
if ( bli_obj_root_is_triangular( a ) )
{
// If alpha is non-unit, typecast and apply it to the scalar
// attached to B (the non-triangular matrix).
if ( !bli_obj_equals( alpha, &BLIS_ONE ) )
{
bli_obj_scalar_apply_scalar( alpha, &b_local );
}
}
else // if ( bli_obj_root_is_triangular( b ) )
{
// If alpha is non-unit, typecast and apply it to the scalar
// attached to A (the non-triangular matrix).
if ( !bli_obj_equals( alpha, &BLIS_ONE ) )
{
bli_obj_scalar_apply_scalar( alpha, &a_local );
}
}
// FGVZ->TMS: Is this barrier still needed?
bli_thread_obarrier( thread );
// Create the next node in the thrinfo_t structure.
bli_thrinfo_grow( rntm, cntl, thread );
// Extract the function pointer from the current control tree node.
f = bli_cntl_var_func( cntl );
// Invoke the variant.
f
(
&a_local,
&b_local,
&c_local,
cntx,
rntm,
cntl,
thread
);
}
