void bli_her2k_u_ker_var2( obj_t* a,
obj_t* bh,
obj_t* b,
obj_t* ah,
obj_t* c,
her2k_t* cntl )
{
herk_t herk_cntl;
obj_t c_local;
// Implement her2k kernel in terms of two calls to the corresponding
// herk kernel.
// Note we have to use BLIS_ONE for the second rank-k product since we
// only want to apply beta once. (And beta might be unit anyway if this
// is not the first iteration of variant 3.)
cntl_gemm_ukrs( (&herk_cntl) ) = cntl_gemm_ukrs( cntl );
bli_obj_alias_to( *c, c_local );
bli_herk_u_ker_var2( a,
bh,
&c_local,
&herk_cntl );
bli_obj_scalar_reset( &c_local );
bli_herk_u_ker_var2( b,
ah,
&c_local,
&herk_cntl );
}
func_t* bli_gemm_cntl_ukrs( gemm_t* cntl )
{
dim_t max_depth = 10;
dim_t i;
for ( i = 0; ; ++i )
{
// If the gemm sub-tree is NULL, we are at the leaf.
if ( cntl_sub_gemm( cntl ) == NULL ) break;
// If the above branch was not taken, we can assume the gemm
// sub-tree is valid. Here, we step down into that sub-tree.
cntl = cntl_sub_gemm( cntl );
// Safeguard against infinite loops due to bad control tree
// configuration.
if ( i == max_depth ) bli_abort();
}
return cntl_gemm_ukrs( cntl );
}
void bli_trsm_rl_ker_var2( obj_t* a,
obj_t* b,
obj_t* c,
trsm_t* cntl,
trsm_thrinfo_t* thread )
{
num_t dt_exec = bli_obj_execution_datatype( *c );
doff_t diagoffb = bli_obj_diag_offset( *b );
pack_t schema_a = bli_obj_pack_schema( *a );
pack_t schema_b = bli_obj_pack_schema( *b );
dim_t m = bli_obj_length( *c );
dim_t n = bli_obj_width( *c );
dim_t k = bli_obj_width( *a );
void* buf_a = bli_obj_buffer_at_off( *a );
inc_t cs_a = bli_obj_col_stride( *a );
dim_t pd_a = bli_obj_panel_dim( *a );
inc_t ps_a = bli_obj_panel_stride( *a );
void* buf_b = bli_obj_buffer_at_off( *b );
inc_t rs_b = bli_obj_row_stride( *b );
dim_t pd_b = bli_obj_panel_dim( *b );
inc_t ps_b = bli_obj_panel_stride( *b );
void* buf_c = bli_obj_buffer_at_off( *c );
inc_t rs_c = bli_obj_row_stride( *c );
inc_t cs_c = bli_obj_col_stride( *c );
void* buf_alpha1;
void* buf_alpha2;
FUNCPTR_T f;
func_t* gemmtrsm_ukrs;
func_t* gemm_ukrs;
void* gemmtrsm_ukr;
void* gemm_ukr;
// Grab the address of the internal scalar buffer for the scalar
// attached to A. This will be the alpha scalar used in the gemmtrsm
// subproblems (ie: the scalar that would be applied to the packed
// copy of A prior to it being updated by the trsm subproblem). This
// scalar may be unit, if for example it was applied during packing.
buf_alpha1 = bli_obj_internal_scalar_buffer( *a );
// Grab the address of the internal scalar buffer for the scalar
// attached to C. This will be the "beta" scalar used in the gemm-only
// subproblems that correspond to micro-panels that do not intersect
// the diagonal. We need this separate scalar because it's possible
// that the alpha attached to B was reset, if it was applied during
// packing.
buf_alpha2 = bli_obj_internal_scalar_buffer( *c );
// Index into the type combination array to extract the correct
// function pointer.
f = ftypes[dt_exec];
// Extract from the control tree node the func_t objects containing
// the gemmtrsm and gemm micro-kernel function addresses, and then
// query the function addresses corresponding to the current datatype.
gemmtrsm_ukrs = cntl_gemmtrsm_u_ukrs( cntl );
gemm_ukrs = cntl_gemm_ukrs( cntl );
gemmtrsm_ukr = bli_func_obj_query( dt_exec, gemmtrsm_ukrs );
gemm_ukr = bli_func_obj_query( dt_exec, gemm_ukrs );
// Invoke the function.
f( diagoffb,
schema_a,
schema_b,
m,
n,
k,
buf_alpha1,
buf_a, cs_a, pd_a, ps_a,
buf_b, rs_b, pd_b, ps_b,
buf_alpha2,
buf_c, rs_c, cs_c,
gemmtrsm_ukr,
gemm_ukr,
thread );
}
void bli_trsm_ru_ker_var2( obj_t* a,
obj_t* b,
obj_t* c,
trsm_t* cntl )
{
num_t dt_exec = bli_obj_execution_datatype( *c );
doff_t diagoffb = bli_obj_diag_offset( *b );
dim_t m = bli_obj_length( *c );
dim_t n = bli_obj_width( *c );
dim_t k = bli_obj_width( *a );
void* buf_a = bli_obj_buffer_at_off( *a );
inc_t cs_a = bli_obj_col_stride( *a );
inc_t pd_a = bli_obj_panel_dim( *a );
inc_t ps_a = bli_obj_panel_stride( *a );
void* buf_b = bli_obj_buffer_at_off( *b );
inc_t rs_b = bli_obj_row_stride( *b );
inc_t pd_b = bli_obj_panel_dim( *b );
inc_t ps_b = bli_obj_panel_stride( *b );
void* buf_c = bli_obj_buffer_at_off( *c );
inc_t rs_c = bli_obj_row_stride( *c );
inc_t cs_c = bli_obj_col_stride( *c );
void* buf_alpha1;
void* buf_alpha2;
FUNCPTR_T f;
func_t* gemmtrsm_ukrs;
func_t* gemm_ukrs;
void* gemmtrsm_ukr;
void* gemm_ukr;
// Grab the address of the internal scalar buffer for the scalar
// attached to A. This will be the alpha scalar used in the gemmtrsm
// subproblems (ie: the scalar that would be applied to the packed
// copy of A prior to it being updated by the trsm subproblem). This
// scalar may be unit, if for example it was applied during packing.
buf_alpha1 = bli_obj_internal_scalar_buffer( *a );
// Grab the address of the internal scalar buffer for the scalar
// attached to C. This will be the "beta" scalar used in the gemm-only
// subproblems that correspond to micro-panels that do not intersect
// the diagonal. We need this separate scalar because it's possible
// that the alpha attached to B was reset, if it was applied during
// packing.
buf_alpha2 = bli_obj_internal_scalar_buffer( *c );
// Index into the type combination array to extract the correct
// function pointer.
f = ftypes[dt_exec];
// Adjust cs_a and rs_b if A and B were packed for 4m or 3m. This
// is needed because cs_a and rs_b are used to index into the
// micro-panels of A and B, respectively, and since the pointer
// types in the macro-kernel (scomplex or dcomplex) will result
// in pointer arithmetic that moves twice as far as it should,
// given the datatypes actually stored (float or double), we must
// halve the strides to compensate.
if ( bli_obj_is_panel_packed_4m( *a ) ||
bli_obj_is_panel_packed_3m( *a ) ) { cs_a /= 2; rs_b /= 2; }
// Extract from the control tree node the func_t objects containing
// the gemmtrsm and gemm micro-kernel function addresses, and then
// query the function addresses corresponding to the current datatype.
gemmtrsm_ukrs = cntl_gemmtrsm_l_ukrs( cntl );
gemm_ukrs = cntl_gemm_ukrs( cntl );
gemmtrsm_ukr = bli_func_obj_query( dt_exec, gemmtrsm_ukrs );
gemm_ukr = bli_func_obj_query( dt_exec, gemm_ukrs );
// Invoke the function.
f( diagoffb,
m,
n,
k,
buf_alpha1,
buf_a, cs_a, pd_a, ps_a,
buf_b, rs_b, pd_b, ps_b,
buf_alpha2,
buf_c, rs_c, cs_c,
gemmtrsm_ukr,
gemm_ukr );
}
void bli_herk_l_ker_var2( obj_t* a,
obj_t* b,
obj_t* c,
gemm_t* cntl,
herk_thrinfo_t* thread )
{
num_t dt_exec = bli_obj_execution_datatype( *c );
doff_t diagoffc = bli_obj_diag_offset( *c );
pack_t schema_a = bli_obj_pack_schema( *a );
pack_t schema_b = bli_obj_pack_schema( *b );
dim_t m = bli_obj_length( *c );
dim_t n = bli_obj_width( *c );
dim_t k = bli_obj_width( *a );
void* buf_a = bli_obj_buffer_at_off( *a );
inc_t cs_a = bli_obj_col_stride( *a );
inc_t pd_a = bli_obj_panel_dim( *a );
inc_t ps_a = bli_obj_panel_stride( *a );
void* buf_b = bli_obj_buffer_at_off( *b );
inc_t rs_b = bli_obj_row_stride( *b );
inc_t pd_b = bli_obj_panel_dim( *b );
inc_t ps_b = bli_obj_panel_stride( *b );
void* buf_c = bli_obj_buffer_at_off( *c );
inc_t rs_c = bli_obj_row_stride( *c );
inc_t cs_c = bli_obj_col_stride( *c );
obj_t scalar_a;
obj_t scalar_b;
void* buf_alpha;
void* buf_beta;
FUNCPTR_T f;
func_t* gemm_ukrs;
void* gemm_ukr;
// Detach and multiply the scalars attached to A and B.
bli_obj_scalar_detach( a, &scalar_a );
bli_obj_scalar_detach( b, &scalar_b );
bli_mulsc( &scalar_a, &scalar_b );
// Grab the addresses of the internal scalar buffers for the scalar
// merged above and the scalar attached to C.
buf_alpha = bli_obj_internal_scalar_buffer( scalar_b );
buf_beta = bli_obj_internal_scalar_buffer( *c );
// Index into the type combination array to extract the correct
// function pointer.
f = ftypes[dt_exec];
// Extract from the control tree node the func_t object containing
// the gemm micro-kernel function addresses, and then query the
// function address corresponding to the current datatype.
gemm_ukrs = cntl_gemm_ukrs( cntl );
gemm_ukr = bli_func_obj_query( dt_exec, gemm_ukrs );
// Invoke the function.
f( diagoffc,
schema_a,
schema_b,
m,
n,
k,
buf_alpha,
buf_a, cs_a, pd_a, ps_a,
buf_b, rs_b, pd_b, ps_b,
buf_beta,
buf_c, rs_c, cs_c,
gemm_ukr,
thread );
}
void bli_symm_front( side_t side,
obj_t* alpha,
obj_t* a,
obj_t* b,
obj_t* beta,
obj_t* c,
gemm_t* cntl )
{
obj_t a_local;
obj_t b_local;
obj_t c_local;
// Check parameters.
if ( bli_error_checking_is_enabled() )
bli_symm_check( side, alpha, a, b, beta, c );
// If alpha is zero, scale by beta and return.
if ( bli_obj_equals( alpha, &BLIS_ZERO ) )
{
bli_scalm( beta, c );
return;
}
// Alias A, B, and C in case we need to apply transformations.
bli_obj_alias_to( *a, a_local );
bli_obj_alias_to( *b, b_local );
bli_obj_alias_to( *c, c_local );
// An optimization: If C is stored by rows and the micro-kernel prefers
// contiguous columns, or if C is stored by columns and the micro-kernel
// prefers contiguous rows, transpose the entire operation to allow the
// micro-kernel to access elements of C in its preferred manner.
if (
( bli_obj_is_row_stored( c_local ) &&
bli_func_prefers_contig_cols( bli_obj_datatype( c_local ),
cntl_gemm_ukrs( cntl ) ) ) ||
( bli_obj_is_col_stored( c_local ) &&
bli_func_prefers_contig_rows( bli_obj_datatype( c_local ),
cntl_gemm_ukrs( cntl ) ) )
)
{
bli_toggle_side( side );
bli_obj_induce_trans( b_local );
bli_obj_induce_trans( c_local );
}
// Swap A and B if multiplying A from the right so that "B" contains
// the symmetric matrix.
if ( bli_is_right( side ) )
{
bli_obj_swap( a_local, b_local );
}
gemm_thrinfo_t** infos = bli_create_gemm_thrinfo_paths();
dim_t n_threads = thread_num_threads( infos[0] );
// Invoke the internal back-end.
bli_level3_thread_decorator( n_threads,
(level3_int_t) bli_gemm_int,
alpha,
&a_local,
&b_local,
beta,
&c_local,
(void*) cntl,
(void**) infos );
bli_gemm_thrinfo_free_paths( infos, n_threads );
}
void bli_trmm_ru_ker_var2( obj_t* a,
obj_t* b,
obj_t* c,
trmm_t* cntl,
trmm_thrinfo_t* thread )
{
num_t dt_exec = bli_obj_execution_datatype( *c );
doff_t diagoffb = bli_obj_diag_offset( *b );
dim_t m = bli_obj_length( *c );
dim_t n = bli_obj_width( *c );
dim_t k = bli_obj_width( *a );
void* buf_a = bli_obj_buffer_at_off( *a );
inc_t cs_a = bli_obj_col_stride( *a );
inc_t pd_a = bli_obj_panel_dim( *a );
inc_t ps_a = bli_obj_panel_stride( *a );
void* buf_b = bli_obj_buffer_at_off( *b );
inc_t rs_b = bli_obj_row_stride( *b );
inc_t pd_b = bli_obj_panel_dim( *b );
inc_t ps_b = bli_obj_panel_stride( *b );
void* buf_c = bli_obj_buffer_at_off( *c );
inc_t rs_c = bli_obj_row_stride( *c );
inc_t cs_c = bli_obj_col_stride( *c );
obj_t scalar_a;
obj_t scalar_b;
void* buf_alpha;
void* buf_beta;
FUNCPTR_T f;
func_t* gemm_ukrs;
void* gemm_ukr;
// Detach and multiply the scalars attached to A and B.
bli_obj_scalar_detach( a, &scalar_a );
bli_obj_scalar_detach( b, &scalar_b );
bli_mulsc( &scalar_a, &scalar_b );
// Grab the addresses of the internal scalar buffers for the scalar
// merged above and the scalar attached to C.
buf_alpha = bli_obj_internal_scalar_buffer( scalar_b );
buf_beta = bli_obj_internal_scalar_buffer( *c );
// Index into the type combination array to extract the correct
// function pointer.
f = ftypes[dt_exec];
// Adjust cs_a and rs_b if A and B were packed for 4m or 3m. This
// is needed because cs_a and rs_b are used to index into the
// micro-panels of A and B, respectively, and since the pointer
// types in the macro-kernel (scomplex or dcomplex) will result
// in pointer arithmetic that moves twice as far as it should,
// given the datatypes actually stored (float or double), we must
// halve the strides to compensate.
if ( bli_obj_is_panel_packed_4m( *a ) ||
bli_obj_is_panel_packed_3m( *a ) ) { cs_a /= 2; rs_b /= 2; }
// Extract from the control tree node the func_t object containing
// the gemm micro-kernel function addresses, and then query the
// function address corresponding to the current datatype.
gemm_ukrs = cntl_gemm_ukrs( cntl );
gemm_ukr = bli_func_obj_query( dt_exec, gemm_ukrs );
// Invoke the function.
f( diagoffb,
m,
n,
k,
buf_alpha,
buf_a, cs_a, pd_a, ps_a,
buf_b, rs_b, pd_b, ps_b,
buf_beta,
buf_c, rs_c, cs_c,
gemm_ukr,
thread );
}
