void bli_gemm_blk_var1f( obj_t* a,
obj_t* b,
obj_t* c,
cntx_t* cntx,
gemm_t* cntl,
gemm_thrinfo_t* thread )
{
//The s is for "lives on the stack"
obj_t b_pack_s;
obj_t a1_pack_s, c1_pack_s;
obj_t a1, c1;
obj_t* a1_pack = NULL;
obj_t* b_pack = NULL;
obj_t* c1_pack = NULL;
dim_t i;
dim_t b_alg;
if( thread_am_ochief( thread ) ) {
// Initialize object for packing B.
bli_obj_init_pack( &b_pack_s );
bli_packm_init( b, &b_pack_s,
cntx, cntl_sub_packm_b( cntl ) );
// Scale C by beta (if instructed).
// Since scalm doesn't support multithreading yet, must be done by chief thread (ew)
bli_scalm_int( &BLIS_ONE,
c,
cntx, cntl_sub_scalm( cntl ) );
}
b_pack = thread_obroadcast( thread, &b_pack_s );
// Initialize objects passed into bli_packm_init for A and C
if( thread_am_ichief( thread ) ) {
bli_obj_init_pack( &a1_pack_s );
bli_obj_init_pack( &c1_pack_s );
}
a1_pack = thread_ibroadcast( thread, &a1_pack_s );
c1_pack = thread_ibroadcast( thread, &c1_pack_s );
// Pack B (if instructed).
bli_packm_int( b, b_pack,
cntx, cntl_sub_packm_b( cntl ),
gemm_thread_sub_opackm( thread ) );
dim_t my_start, my_end;
bli_get_range_t2b( thread, a,
bli_cntx_get_bmult( cntl_bszid( cntl ), cntx ),
&my_start, &my_end );
// Partition along the m dimension.
for ( i = my_start; i < my_end; i += b_alg )
{
// Determine the current algorithmic blocksize.
// NOTE: Use of a (for execution datatype) is intentional!
// This causes the right blocksize to be used if c and a are
// complex and b is real.
b_alg = bli_determine_blocksize_f( i, my_end, a,
cntl_bszid( cntl ), cntx );
// Acquire partitions for A1 and C1.
bli_acquire_mpart_t2b( BLIS_SUBPART1,
i, b_alg, a, &a1 );
bli_acquire_mpart_t2b( BLIS_SUBPART1,
i, b_alg, c, &c1 );
// Initialize objects for packing A1 and C1.
if( thread_am_ichief( thread ) ) {
bli_packm_init( &a1, a1_pack,
cntx, cntl_sub_packm_a( cntl ) );
bli_packm_init( &c1, c1_pack,
cntx, cntl_sub_packm_c( cntl ) );
}
thread_ibarrier( thread );
// Pack A1 (if instructed).
bli_packm_int( &a1, a1_pack,
cntx, cntl_sub_packm_a( cntl ),
gemm_thread_sub_ipackm( thread ) );
// Pack C1 (if instructed).
bli_packm_int( &c1, c1_pack,
cntx, cntl_sub_packm_c( cntl ),
gemm_thread_sub_ipackm( thread ) );
// Perform gemm subproblem.
bli_gemm_int( &BLIS_ONE,
a1_pack,
b_pack,
&BLIS_ONE,
c1_pack,
cntx,
cntl_sub_gemm( cntl ),
gemm_thread_sub_gemm( thread ) );
thread_ibarrier( thread );
// Unpack C1 (if C1 was packed).
// Currently must be done by 1 thread
bli_unpackm_int( c1_pack, &c1,
cntx, cntl_sub_unpackm_c( cntl ),
gemm_thread_sub_ipackm( thread ) );
}
// If any packing buffers were acquired within packm, release them back
// to the memory manager.
thread_obarrier( thread );
if( thread_am_ochief( thread ) )
bli_packm_release( b_pack, cntl_sub_packm_b( cntl ) );
if( thread_am_ichief( thread ) ){
bli_packm_release( a1_pack, cntl_sub_packm_a( cntl ) );
bli_packm_release( c1_pack, cntl_sub_packm_c( cntl ) );
}
}
void bli_trsm_blk_var1f( obj_t* a,
obj_t* b,
obj_t* c,
trsm_t* cntl,
trsm_thrinfo_t* thread )
{
obj_t b_pack_s;
obj_t a1_pack_s;
obj_t a1, c1;
obj_t* b_pack = NULL;
obj_t* a1_pack = NULL;
dim_t i;
dim_t b_alg;
dim_t m_trans;
dim_t offA;
// Initialize object for packing B.
if( thread_am_ochief( thread ) ) {
bli_obj_init_pack( &b_pack_s );
bli_packm_init( b, &b_pack_s,
cntl_sub_packm_b( cntl ) );
}
b_pack = thread_obroadcast( thread, &b_pack_s );
// Initialize object for packing B.
if( thread_am_ichief( thread ) ) {
bli_obj_init_pack( &a1_pack_s );
}
a1_pack = thread_ibroadcast( thread, &a1_pack_s );
// Pack B1 (if instructed).
bli_packm_int( b, b_pack,
cntl_sub_packm_b( cntl ),
trsm_thread_sub_opackm( thread ) );
// Set the default length of and offset to the non-zero part of A.
m_trans = bli_obj_length_after_trans( *a );
offA = 0;
// If A is lower triangular, we have to adjust where the non-zero part of
// A begins.
if ( bli_obj_is_lower( *a ) )
offA = bli_abs( bli_obj_diag_offset_after_trans( *a ) );
dim_t start, end;
num_t dt = bli_obj_execution_datatype( *a );
bli_get_range_t2b( thread, offA, m_trans,
//bli_lcm( bli_info_get_default_nr( BLIS_TRSM, dt ), bli_info_get_default_mr( BLIS_TRSM, dt ) ),
bli_info_get_default_mc( BLIS_TRSM, dt ),
&start, &end );
// Partition along the remaining portion of the m dimension.
for ( i = start; i < end; i += b_alg )
{
// Determine the current algorithmic blocksize.
b_alg = bli_determine_blocksize_f( i, end, a,
cntl_blocksize( cntl ) );
// Acquire partitions for A1 and C1.
bli_acquire_mpart_t2b( BLIS_SUBPART1,
i, b_alg, a, &a1 );
bli_acquire_mpart_t2b( BLIS_SUBPART1,
i, b_alg, c, &c1 );
// Initialize object for packing A1.
if( thread_am_ichief( thread ) ) {
bli_packm_init( &a1, a1_pack,
cntl_sub_packm_a( cntl ) );
}
thread_ibarrier( thread );
// Pack A1 (if instructed).
bli_packm_int( &a1, a1_pack,
cntl_sub_packm_a( cntl ),
trsm_thread_sub_ipackm( thread ) );
// Perform trsm subproblem.
bli_trsm_int( &BLIS_ONE,
a1_pack,
b_pack,
&BLIS_ONE,
&c1,
cntl_sub_trsm( cntl ),
trsm_thread_sub_trsm( thread ) );
thread_ibarrier( thread );
}
// If any packing buffers were acquired within packm, release them back
// to the memory manager.
thread_obarrier( thread );
if( thread_am_ochief( thread ) )
bli_packm_release( b_pack, cntl_sub_packm_b( cntl ) );
if( thread_am_ichief( thread ) )
bli_packm_release( a1_pack, cntl_sub_packm_a( cntl ) );
}
void bli_gemm_blk_var4f( obj_t* a,
obj_t* b,
obj_t* c,
gemm_t* cntl,
gemm_thrinfo_t* thread )
{
extern packm_t* gemm3mh_packa_cntl_ro;
extern packm_t* gemm3mh_packa_cntl_io;
extern packm_t* gemm3mh_packa_cntl_rpi;
packm_t* packa_cntl_ro = gemm3mh_packa_cntl_ro;
packm_t* packa_cntl_io = gemm3mh_packa_cntl_io;
packm_t* packa_cntl_rpi = gemm3mh_packa_cntl_rpi;
//The s is for "lives on the stack"
obj_t b_pack_s;
obj_t a1_pack_s, c1_pack_s;
obj_t a1, c1;
obj_t* a1_pack = NULL;
obj_t* b_pack = NULL;
obj_t* c1_pack = NULL;
dim_t i;
dim_t b_alg;
dim_t m_trans;
if( thread_am_ochief( thread ) ) {
// Initialize object for packing B.
bli_obj_init_pack( &b_pack_s );
bli_packm_init( b, &b_pack_s,
cntl_sub_packm_b( cntl ) );
// Scale C by beta (if instructed).
// Since scalm doesn't support multithreading yet, must be done by chief thread (ew)
bli_scalm_int( &BLIS_ONE,
c,
cntl_sub_scalm( cntl ) );
}
b_pack = thread_obroadcast( thread, &b_pack_s );
// Initialize objects passed into bli_packm_init for A and C
if( thread_am_ichief( thread ) ) {
bli_obj_init_pack( &a1_pack_s );
bli_obj_init_pack( &c1_pack_s );
}
a1_pack = thread_ibroadcast( thread, &a1_pack_s );
c1_pack = thread_ibroadcast( thread, &c1_pack_s );
// Pack B (if instructed).
bli_packm_int( b, b_pack,
cntl_sub_packm_b( cntl ),
gemm_thread_sub_opackm( thread ) );
// Query dimension in partitioning direction.
m_trans = bli_obj_length_after_trans( *a );
dim_t start, end;
bli_get_range_t2b( thread, 0, m_trans,
bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ),
&start, &end );
// Partition along the m dimension.
for ( i = start; i < end; i += b_alg )
{
// Determine the current algorithmic blocksize.
// NOTE: Use of a (for execution datatype) is intentional!
// This causes the right blocksize to be used if c and a are
// complex and b is real.
b_alg = bli_determine_blocksize_f( i, end, a,
cntl_blocksize( cntl ) );
// Acquire partitions for A1 and C1.
bli_acquire_mpart_t2b( BLIS_SUBPART1,
i, b_alg, a, &a1 );
bli_acquire_mpart_t2b( BLIS_SUBPART1,
i, b_alg, c, &c1 );
// Initialize objects for packing A1 and C1.
if( thread_am_ichief( thread ) ) {
bli_packm_init( &a1, a1_pack,
packa_cntl_ro );
bli_packm_init( &c1, c1_pack,
cntl_sub_packm_c( cntl ) );
}
thread_ibarrier( thread );
// Pack A1 (if instructed).
bli_packm_int( &a1, a1_pack,
packa_cntl_ro,
gemm_thread_sub_ipackm( thread ) );
// Pack C1 (if instructed).
bli_packm_int( &c1, c1_pack,
cntl_sub_packm_c( cntl ),
gemm_thread_sub_ipackm( thread ) );
// Perform gemm subproblem.
bli_gemm_int( &BLIS_ONE,
a1_pack,
b_pack,
&BLIS_ONE,
c1_pack,
cntl_sub_gemm( cntl ),
gemm_thread_sub_gemm( thread ) );
thread_ibarrier( thread );
// Only apply beta within the first of three subproblems.
if ( thread_am_ichief( thread ) ) bli_obj_scalar_reset( c1_pack );
// Initialize objects for packing A1 and C1.
if( thread_am_ichief( thread ) ) {
bli_packm_init( &a1, a1_pack,
packa_cntl_io );
}
thread_ibarrier( thread );
// Pack A1 (if instructed).
bli_packm_int( &a1, a1_pack,
packa_cntl_io,
gemm_thread_sub_ipackm( thread ) );
// Perform gemm subproblem.
bli_gemm_int( &BLIS_ONE,
a1_pack,
b_pack,
&BLIS_ONE,
c1_pack,
cntl_sub_gemm( cntl ),
gemm_thread_sub_gemm( thread ) );
thread_ibarrier( thread );
// Initialize objects for packing A1 and C1.
if( thread_am_ichief( thread ) ) {
bli_packm_init( &a1, a1_pack,
packa_cntl_rpi );
}
thread_ibarrier( thread );
// Pack A1 (if instructed).
bli_packm_int( &a1, a1_pack,
packa_cntl_rpi,
gemm_thread_sub_ipackm( thread ) );
// Perform gemm subproblem.
bli_gemm_int( &BLIS_ONE,
a1_pack,
b_pack,
&BLIS_ONE,
c1_pack,
cntl_sub_gemm( cntl ),
gemm_thread_sub_gemm( thread ) );
thread_ibarrier( thread );
// Unpack C1 (if C1 was packed).
// Currently must be done by 1 thread
bli_unpackm_int( c1_pack, &c1,
cntl_sub_unpackm_c( cntl ),
gemm_thread_sub_ipackm( thread ) );
}
// If any packing buffers were acquired within packm, release them back
// to the memory manager.
thread_obarrier( thread );
if( thread_am_ochief( thread ) )
bli_packm_release( b_pack, cntl_sub_packm_b( cntl ) );
if( thread_am_ichief( thread ) ){
// It doesn't matter which packm cntl node we pass in, as long
// as it is valid, packm_release() will release the mem_t entry.
bli_packm_release( a1_pack, packa_cntl_ro );
bli_packm_release( c1_pack, cntl_sub_packm_c( cntl ) );
}
}
