void bli_herk_blk_var3f( obj_t* a,
obj_t* ah,
obj_t* c,
herk_t* cntl,
herk_thrinfo_t* thread )
{
obj_t c_pack_s;
obj_t a1_pack_s, ah1_pack_s;
obj_t a1, ah1;
obj_t* a1_pack = NULL;
obj_t* ah1_pack = NULL;
obj_t* c_pack = NULL;
dim_t i;
dim_t b_alg;
dim_t k_trans;
if( thread_am_ochief( thread ) ) {
// Initialize object for packing C.
bli_obj_init_pack( &c_pack_s );
bli_packm_init( c, &c_pack_s,
cntl_sub_packm_c( cntl ) );
// Scale C by beta (if instructed).
bli_scalm_int( &BLIS_ONE,
c,
cntl_sub_scalm( cntl ) );
}
c_pack = thread_obroadcast( thread, &c_pack_s );
// Initialize all pack objects that are passed into packm_init().
if( thread_am_ichief( thread ) ) {
bli_obj_init_pack( &a1_pack_s );
bli_obj_init_pack( &ah1_pack_s );
}
a1_pack = thread_ibroadcast( thread, &a1_pack_s );
ah1_pack = thread_ibroadcast( thread, &ah1_pack_s );
// Pack C (if instructed).
bli_packm_int( c, c_pack,
cntl_sub_packm_c( cntl ),
herk_thread_sub_opackm( thread ) );
// Query dimension in partitioning direction.
k_trans = bli_obj_width_after_trans( *a );
// Partition along the k dimension.
for ( i = 0; i < k_trans; i += b_alg )
{
// Determine the current algorithmic blocksize.
b_alg = bli_determine_blocksize_f( i, k_trans, a,
cntl_blocksize( cntl ) );
// Acquire partitions for A1 and A1'.
bli_acquire_mpart_l2r( BLIS_SUBPART1,
i, b_alg, a, &a1 );
bli_acquire_mpart_t2b( BLIS_SUBPART1,
i, b_alg, ah, &ah1 );
// Initialize objects for packing A1 and A1'.
if( thread_am_ichief( thread ) ) {
bli_packm_init( &a1, a1_pack,
cntl_sub_packm_a( cntl ) );
bli_packm_init( &ah1, ah1_pack,
cntl_sub_packm_b( cntl ) );
}
thread_ibarrier( thread );
// Pack A1 (if instructed).
bli_packm_int( &a1, a1_pack,
cntl_sub_packm_a( cntl ),
herk_thread_sub_ipackm( thread ) );
// Pack B1 (if instructed).
bli_packm_int( &ah1, ah1_pack,
cntl_sub_packm_b( cntl ),
herk_thread_sub_ipackm( thread ) );
// Perform herk subproblem.
bli_herk_int( &BLIS_ONE,
a1_pack,
ah1_pack,
&BLIS_ONE,
c_pack,
cntl_sub_herk( cntl ),
herk_thread_sub_herk( thread ) );
// This variant executes multiple rank-k updates. Therefore, if the
// internal beta scalar on matrix C is non-zero, we must use it
// only for the first iteration (and then BLIS_ONE for all others).
// And since c_pack is a local obj_t, we can simply overwrite the
// internal beta scalar with BLIS_ONE once it has been used in the
// first iteration.
if ( i == 0 ) thread_ibarrier( thread );
if ( i == 0 && thread_am_ichief( thread ) ) bli_obj_scalar_reset( c_pack );
}
thread_obarrier( thread );
// Unpack C (if C was packed).
bli_unpackm_int( c_pack, c,
cntl_sub_unpackm_c( cntl ),
herk_thread_sub_opackm( thread ) );
// If any packing buffers were acquired within packm, release them back
// to the memory manager.
if( thread_am_ochief( thread ) ) {
bli_obj_release_pack( c_pack );
}
if( thread_am_ichief( thread ) ) {
bli_obj_release_pack( a1_pack );
bli_obj_release_pack( ah1_pack );
}
}
void bli_herk_blk_var1f( obj_t* a,
obj_t* ah,
obj_t* c,
gemm_t* cntl,
herk_thrinfo_t* thread )
{
obj_t ah_pack_s;
obj_t a1_pack_s, c1_pack_s;
obj_t a1, c1;
obj_t* a1_pack;
obj_t* c1_pack;
obj_t* ah_pack;
dim_t i;
dim_t b_alg;
// Prune any zero region that exists along the partitioning dimension.
bli_herk_prune_unref_mparts_m( a, ah, c );
if( thread_am_ochief( thread ) ) {
// Initialize object for packing A'.
bli_obj_init_pack( &ah_pack_s );
bli_packm_init( ah, &ah_pack_s,
cntl_sub_packm_b( cntl ) );
// Scale C by beta (if instructed).
// Since scalm doesn't support multithreading yet, must be done by chief thread (ew)
bli_scalm_int( &BLIS_ONE,
c,
cntl_sub_scalm( cntl ) );
}
ah_pack = thread_obroadcast( thread, &ah_pack_s );
// Initialize pack objects that are passed into packm_init() for A and C.
if( thread_am_ichief( thread ) ) {
bli_obj_init_pack( &a1_pack_s );
bli_obj_init_pack( &c1_pack_s );
}
a1_pack = thread_ibroadcast( thread, &a1_pack_s );
c1_pack = thread_ibroadcast( thread, &c1_pack_s );
// Pack A' (if instructed).
bli_packm_int( ah, ah_pack,
cntl_sub_packm_b( cntl ),
herk_thread_sub_opackm( thread ) );
dim_t my_start, my_end;
bli_get_range_weighted_t2b( thread, c,
bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ),
&my_start, &my_end );
// Partition along the m dimension.
for ( i = my_start; i < my_end; i += b_alg )
{
// Determine the current algorithmic blocksize.
b_alg = bli_determine_blocksize_f( i, my_end, a,
cntl_blocksize( cntl ) );
// Acquire partitions for A1 and C1.
bli_acquire_mpart_t2b( BLIS_SUBPART1,
i, b_alg, a, &a1 );
bli_acquire_mpart_t2b( BLIS_SUBPART1,
i, b_alg, c, &c1 );
// Initialize objects for packing A1 and C1.
if( thread_am_ichief( thread ) ) {
bli_packm_init( &a1, a1_pack,
cntl_sub_packm_a( cntl ) );
bli_packm_init( &c1, c1_pack,
cntl_sub_packm_c( cntl ) );
}
thread_ibarrier( thread );
// Pack A1 (if instructed).
bli_packm_int( &a1, a1_pack,
cntl_sub_packm_a( cntl ),
herk_thread_sub_ipackm( thread ) );
// Pack C1 (if instructed).
bli_packm_int( &c1, c1_pack,
cntl_sub_packm_c( cntl ),
herk_thread_sub_ipackm( thread ) );
// Perform herk subproblem.
bli_herk_int( &BLIS_ONE,
a1_pack,
ah_pack,
&BLIS_ONE,
c1_pack,
cntl_sub_gemm( cntl ),
herk_thread_sub_herk( thread ) );
thread_ibarrier( thread );
// Unpack C1 (if C1 was packed).
bli_unpackm_int( c1_pack, &c1,
cntl_sub_unpackm_c( cntl ),
herk_thread_sub_ipackm( thread ) );
}
// If any packing buffers were acquired within packm, release them back
// to the memory manager.
thread_obarrier( thread );
if( thread_am_ochief( thread ) )
bli_packm_release( ah_pack, cntl_sub_packm_b( cntl ) );
if( thread_am_ichief( thread ) ) {
bli_packm_release( a1_pack, cntl_sub_packm_a( cntl ) );
bli_packm_release( c1_pack, cntl_sub_packm_c( cntl ) );
}
}
void bli_herk_blk_var2f( obj_t* a,
obj_t* ah,
obj_t* c,
gemm_t* cntl,
herk_thrinfo_t* thread )
{
obj_t a_pack_s;
obj_t ah1_pack_s, c1S_pack_s;
obj_t ah1, c1, c1S;
obj_t aS_pack;
obj_t* a_pack;
obj_t* ah1_pack;
obj_t* c1S_pack;
dim_t i;
dim_t b_alg;
dim_t n_trans;
subpart_t stored_part;
// The upper and lower variants are identical, except for which
// merged subpartition is acquired in the loop body.
if ( bli_obj_is_lower( *c ) ) stored_part = BLIS_SUBPART1B;
else stored_part = BLIS_SUBPART1T;
if( thread_am_ochief( thread ) ) {
// Initialize object for packing A
bli_obj_init_pack( &a_pack_s );
bli_packm_init( a, &a_pack_s,
cntl_sub_packm_a( cntl ) );
// Scale C by beta (if instructed).
bli_scalm_int( &BLIS_ONE,
c,
cntl_sub_scalm( cntl ) );
}
a_pack = thread_obroadcast( thread, &a_pack_s );
// Initialize pack objects for C and A' that are passed into packm_init().
if( thread_am_ichief( thread ) ) {
bli_obj_init_pack( &ah1_pack_s );
bli_obj_init_pack( &c1S_pack_s );
}
ah1_pack = thread_ibroadcast( thread, &ah1_pack_s );
c1S_pack = thread_ibroadcast( thread, &c1S_pack_s );
// Pack A (if instructed).
bli_packm_int( a, a_pack,
cntl_sub_packm_a( cntl ),
herk_thread_sub_opackm( thread ) );
// Query dimension in partitioning direction.
n_trans = bli_obj_width_after_trans( *c );
dim_t start, end;
// Needs to be replaced with a weighted range because triangle
bli_get_range_weighted( thread, 0, n_trans,
bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ),
bli_obj_is_lower( *c ), &start, &end );
// Partition along the n dimension.
for ( i = start; i < end; i += b_alg )
{
// Determine the current algorithmic blocksize.
b_alg = bli_determine_blocksize_f( i, end, a,
cntl_blocksize( cntl ) );
// Acquire partitions for A1' and C1.
bli_acquire_mpart_l2r( BLIS_SUBPART1,
i, b_alg, ah, &ah1 );
bli_acquire_mpart_l2r( BLIS_SUBPART1,
i, b_alg, c, &c1 );
// Partition off the stored region of C1 and the corresponding region
// of A_pack.
bli_acquire_mpart_t2b( stored_part,
i, b_alg, &c1, &c1S );
bli_acquire_mpart_t2b( stored_part,
i, b_alg, a_pack, &aS_pack );
// Initialize objects for packing A1' and C1.
if( thread_am_ichief( thread ) ) {
bli_packm_init( &ah1, ah1_pack,
cntl_sub_packm_b( cntl ) );
bli_packm_init( &c1S, c1S_pack,
cntl_sub_packm_c( cntl ) );
}
thread_ibarrier( thread ) ;
// Pack A1' (if instructed).
bli_packm_int( &ah1, ah1_pack,
cntl_sub_packm_b( cntl ),
herk_thread_sub_ipackm( thread ) );
// Pack C1 (if instructed).
bli_packm_int( &c1S, c1S_pack,
cntl_sub_packm_c( cntl ),
herk_thread_sub_ipackm( thread ) ) ;
// Perform herk subproblem.
bli_herk_int( &BLIS_ONE,
&aS_pack,
ah1_pack,
&BLIS_ONE,
c1S_pack,
cntl_sub_gemm( cntl ),
herk_thread_sub_herk( thread ) );
thread_ibarrier( thread );
// Unpack C1 (if C1 was packed).
bli_unpackm_int( c1S_pack, &c1S,
cntl_sub_unpackm_c( cntl ),
herk_thread_sub_ipackm( thread ) );
}
// If any packing buffers were acquired within packm, release them back
// to the memory manager.
thread_obarrier( thread );
if( thread_am_ochief( thread ) )
bli_packm_release( a_pack, cntl_sub_packm_a( cntl ) );
if( thread_am_ichief( thread ) ) {
bli_packm_release( ah1_pack, cntl_sub_packm_b( cntl ) );
bli_packm_release( c1S_pack, cntl_sub_packm_c( cntl ) );
}
}