void advec_cell_kernel4_xdir_c_wrapper( double *p_a0, double *p_a1, double *p_a2, double *p_a3, double *p_a4, double *p_a5, double *p_a6, double *p_a7, double *p_a8, double *p_a9, double *p_a10, int x_size, int y_size, int z_size) { #ifdef OPS_GPU #pragma acc parallel deviceptr(p_a0,p_a1,p_a2,p_a3,p_a4,p_a5,p_a6,p_a7,p_a8,p_a9,p_a10) #pragma acc loop #endif for ( int n_z=0; n_z<z_size; n_z++ ){ #ifdef OPS_GPU #pragma acc loop #endif for ( int n_y=0; n_y<y_size; n_y++ ){ #ifdef OPS_GPU #pragma acc loop #endif for ( int n_x=0; n_x<x_size; n_x++ ){ advec_cell_kernel4_xdir( p_a0 + n_x*1 + n_y*xdim0_advec_cell_kernel4_xdir*1 + n_z*xdim0_advec_cell_kernel4_xdir*ydim0_advec_cell_kernel4_xdir*1, p_a1 + n_x*1 + n_y*xdim1_advec_cell_kernel4_xdir*1 + n_z*xdim1_advec_cell_kernel4_xdir*ydim1_advec_cell_kernel4_xdir*1, p_a2 + n_x*1 + n_y*xdim2_advec_cell_kernel4_xdir*1 + n_z*xdim2_advec_cell_kernel4_xdir*ydim2_advec_cell_kernel4_xdir*1, p_a3 + n_x*1 + n_y*xdim3_advec_cell_kernel4_xdir*1 + n_z*xdim3_advec_cell_kernel4_xdir*ydim3_advec_cell_kernel4_xdir*1, p_a4 + n_x*1 + n_y*xdim4_advec_cell_kernel4_xdir*1 + n_z*xdim4_advec_cell_kernel4_xdir*ydim4_advec_cell_kernel4_xdir*1, p_a5 + n_x*1 + n_y*xdim5_advec_cell_kernel4_xdir*1 + n_z*xdim5_advec_cell_kernel4_xdir*ydim5_advec_cell_kernel4_xdir*1, p_a6 + n_x*1 + n_y*xdim6_advec_cell_kernel4_xdir*1 + n_z*xdim6_advec_cell_kernel4_xdir*ydim6_advec_cell_kernel4_xdir*1, p_a7 + n_x*1 + n_y*xdim7_advec_cell_kernel4_xdir*1 + n_z*xdim7_advec_cell_kernel4_xdir*ydim7_advec_cell_kernel4_xdir*1, p_a8 + n_x*1 + n_y*xdim8_advec_cell_kernel4_xdir*1 + n_z*xdim8_advec_cell_kernel4_xdir*ydim8_advec_cell_kernel4_xdir*1, p_a9 + n_x*1 + n_y*xdim9_advec_cell_kernel4_xdir*1 + n_z*xdim9_advec_cell_kernel4_xdir*ydim9_advec_cell_kernel4_xdir*1, p_a10 + n_x*1 + n_y*xdim10_advec_cell_kernel4_xdir*1 + n_z*xdim10_advec_cell_kernel4_xdir*ydim10_advec_cell_kernel4_xdir*1 ); } } } }
// host stub function void ops_par_loop_advec_cell_kernel4_xdir( char const *name, ops_block block, int dim, int *range, ops_arg arg0, ops_arg arg1, ops_arg arg2, ops_arg arg3, ops_arg arg4, ops_arg arg5, ops_arg arg6, ops_arg arg7, ops_arg arg8, ops_arg arg9, ops_arg arg10) { // Timing double t1, t2, c1, c2; char *p_a[11]; int offs[11][3]; ops_arg args[11] = {arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10}; #ifdef CHECKPOINTING if (!ops_checkpointing_before(args, 11, range, 111)) return; #endif if (OPS_diags > 1) { ops_timing_realloc(111, "advec_cell_kernel4_xdir"); OPS_kernels[111].count++; ops_timers_core(&c2, &t2); } // compute locally allocated range for the sub-block int start[3]; int end[3]; #ifdef OPS_MPI sub_block_list sb = OPS_sub_block_list[block->index]; #endif #ifdef OPS_DEBUG ops_register_args(args, "advec_cell_kernel4_xdir"); #endif int arg_idx[3]; int arg_idx_base[3]; #ifdef OPS_MPI if (compute_ranges(args, 11, block, range, start, end, arg_idx) < 0) return; #else // OPS_MPI for (int n = 0; n < 3; n++) { start[n] = range[2 * n]; end[n] = range[2 * n + 1]; arg_idx[n] = start[n]; } #endif // OPS_MPI for (int n = 0; n < 3; n++) { arg_idx_base[n] = arg_idx[n]; } offs[0][0] = args[0].stencil->stride[0] * 1; // unit step in x dimension offs[0][1] = off3D(1, &start[0], &end[0], args[0].dat->size, args[0].stencil->stride) - offs[0][0]; offs[0][2] = off3D(2, &start[0], &end[0], args[0].dat->size, args[0].stencil->stride) - offs[0][1] - offs[0][0]; offs[1][0] = args[1].stencil->stride[0] * 1; // unit step in x dimension offs[1][1] = off3D(1, &start[0], &end[0], args[1].dat->size, args[1].stencil->stride) - offs[1][0]; offs[1][2] = off3D(2, &start[0], &end[0], args[1].dat->size, args[1].stencil->stride) - offs[1][1] - offs[1][0]; offs[2][0] = args[2].stencil->stride[0] * 1; // unit step in x dimension offs[2][1] = off3D(1, &start[0], &end[0], args[2].dat->size, args[2].stencil->stride) - offs[2][0]; offs[2][2] = off3D(2, &start[0], &end[0], args[2].dat->size, args[2].stencil->stride) - offs[2][1] - offs[2][0]; offs[3][0] = args[3].stencil->stride[0] * 1; // unit step in x dimension offs[3][1] = off3D(1, &start[0], &end[0], args[3].dat->size, args[3].stencil->stride) - offs[3][0]; offs[3][2] = off3D(2, &start[0], &end[0], args[3].dat->size, args[3].stencil->stride) - offs[3][1] - offs[3][0]; offs[4][0] = args[4].stencil->stride[0] * 1; // unit step in x dimension offs[4][1] = off3D(1, &start[0], &end[0], args[4].dat->size, args[4].stencil->stride) - offs[4][0]; offs[4][2] = off3D(2, &start[0], &end[0], args[4].dat->size, args[4].stencil->stride) - offs[4][1] - offs[4][0]; offs[5][0] = args[5].stencil->stride[0] * 1; // unit step in x dimension offs[5][1] = off3D(1, &start[0], &end[0], args[5].dat->size, args[5].stencil->stride) - offs[5][0]; offs[5][2] = off3D(2, &start[0], &end[0], args[5].dat->size, args[5].stencil->stride) - offs[5][1] - offs[5][0]; offs[6][0] = args[6].stencil->stride[0] * 1; // unit step in x dimension offs[6][1] = off3D(1, &start[0], &end[0], args[6].dat->size, args[6].stencil->stride) - offs[6][0]; offs[6][2] = off3D(2, &start[0], &end[0], args[6].dat->size, args[6].stencil->stride) - offs[6][1] - offs[6][0]; offs[7][0] = args[7].stencil->stride[0] * 1; // unit step in x dimension offs[7][1] = off3D(1, &start[0], &end[0], args[7].dat->size, args[7].stencil->stride) - offs[7][0]; offs[7][2] = off3D(2, &start[0], &end[0], args[7].dat->size, args[7].stencil->stride) - offs[7][1] - offs[7][0]; offs[8][0] = args[8].stencil->stride[0] * 1; // unit step in x dimension offs[8][1] = off3D(1, &start[0], &end[0], args[8].dat->size, args[8].stencil->stride) - offs[8][0]; offs[8][2] = off3D(2, &start[0], &end[0], args[8].dat->size, args[8].stencil->stride) - offs[8][1] - offs[8][0]; offs[9][0] = args[9].stencil->stride[0] * 1; // unit step in x dimension offs[9][1] = off3D(1, &start[0], &end[0], args[9].dat->size, args[9].stencil->stride) - offs[9][0]; offs[9][2] = off3D(2, &start[0], &end[0], args[9].dat->size, args[9].stencil->stride) - offs[9][1] - offs[9][0]; offs[10][0] = args[10].stencil->stride[0] * 1; // unit step in x dimension offs[10][1] = off3D(1, &start[0], &end[0], args[10].dat->size, args[10].stencil->stride) - offs[10][0]; offs[10][2] = off3D(2, &start[0], &end[0], args[10].dat->size, args[10].stencil->stride) - offs[10][1] - offs[10][0]; int off0_0 = offs[0][0]; int off0_1 = offs[0][1]; int off0_2 = offs[0][2]; int dat0 = (OPS_soa ? args[0].dat->type_size : args[0].dat->elem_size); int off1_0 = offs[1][0]; int off1_1 = offs[1][1]; int off1_2 = offs[1][2]; int dat1 = (OPS_soa ? args[1].dat->type_size : args[1].dat->elem_size); int off2_0 = offs[2][0]; int off2_1 = offs[2][1]; int off2_2 = offs[2][2]; int dat2 = (OPS_soa ? args[2].dat->type_size : args[2].dat->elem_size); int off3_0 = offs[3][0]; int off3_1 = offs[3][1]; int off3_2 = offs[3][2]; int dat3 = (OPS_soa ? args[3].dat->type_size : args[3].dat->elem_size); int off4_0 = offs[4][0]; int off4_1 = offs[4][1]; int off4_2 = offs[4][2]; int dat4 = (OPS_soa ? args[4].dat->type_size : args[4].dat->elem_size); int off5_0 = offs[5][0]; int off5_1 = offs[5][1]; int off5_2 = offs[5][2]; int dat5 = (OPS_soa ? args[5].dat->type_size : args[5].dat->elem_size); int off6_0 = offs[6][0]; int off6_1 = offs[6][1]; int off6_2 = offs[6][2]; int dat6 = (OPS_soa ? args[6].dat->type_size : args[6].dat->elem_size); int off7_0 = offs[7][0]; int off7_1 = offs[7][1]; int off7_2 = offs[7][2]; int dat7 = (OPS_soa ? args[7].dat->type_size : args[7].dat->elem_size); int off8_0 = offs[8][0]; int off8_1 = offs[8][1]; int off8_2 = offs[8][2]; int dat8 = (OPS_soa ? args[8].dat->type_size : args[8].dat->elem_size); int off9_0 = offs[9][0]; int off9_1 = offs[9][1]; int off9_2 = offs[9][2]; int dat9 = (OPS_soa ? args[9].dat->type_size : args[9].dat->elem_size); int off10_0 = offs[10][0]; int off10_1 = offs[10][1]; int off10_2 = offs[10][2]; int dat10 = (OPS_soa ? args[10].dat->type_size : args[10].dat->elem_size); // set up initial pointers and exchange halos if necessary int base0 = args[0].dat->base_offset + (OPS_soa ? args[0].dat->type_size : args[0].dat->elem_size) * start[0] * args[0].stencil->stride[0]; base0 = base0 + (OPS_soa ? args[0].dat->type_size : args[0].dat->elem_size) * args[0].dat->size[0] * start[1] * args[0].stencil->stride[1]; base0 = base0 + (OPS_soa ? args[0].dat->type_size : args[0].dat->elem_size) * args[0].dat->size[0] * args[0].dat->size[1] * start[2] * args[0].stencil->stride[2]; p_a[0] = (char *)args[0].data + base0; int base1 = args[1].dat->base_offset + (OPS_soa ? args[1].dat->type_size : args[1].dat->elem_size) * start[0] * args[1].stencil->stride[0]; base1 = base1 + (OPS_soa ? args[1].dat->type_size : args[1].dat->elem_size) * args[1].dat->size[0] * start[1] * args[1].stencil->stride[1]; base1 = base1 + (OPS_soa ? args[1].dat->type_size : args[1].dat->elem_size) * args[1].dat->size[0] * args[1].dat->size[1] * start[2] * args[1].stencil->stride[2]; p_a[1] = (char *)args[1].data + base1; int base2 = args[2].dat->base_offset + (OPS_soa ? args[2].dat->type_size : args[2].dat->elem_size) * start[0] * args[2].stencil->stride[0]; base2 = base2 + (OPS_soa ? args[2].dat->type_size : args[2].dat->elem_size) * args[2].dat->size[0] * start[1] * args[2].stencil->stride[1]; base2 = base2 + (OPS_soa ? args[2].dat->type_size : args[2].dat->elem_size) * args[2].dat->size[0] * args[2].dat->size[1] * start[2] * args[2].stencil->stride[2]; p_a[2] = (char *)args[2].data + base2; int base3 = args[3].dat->base_offset + (OPS_soa ? args[3].dat->type_size : args[3].dat->elem_size) * start[0] * args[3].stencil->stride[0]; base3 = base3 + (OPS_soa ? args[3].dat->type_size : args[3].dat->elem_size) * args[3].dat->size[0] * start[1] * args[3].stencil->stride[1]; base3 = base3 + (OPS_soa ? args[3].dat->type_size : args[3].dat->elem_size) * args[3].dat->size[0] * args[3].dat->size[1] * start[2] * args[3].stencil->stride[2]; p_a[3] = (char *)args[3].data + base3; int base4 = args[4].dat->base_offset + (OPS_soa ? args[4].dat->type_size : args[4].dat->elem_size) * start[0] * args[4].stencil->stride[0]; base4 = base4 + (OPS_soa ? args[4].dat->type_size : args[4].dat->elem_size) * args[4].dat->size[0] * start[1] * args[4].stencil->stride[1]; base4 = base4 + (OPS_soa ? args[4].dat->type_size : args[4].dat->elem_size) * args[4].dat->size[0] * args[4].dat->size[1] * start[2] * args[4].stencil->stride[2]; p_a[4] = (char *)args[4].data + base4; int base5 = args[5].dat->base_offset + (OPS_soa ? args[5].dat->type_size : args[5].dat->elem_size) * start[0] * args[5].stencil->stride[0]; base5 = base5 + (OPS_soa ? args[5].dat->type_size : args[5].dat->elem_size) * args[5].dat->size[0] * start[1] * args[5].stencil->stride[1]; base5 = base5 + (OPS_soa ? args[5].dat->type_size : args[5].dat->elem_size) * args[5].dat->size[0] * args[5].dat->size[1] * start[2] * args[5].stencil->stride[2]; p_a[5] = (char *)args[5].data + base5; int base6 = args[6].dat->base_offset + (OPS_soa ? args[6].dat->type_size : args[6].dat->elem_size) * start[0] * args[6].stencil->stride[0]; base6 = base6 + (OPS_soa ? args[6].dat->type_size : args[6].dat->elem_size) * args[6].dat->size[0] * start[1] * args[6].stencil->stride[1]; base6 = base6 + (OPS_soa ? args[6].dat->type_size : args[6].dat->elem_size) * args[6].dat->size[0] * args[6].dat->size[1] * start[2] * args[6].stencil->stride[2]; p_a[6] = (char *)args[6].data + base6; int base7 = args[7].dat->base_offset + (OPS_soa ? args[7].dat->type_size : args[7].dat->elem_size) * start[0] * args[7].stencil->stride[0]; base7 = base7 + (OPS_soa ? args[7].dat->type_size : args[7].dat->elem_size) * args[7].dat->size[0] * start[1] * args[7].stencil->stride[1]; base7 = base7 + (OPS_soa ? args[7].dat->type_size : args[7].dat->elem_size) * args[7].dat->size[0] * args[7].dat->size[1] * start[2] * args[7].stencil->stride[2]; p_a[7] = (char *)args[7].data + base7; int base8 = args[8].dat->base_offset + (OPS_soa ? args[8].dat->type_size : args[8].dat->elem_size) * start[0] * args[8].stencil->stride[0]; base8 = base8 + (OPS_soa ? args[8].dat->type_size : args[8].dat->elem_size) * args[8].dat->size[0] * start[1] * args[8].stencil->stride[1]; base8 = base8 + (OPS_soa ? args[8].dat->type_size : args[8].dat->elem_size) * args[8].dat->size[0] * args[8].dat->size[1] * start[2] * args[8].stencil->stride[2]; p_a[8] = (char *)args[8].data + base8; int base9 = args[9].dat->base_offset + (OPS_soa ? args[9].dat->type_size : args[9].dat->elem_size) * start[0] * args[9].stencil->stride[0]; base9 = base9 + (OPS_soa ? args[9].dat->type_size : args[9].dat->elem_size) * args[9].dat->size[0] * start[1] * args[9].stencil->stride[1]; base9 = base9 + (OPS_soa ? args[9].dat->type_size : args[9].dat->elem_size) * args[9].dat->size[0] * args[9].dat->size[1] * start[2] * args[9].stencil->stride[2]; p_a[9] = (char *)args[9].data + base9; int base10 = args[10].dat->base_offset + (OPS_soa ? args[10].dat->type_size : args[10].dat->elem_size) * start[0] * args[10].stencil->stride[0]; base10 = base10 + (OPS_soa ? args[10].dat->type_size : args[10].dat->elem_size) * args[10].dat->size[0] * start[1] * args[10].stencil->stride[1]; base10 = base10 + (OPS_soa ? args[10].dat->type_size : args[10].dat->elem_size) * args[10].dat->size[0] * args[10].dat->size[1] * start[2] * args[10].stencil->stride[2]; p_a[10] = (char *)args[10].data + base10; // initialize global variable with the dimension of dats xdim0 = args[0].dat->size[0]; ydim0 = args[0].dat->size[1]; xdim1 = args[1].dat->size[0]; ydim1 = args[1].dat->size[1]; xdim2 = args[2].dat->size[0]; ydim2 = args[2].dat->size[1]; xdim3 = args[3].dat->size[0]; ydim3 = args[3].dat->size[1]; xdim4 = args[4].dat->size[0]; ydim4 = args[4].dat->size[1]; xdim5 = args[5].dat->size[0]; ydim5 = args[5].dat->size[1]; xdim6 = args[6].dat->size[0]; ydim6 = args[6].dat->size[1]; xdim7 = args[7].dat->size[0]; ydim7 = args[7].dat->size[1]; xdim8 = args[8].dat->size[0]; ydim8 = args[8].dat->size[1]; xdim9 = args[9].dat->size[0]; ydim9 = args[9].dat->size[1]; xdim10 = args[10].dat->size[0]; ydim10 = args[10].dat->size[1]; // Halo Exchanges ops_H_D_exchanges_host(args, 11); ops_halo_exchanges(args, 11, range); ops_H_D_exchanges_host(args, 11); if (OPS_diags > 1) { ops_timers_core(&c1, &t1); OPS_kernels[111].mpi_time += t1 - t2; } int n_x; for (int n_z = start[2]; n_z < end[2]; n_z++) { for (int n_y = start[1]; n_y < end[1]; n_y++) { #pragma novector for (n_x = start[0]; n_x < start[0] + ((end[0] - start[0]) / SIMD_VEC) * SIMD_VEC; n_x += SIMD_VEC) { // call kernel function, passing in pointers to data -vectorised #pragma simd for (int i = 0; i < SIMD_VEC; i++) { advec_cell_kernel4_xdir( (double *)p_a[0] + i * 1 * 1, (double *)p_a[1] + i * 1 * 1, (double *)p_a[2] + i * 1 * 1, (double *)p_a[3] + i * 1 * 1, (double *)p_a[4] + i * 1 * 1, (double *)p_a[5] + i * 1 * 1, (double *)p_a[6] + i * 1 * 1, (double *)p_a[7] + i * 1 * 1, (double *)p_a[8] + i * 1 * 1, (double *)p_a[9] + i * 1 * 1, (double *)p_a[10] + i * 1 * 1); } // shift pointers to data x direction p_a[0] = p_a[0] + (dat0 * off0_0) * SIMD_VEC; p_a[1] = p_a[1] + (dat1 * off1_0) * SIMD_VEC; p_a[2] = p_a[2] + (dat2 * off2_0) * SIMD_VEC; p_a[3] = p_a[3] + (dat3 * off3_0) * SIMD_VEC; p_a[4] = p_a[4] + (dat4 * off4_0) * SIMD_VEC; p_a[5] = p_a[5] + (dat5 * off5_0) * SIMD_VEC; p_a[6] = p_a[6] + (dat6 * off6_0) * SIMD_VEC; p_a[7] = p_a[7] + (dat7 * off7_0) * SIMD_VEC; p_a[8] = p_a[8] + (dat8 * off8_0) * SIMD_VEC; p_a[9] = p_a[9] + (dat9 * off9_0) * SIMD_VEC; p_a[10] = p_a[10] + (dat10 * off10_0) * SIMD_VEC; } for (int n_x = start[0] + ((end[0] - start[0]) / SIMD_VEC) * SIMD_VEC; n_x < end[0]; n_x++) { // call kernel function, passing in pointers to data - remainder advec_cell_kernel4_xdir( (double *)p_a[0], (double *)p_a[1], (double *)p_a[2], (double *)p_a[3], (double *)p_a[4], (double *)p_a[5], (double *)p_a[6], (double *)p_a[7], (double *)p_a[8], (double *)p_a[9], (double *)p_a[10]); // shift pointers to data x direction p_a[0] = p_a[0] + (dat0 * off0_0); p_a[1] = p_a[1] + (dat1 * off1_0); p_a[2] = p_a[2] + (dat2 * off2_0); p_a[3] = p_a[3] + (dat3 * off3_0); p_a[4] = p_a[4] + (dat4 * off4_0); p_a[5] = p_a[5] + (dat5 * off5_0); p_a[6] = p_a[6] + (dat6 * off6_0); p_a[7] = p_a[7] + (dat7 * off7_0); p_a[8] = p_a[8] + (dat8 * off8_0); p_a[9] = p_a[9] + (dat9 * off9_0); p_a[10] = p_a[10] + (dat10 * off10_0); } // shift pointers to data y direction p_a[0] = p_a[0] + (dat0 * off0_1); p_a[1] = p_a[1] + (dat1 * off1_1); p_a[2] = p_a[2] + (dat2 * off2_1); p_a[3] = p_a[3] + (dat3 * off3_1); p_a[4] = p_a[4] + (dat4 * off4_1); p_a[5] = p_a[5] + (dat5 * off5_1); p_a[6] = p_a[6] + (dat6 * off6_1); p_a[7] = p_a[7] + (dat7 * off7_1); p_a[8] = p_a[8] + (dat8 * off8_1); p_a[9] = p_a[9] + (dat9 * off9_1); p_a[10] = p_a[10] + (dat10 * off10_1); } // shift pointers to data z direction p_a[0] = p_a[0] + (dat0 * off0_2); p_a[1] = p_a[1] + (dat1 * off1_2); p_a[2] = p_a[2] + (dat2 * off2_2); p_a[3] = p_a[3] + (dat3 * off3_2); p_a[4] = p_a[4] + (dat4 * off4_2); p_a[5] = p_a[5] + (dat5 * off5_2); p_a[6] = p_a[6] + (dat6 * off6_2); p_a[7] = p_a[7] + (dat7 * off7_2); p_a[8] = p_a[8] + (dat8 * off8_2); p_a[9] = p_a[9] + (dat9 * off9_2); p_a[10] = p_a[10] + (dat10 * off10_2); } if (OPS_diags > 1) { ops_timers_core(&c2, &t2); OPS_kernels[111].time += t2 - t1; } ops_set_dirtybit_host(args, 11); ops_set_halo_dirtybit3(&args[0], range); ops_set_halo_dirtybit3(&args[1], range); ops_set_halo_dirtybit3(&args[6], range); ops_set_halo_dirtybit3(&args[7], range); ops_set_halo_dirtybit3(&args[8], range); ops_set_halo_dirtybit3(&args[9], range); if (OPS_diags > 1) { // Update kernel record ops_timers_core(&c1, &t1); OPS_kernels[111].mpi_time += t1 - t2; OPS_kernels[111].transfer += ops_compute_transfer(dim, start, end, &arg0); OPS_kernels[111].transfer += ops_compute_transfer(dim, start, end, &arg1); OPS_kernels[111].transfer += ops_compute_transfer(dim, start, end, &arg2); OPS_kernels[111].transfer += ops_compute_transfer(dim, start, end, &arg3); OPS_kernels[111].transfer += ops_compute_transfer(dim, start, end, &arg4); OPS_kernels[111].transfer += ops_compute_transfer(dim, start, end, &arg5); OPS_kernels[111].transfer += ops_compute_transfer(dim, start, end, &arg6); OPS_kernels[111].transfer += ops_compute_transfer(dim, start, end, &arg7); OPS_kernels[111].transfer += ops_compute_transfer(dim, start, end, &arg8); OPS_kernels[111].transfer += ops_compute_transfer(dim, start, end, &arg9); OPS_kernels[111].transfer += ops_compute_transfer(dim, start, end, &arg10); } }
// host stub function void ops_par_loop_advec_cell_kernel4_xdir( char const *name, ops_block block, int dim, int *range, ops_arg arg0, ops_arg arg1, ops_arg arg2, ops_arg arg3, ops_arg arg4, ops_arg arg5, ops_arg arg6, ops_arg arg7, ops_arg arg8, ops_arg arg9, ops_arg arg10) { // Timing double t1, t2, c1, c2; int offs[11][3]; ops_arg args[11] = {arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10}; #ifdef CHECKPOINTING if (!ops_checkpointing_before(args, 11, range, 111)) return; #endif if (OPS_diags > 1) { ops_timing_realloc(111, "advec_cell_kernel4_xdir"); OPS_kernels[111].count++; ops_timers_core(&c1, &t1); } #ifdef OPS_MPI sub_block_list sb = OPS_sub_block_list[block->index]; #endif // compute locally allocated range for the sub-block int start[3]; int end[3]; int arg_idx[3]; #ifdef OPS_MPI if (!sb->owned) return; for (int n = 0; n < 3; n++) { start[n] = sb->decomp_disp[n]; end[n] = sb->decomp_disp[n] + sb->decomp_size[n]; if (start[n] >= range[2 * n]) { start[n] = 0; } else { start[n] = range[2 * n] - start[n]; } if (sb->id_m[n] == MPI_PROC_NULL && range[2 * n] < 0) start[n] = range[2 * n]; if (end[n] >= range[2 * n + 1]) { end[n] = range[2 * n + 1] - sb->decomp_disp[n]; } else { end[n] = sb->decomp_size[n]; } if (sb->id_p[n] == MPI_PROC_NULL && (range[2 * n + 1] > sb->decomp_disp[n] + sb->decomp_size[n])) end[n] += (range[2 * n + 1] - sb->decomp_disp[n] - sb->decomp_size[n]); if (end[n] < start[n]) end[n] = start[n]; } #else for (int n = 0; n < 3; n++) { start[n] = range[2 * n]; end[n] = range[2 * n + 1]; } #endif #ifdef OPS_DEBUG ops_register_args(args, "advec_cell_kernel4_xdir"); #endif offs[0][0] = args[0].stencil->stride[0] * 1; // unit step in x dimension offs[0][1] = off3D(1, &start[0], &end[0], args[0].dat->size, args[0].stencil->stride) - offs[0][0]; offs[0][2] = off3D(2, &start[0], &end[0], args[0].dat->size, args[0].stencil->stride) - offs[0][1] - offs[0][0]; offs[1][0] = args[1].stencil->stride[0] * 1; // unit step in x dimension offs[1][1] = off3D(1, &start[0], &end[0], args[1].dat->size, args[1].stencil->stride) - offs[1][0]; offs[1][2] = off3D(2, &start[0], &end[0], args[1].dat->size, args[1].stencil->stride) - offs[1][1] - offs[1][0]; offs[2][0] = args[2].stencil->stride[0] * 1; // unit step in x dimension offs[2][1] = off3D(1, &start[0], &end[0], args[2].dat->size, args[2].stencil->stride) - offs[2][0]; offs[2][2] = off3D(2, &start[0], &end[0], args[2].dat->size, args[2].stencil->stride) - offs[2][1] - offs[2][0]; offs[3][0] = args[3].stencil->stride[0] * 1; // unit step in x dimension offs[3][1] = off3D(1, &start[0], &end[0], args[3].dat->size, args[3].stencil->stride) - offs[3][0]; offs[3][2] = off3D(2, &start[0], &end[0], args[3].dat->size, args[3].stencil->stride) - offs[3][1] - offs[3][0]; offs[4][0] = args[4].stencil->stride[0] * 1; // unit step in x dimension offs[4][1] = off3D(1, &start[0], &end[0], args[4].dat->size, args[4].stencil->stride) - offs[4][0]; offs[4][2] = off3D(2, &start[0], &end[0], args[4].dat->size, args[4].stencil->stride) - offs[4][1] - offs[4][0]; offs[5][0] = args[5].stencil->stride[0] * 1; // unit step in x dimension offs[5][1] = off3D(1, &start[0], &end[0], args[5].dat->size, args[5].stencil->stride) - offs[5][0]; offs[5][2] = off3D(2, &start[0], &end[0], args[5].dat->size, args[5].stencil->stride) - offs[5][1] - offs[5][0]; offs[6][0] = args[6].stencil->stride[0] * 1; // unit step in x dimension offs[6][1] = off3D(1, &start[0], &end[0], args[6].dat->size, args[6].stencil->stride) - offs[6][0]; offs[6][2] = off3D(2, &start[0], &end[0], args[6].dat->size, args[6].stencil->stride) - offs[6][1] - offs[6][0]; offs[7][0] = args[7].stencil->stride[0] * 1; // unit step in x dimension offs[7][1] = off3D(1, &start[0], &end[0], args[7].dat->size, args[7].stencil->stride) - offs[7][0]; offs[7][2] = off3D(2, &start[0], &end[0], args[7].dat->size, args[7].stencil->stride) - offs[7][1] - offs[7][0]; offs[8][0] = args[8].stencil->stride[0] * 1; // unit step in x dimension offs[8][1] = off3D(1, &start[0], &end[0], args[8].dat->size, args[8].stencil->stride) - offs[8][0]; offs[8][2] = off3D(2, &start[0], &end[0], args[8].dat->size, args[8].stencil->stride) - offs[8][1] - offs[8][0]; offs[9][0] = args[9].stencil->stride[0] * 1; // unit step in x dimension offs[9][1] = off3D(1, &start[0], &end[0], args[9].dat->size, args[9].stencil->stride) - offs[9][0]; offs[9][2] = off3D(2, &start[0], &end[0], args[9].dat->size, args[9].stencil->stride) - offs[9][1] - offs[9][0]; offs[10][0] = args[10].stencil->stride[0] * 1; // unit step in x dimension offs[10][1] = off3D(1, &start[0], &end[0], args[10].dat->size, args[10].stencil->stride) - offs[10][0]; offs[10][2] = off3D(2, &start[0], &end[0], args[10].dat->size, args[10].stencil->stride) - offs[10][1] - offs[10][0]; int off0_0 = offs[0][0]; int off0_1 = offs[0][1]; int off0_2 = offs[0][2]; int dat0 = (OPS_soa ? args[0].dat->type_size : args[0].dat->elem_size); int off1_0 = offs[1][0]; int off1_1 = offs[1][1]; int off1_2 = offs[1][2]; int dat1 = (OPS_soa ? args[1].dat->type_size : args[1].dat->elem_size); int off2_0 = offs[2][0]; int off2_1 = offs[2][1]; int off2_2 = offs[2][2]; int dat2 = (OPS_soa ? args[2].dat->type_size : args[2].dat->elem_size); int off3_0 = offs[3][0]; int off3_1 = offs[3][1]; int off3_2 = offs[3][2]; int dat3 = (OPS_soa ? args[3].dat->type_size : args[3].dat->elem_size); int off4_0 = offs[4][0]; int off4_1 = offs[4][1]; int off4_2 = offs[4][2]; int dat4 = (OPS_soa ? args[4].dat->type_size : args[4].dat->elem_size); int off5_0 = offs[5][0]; int off5_1 = offs[5][1]; int off5_2 = offs[5][2]; int dat5 = (OPS_soa ? args[5].dat->type_size : args[5].dat->elem_size); int off6_0 = offs[6][0]; int off6_1 = offs[6][1]; int off6_2 = offs[6][2]; int dat6 = (OPS_soa ? args[6].dat->type_size : args[6].dat->elem_size); int off7_0 = offs[7][0]; int off7_1 = offs[7][1]; int off7_2 = offs[7][2]; int dat7 = (OPS_soa ? args[7].dat->type_size : args[7].dat->elem_size); int off8_0 = offs[8][0]; int off8_1 = offs[8][1]; int off8_2 = offs[8][2]; int dat8 = (OPS_soa ? args[8].dat->type_size : args[8].dat->elem_size); int off9_0 = offs[9][0]; int off9_1 = offs[9][1]; int off9_2 = offs[9][2]; int dat9 = (OPS_soa ? args[9].dat->type_size : args[9].dat->elem_size); int off10_0 = offs[10][0]; int off10_1 = offs[10][1]; int off10_2 = offs[10][2]; int dat10 = (OPS_soa ? args[10].dat->type_size : args[10].dat->elem_size); // Halo Exchanges ops_H_D_exchanges_host(args, 11); ops_halo_exchanges(args, 11, range); ops_H_D_exchanges_host(args, 11); #ifdef _OPENMP int nthreads = omp_get_max_threads(); #else int nthreads = 1; #endif xdim0 = args[0].dat->size[0]; ydim0 = args[0].dat->size[1]; xdim1 = args[1].dat->size[0]; ydim1 = args[1].dat->size[1]; xdim2 = args[2].dat->size[0]; ydim2 = args[2].dat->size[1]; xdim3 = args[3].dat->size[0]; ydim3 = args[3].dat->size[1]; xdim4 = args[4].dat->size[0]; ydim4 = args[4].dat->size[1]; xdim5 = args[5].dat->size[0]; ydim5 = args[5].dat->size[1]; xdim6 = args[6].dat->size[0]; ydim6 = args[6].dat->size[1]; xdim7 = args[7].dat->size[0]; ydim7 = args[7].dat->size[1]; xdim8 = args[8].dat->size[0]; ydim8 = args[8].dat->size[1]; xdim9 = args[9].dat->size[0]; ydim9 = args[9].dat->size[1]; xdim10 = args[10].dat->size[0]; ydim10 = args[10].dat->size[1]; if (OPS_diags > 1) { ops_timers_core(&c2, &t2); OPS_kernels[111].mpi_time += t2 - t1; } #pragma omp parallel for for (int thr = 0; thr < nthreads; thr++) { int z_size = end[2] - start[2]; char *p_a[11]; int start_i = start[2] + ((z_size - 1) / nthreads + 1) * thr; int finish_i = start[2] + MIN(((z_size - 1) / nthreads + 1) * (thr + 1), z_size); // get address per thread int start0 = start[0]; int start1 = start[1]; int start2 = start_i; // set up initial pointers int d_m[OPS_MAX_DIM]; #ifdef OPS_MPI for (int d = 0; d < dim; d++) d_m[d] = args[0].dat->d_m[d] + OPS_sub_dat_list[args[0].dat->index]->d_im[d]; #else for (int d = 0; d < dim; d++) d_m[d] = args[0].dat->d_m[d]; #endif int base0 = dat0 * 1 * (start0 * args[0].stencil->stride[0] - args[0].dat->base[0] - d_m[0]); base0 = base0 + dat0 * args[0].dat->size[0] * (start1 * args[0].stencil->stride[1] - args[0].dat->base[1] - d_m[1]); base0 = base0 + dat0 * args[0].dat->size[0] * args[0].dat->size[1] * (start2 * args[0].stencil->stride[2] - args[0].dat->base[2] - d_m[2]); p_a[0] = (char *)args[0].data + base0; #ifdef OPS_MPI for (int d = 0; d < dim; d++) d_m[d] = args[1].dat->d_m[d] + OPS_sub_dat_list[args[1].dat->index]->d_im[d]; #else for (int d = 0; d < dim; d++) d_m[d] = args[1].dat->d_m[d]; #endif int base1 = dat1 * 1 * (start0 * args[1].stencil->stride[0] - args[1].dat->base[0] - d_m[0]); base1 = base1 + dat1 * args[1].dat->size[0] * (start1 * args[1].stencil->stride[1] - args[1].dat->base[1] - d_m[1]); base1 = base1 + dat1 * args[1].dat->size[0] * args[1].dat->size[1] * (start2 * args[1].stencil->stride[2] - args[1].dat->base[2] - d_m[2]); p_a[1] = (char *)args[1].data + base1; #ifdef OPS_MPI for (int d = 0; d < dim; d++) d_m[d] = args[2].dat->d_m[d] + OPS_sub_dat_list[args[2].dat->index]->d_im[d]; #else for (int d = 0; d < dim; d++) d_m[d] = args[2].dat->d_m[d]; #endif int base2 = dat2 * 1 * (start0 * args[2].stencil->stride[0] - args[2].dat->base[0] - d_m[0]); base2 = base2 + dat2 * args[2].dat->size[0] * (start1 * args[2].stencil->stride[1] - args[2].dat->base[1] - d_m[1]); base2 = base2 + dat2 * args[2].dat->size[0] * args[2].dat->size[1] * (start2 * args[2].stencil->stride[2] - args[2].dat->base[2] - d_m[2]); p_a[2] = (char *)args[2].data + base2; #ifdef OPS_MPI for (int d = 0; d < dim; d++) d_m[d] = args[3].dat->d_m[d] + OPS_sub_dat_list[args[3].dat->index]->d_im[d]; #else for (int d = 0; d < dim; d++) d_m[d] = args[3].dat->d_m[d]; #endif int base3 = dat3 * 1 * (start0 * args[3].stencil->stride[0] - args[3].dat->base[0] - d_m[0]); base3 = base3 + dat3 * args[3].dat->size[0] * (start1 * args[3].stencil->stride[1] - args[3].dat->base[1] - d_m[1]); base3 = base3 + dat3 * args[3].dat->size[0] * args[3].dat->size[1] * (start2 * args[3].stencil->stride[2] - args[3].dat->base[2] - d_m[2]); p_a[3] = (char *)args[3].data + base3; #ifdef OPS_MPI for (int d = 0; d < dim; d++) d_m[d] = args[4].dat->d_m[d] + OPS_sub_dat_list[args[4].dat->index]->d_im[d]; #else for (int d = 0; d < dim; d++) d_m[d] = args[4].dat->d_m[d]; #endif int base4 = dat4 * 1 * (start0 * args[4].stencil->stride[0] - args[4].dat->base[0] - d_m[0]); base4 = base4 + dat4 * args[4].dat->size[0] * (start1 * args[4].stencil->stride[1] - args[4].dat->base[1] - d_m[1]); base4 = base4 + dat4 * args[4].dat->size[0] * args[4].dat->size[1] * (start2 * args[4].stencil->stride[2] - args[4].dat->base[2] - d_m[2]); p_a[4] = (char *)args[4].data + base4; #ifdef OPS_MPI for (int d = 0; d < dim; d++) d_m[d] = args[5].dat->d_m[d] + OPS_sub_dat_list[args[5].dat->index]->d_im[d]; #else for (int d = 0; d < dim; d++) d_m[d] = args[5].dat->d_m[d]; #endif int base5 = dat5 * 1 * (start0 * args[5].stencil->stride[0] - args[5].dat->base[0] - d_m[0]); base5 = base5 + dat5 * args[5].dat->size[0] * (start1 * args[5].stencil->stride[1] - args[5].dat->base[1] - d_m[1]); base5 = base5 + dat5 * args[5].dat->size[0] * args[5].dat->size[1] * (start2 * args[5].stencil->stride[2] - args[5].dat->base[2] - d_m[2]); p_a[5] = (char *)args[5].data + base5; #ifdef OPS_MPI for (int d = 0; d < dim; d++) d_m[d] = args[6].dat->d_m[d] + OPS_sub_dat_list[args[6].dat->index]->d_im[d]; #else for (int d = 0; d < dim; d++) d_m[d] = args[6].dat->d_m[d]; #endif int base6 = dat6 * 1 * (start0 * args[6].stencil->stride[0] - args[6].dat->base[0] - d_m[0]); base6 = base6 + dat6 * args[6].dat->size[0] * (start1 * args[6].stencil->stride[1] - args[6].dat->base[1] - d_m[1]); base6 = base6 + dat6 * args[6].dat->size[0] * args[6].dat->size[1] * (start2 * args[6].stencil->stride[2] - args[6].dat->base[2] - d_m[2]); p_a[6] = (char *)args[6].data + base6; #ifdef OPS_MPI for (int d = 0; d < dim; d++) d_m[d] = args[7].dat->d_m[d] + OPS_sub_dat_list[args[7].dat->index]->d_im[d]; #else for (int d = 0; d < dim; d++) d_m[d] = args[7].dat->d_m[d]; #endif int base7 = dat7 * 1 * (start0 * args[7].stencil->stride[0] - args[7].dat->base[0] - d_m[0]); base7 = base7 + dat7 * args[7].dat->size[0] * (start1 * args[7].stencil->stride[1] - args[7].dat->base[1] - d_m[1]); base7 = base7 + dat7 * args[7].dat->size[0] * args[7].dat->size[1] * (start2 * args[7].stencil->stride[2] - args[7].dat->base[2] - d_m[2]); p_a[7] = (char *)args[7].data + base7; #ifdef OPS_MPI for (int d = 0; d < dim; d++) d_m[d] = args[8].dat->d_m[d] + OPS_sub_dat_list[args[8].dat->index]->d_im[d]; #else for (int d = 0; d < dim; d++) d_m[d] = args[8].dat->d_m[d]; #endif int base8 = dat8 * 1 * (start0 * args[8].stencil->stride[0] - args[8].dat->base[0] - d_m[0]); base8 = base8 + dat8 * args[8].dat->size[0] * (start1 * args[8].stencil->stride[1] - args[8].dat->base[1] - d_m[1]); base8 = base8 + dat8 * args[8].dat->size[0] * args[8].dat->size[1] * (start2 * args[8].stencil->stride[2] - args[8].dat->base[2] - d_m[2]); p_a[8] = (char *)args[8].data + base8; #ifdef OPS_MPI for (int d = 0; d < dim; d++) d_m[d] = args[9].dat->d_m[d] + OPS_sub_dat_list[args[9].dat->index]->d_im[d]; #else for (int d = 0; d < dim; d++) d_m[d] = args[9].dat->d_m[d]; #endif int base9 = dat9 * 1 * (start0 * args[9].stencil->stride[0] - args[9].dat->base[0] - d_m[0]); base9 = base9 + dat9 * args[9].dat->size[0] * (start1 * args[9].stencil->stride[1] - args[9].dat->base[1] - d_m[1]); base9 = base9 + dat9 * args[9].dat->size[0] * args[9].dat->size[1] * (start2 * args[9].stencil->stride[2] - args[9].dat->base[2] - d_m[2]); p_a[9] = (char *)args[9].data + base9; #ifdef OPS_MPI for (int d = 0; d < dim; d++) d_m[d] = args[10].dat->d_m[d] + OPS_sub_dat_list[args[10].dat->index]->d_im[d]; #else for (int d = 0; d < dim; d++) d_m[d] = args[10].dat->d_m[d]; #endif int base10 = dat10 * 1 * (start0 * args[10].stencil->stride[0] - args[10].dat->base[0] - d_m[0]); base10 = base10 + dat10 * args[10].dat->size[0] * (start1 * args[10].stencil->stride[1] - args[10].dat->base[1] - d_m[1]); base10 = base10 + dat10 * args[10].dat->size[0] * args[10].dat->size[1] * (start2 * args[10].stencil->stride[2] - args[10].dat->base[2] - d_m[2]); p_a[10] = (char *)args[10].data + base10; for (int n_z = start_i; n_z < finish_i; n_z++) { for (int n_y = start[1]; n_y < end[1]; n_y++) { for (int n_x = start[0]; n_x < start[0] + (end[0] - start[0]) / SIMD_VEC; n_x++) { // call kernel function, passing in pointers to data -vectorised #pragma simd for (int i = 0; i < SIMD_VEC; i++) { advec_cell_kernel4_xdir( (double *)p_a[0] + i * 1 * 1, (double *)p_a[1] + i * 1 * 1, (const double *)p_a[2] + i * 1 * 1, (const double *)p_a[3] + i * 1 * 1, (const double *)p_a[4] + i * 1 * 1, (const double *)p_a[5] + i * 1 * 1, (double *)p_a[6] + i * 1 * 1, (double *)p_a[7] + i * 1 * 1, (double *)p_a[8] + i * 1 * 1, (double *)p_a[9] + i * 1 * 1, (const double *)p_a[10] + i * 1 * 1); } // shift pointers to data x direction p_a[0] = p_a[0] + (dat0 * off0_0) * SIMD_VEC; p_a[1] = p_a[1] + (dat1 * off1_0) * SIMD_VEC; p_a[2] = p_a[2] + (dat2 * off2_0) * SIMD_VEC; p_a[3] = p_a[3] + (dat3 * off3_0) * SIMD_VEC; p_a[4] = p_a[4] + (dat4 * off4_0) * SIMD_VEC; p_a[5] = p_a[5] + (dat5 * off5_0) * SIMD_VEC; p_a[6] = p_a[6] + (dat6 * off6_0) * SIMD_VEC; p_a[7] = p_a[7] + (dat7 * off7_0) * SIMD_VEC; p_a[8] = p_a[8] + (dat8 * off8_0) * SIMD_VEC; p_a[9] = p_a[9] + (dat9 * off9_0) * SIMD_VEC; p_a[10] = p_a[10] + (dat10 * off10_0) * SIMD_VEC; } for (int n_x = start[0] + ((end[0] - start[0]) / SIMD_VEC) * SIMD_VEC; n_x < end[0]; n_x++) { // call kernel function, passing in pointers to data - remainder advec_cell_kernel4_xdir( (double *)p_a[0], (double *)p_a[1], (const double *)p_a[2], (const double *)p_a[3], (const double *)p_a[4], (const double *)p_a[5], (double *)p_a[6], (double *)p_a[7], (double *)p_a[8], (double *)p_a[9], (const double *)p_a[10]); // shift pointers to data x direction p_a[0] = p_a[0] + (dat0 * off0_0); p_a[1] = p_a[1] + (dat1 * off1_0); p_a[2] = p_a[2] + (dat2 * off2_0); p_a[3] = p_a[3] + (dat3 * off3_0); p_a[4] = p_a[4] + (dat4 * off4_0); p_a[5] = p_a[5] + (dat5 * off5_0); p_a[6] = p_a[6] + (dat6 * off6_0); p_a[7] = p_a[7] + (dat7 * off7_0); p_a[8] = p_a[8] + (dat8 * off8_0); p_a[9] = p_a[9] + (dat9 * off9_0); p_a[10] = p_a[10] + (dat10 * off10_0); } // shift pointers to data y direction p_a[0] = p_a[0] + (dat0 * off0_1); p_a[1] = p_a[1] + (dat1 * off1_1); p_a[2] = p_a[2] + (dat2 * off2_1); p_a[3] = p_a[3] + (dat3 * off3_1); p_a[4] = p_a[4] + (dat4 * off4_1); p_a[5] = p_a[5] + (dat5 * off5_1); p_a[6] = p_a[6] + (dat6 * off6_1); p_a[7] = p_a[7] + (dat7 * off7_1); p_a[8] = p_a[8] + (dat8 * off8_1); p_a[9] = p_a[9] + (dat9 * off9_1); p_a[10] = p_a[10] + (dat10 * off10_1); } // shift pointers to data z direction p_a[0] = p_a[0] + (dat0 * off0_2); p_a[1] = p_a[1] + (dat1 * off1_2); p_a[2] = p_a[2] + (dat2 * off2_2); p_a[3] = p_a[3] + (dat3 * off3_2); p_a[4] = p_a[4] + (dat4 * off4_2); p_a[5] = p_a[5] + (dat5 * off5_2); p_a[6] = p_a[6] + (dat6 * off6_2); p_a[7] = p_a[7] + (dat7 * off7_2); p_a[8] = p_a[8] + (dat8 * off8_2); p_a[9] = p_a[9] + (dat9 * off9_2); p_a[10] = p_a[10] + (dat10 * off10_2); } } if (OPS_diags > 1) { ops_timers_core(&c1, &t1); OPS_kernels[111].time += t1 - t2; } ops_set_dirtybit_host(args, 11); ops_set_halo_dirtybit3(&args[0], range); ops_set_halo_dirtybit3(&args[1], range); ops_set_halo_dirtybit3(&args[6], range); ops_set_halo_dirtybit3(&args[7], range); ops_set_halo_dirtybit3(&args[8], range); ops_set_halo_dirtybit3(&args[9], range); if (OPS_diags > 1) { // Update kernel record ops_timers_core(&c2, &t2); OPS_kernels[111].mpi_time += t2 - t1; OPS_kernels[111].transfer += ops_compute_transfer(dim, start, end, &arg0); OPS_kernels[111].transfer += ops_compute_transfer(dim, start, end, &arg1); OPS_kernels[111].transfer += ops_compute_transfer(dim, start, end, &arg2); OPS_kernels[111].transfer += ops_compute_transfer(dim, start, end, &arg3); OPS_kernels[111].transfer += ops_compute_transfer(dim, start, end, &arg4); OPS_kernels[111].transfer += ops_compute_transfer(dim, start, end, &arg5); OPS_kernels[111].transfer += ops_compute_transfer(dim, start, end, &arg6); OPS_kernels[111].transfer += ops_compute_transfer(dim, start, end, &arg7); OPS_kernels[111].transfer += ops_compute_transfer(dim, start, end, &arg8); OPS_kernels[111].transfer += ops_compute_transfer(dim, start, end, &arg9); OPS_kernels[111].transfer += ops_compute_transfer(dim, start, end, &arg10); } }