void TypeCompiler::generateConstructor(FunctionLiteral *function,
ConstructorInfo *constructor)
{
currentMethod = constructor;
currentFunctionLiteral = function;
CompilationUnit *cunit = cls->pkgDecl->compilationUnit;
CodeState codeState;
FuncState funcState;
initCodeState(&codeState, &funcState, cunit->filename);
cs = &codeState;
parList(function, true);
declareLocalVariables(function);
visitStatementArray(function->statements);
closeCodeState(&codeState);
constructor->setByteCode(generateByteCode(funcState.f));
currentMethod = NULL;
}
void TypeCompiler::functionBody(ExpDesc *e, FunctionLiteral *flit, int line)
{
FuncState new_fs;
BC::openFunction(cs, &new_fs);
new_fs.f->linedefined = line;
parList(flit, false);
// setup closure info here so it is captured as an upvalue
char funcinfo[256];
snprintf(funcinfo, 250, "__ls_funcinfo_numargs_%i", flit->childIndex);
ExpDesc finfo;
BC::singleVar(cs, &finfo, funcinfo);
declareLocalVariables(flit);
chunk(flit->statements);
new_fs.f->lastlinedefined = flit->lineNumber;
BC::closeFunction(cs);
BC::pushClosure(cs, &new_fs, e);
}
void TypeCompiler::generateMethod(FunctionLiteral *function,
MethodInfo *method)
{
currentMethod = method;
currentMethodCoroutine = function->isCoroutine;
currentFunctionLiteral = function;
CompilationUnit *cunit = cls->pkgDecl->compilationUnit;
CodeState codeState;
FuncState funcState;
initCodeState(&codeState, &funcState, cunit->filename);
cs = &codeState;
parList(function, !function->isStatic);
declareLocalVariables(function);
// we insert a yield to account for argument passing
if (function->isCoroutine)
{
ExpDesc yield;
insertYield(&yield);
}
visitStatementArray(function->statements);
closeCodeState(&codeState);
method->setByteCode(generateByteCode(funcState.f));
currentMethod = NULL;
currentMethodCoroutine = false;
}
static ssize_t
generator(
char *buf,
size_t buflen,
const struct SubproblemDim *subdims,
const struct PGranularity *pgran,
void *extra)
{
char tmp[1024];
struct KgenContext *ctx;
ssize_t ret;
CLBLASKernExtra *kextra = (CLBLASKernExtra*)extra;
DataType dtype = kextra->dtype;
KernelExtraFlags kflags = kextra->flags;
CLBLASKernExtra extraNew;
BlasGenSettings gset;
TileMulOpts mulOpts;
const char *ptrName;
UpdateResultFlags upFlags = 0;
TilePostFetchPrivate pfPriv;
unsigned int l1Pans;
bool b;
Tile parTile;
TrsmExtraParams *extraParams = (TrsmExtraParams *)kextra->solverPriv;
int ldsLarge, lds_diagonal;
bool isInline;
TileSet tileSet;
char copy2LDSFuncName[FUNC_NAME_MAXLEN];
TailStatus tailStatus = 0;
FetchAddrMode addrMode = 0;
bool tailM = ((kflags & KEXTRA_TAILS_M) != 0);
bool tailN = ((kflags & KEXTRA_TAILS_N) != 0);
size_t alignK;
if (pgran->wgDim != 1) {
return -EINVAL;
}
l1Pans = (unsigned int)(subdims[0].x / subdims[1].x);
memset(&gset, 0, sizeof(gset));
gset.flags = BGF_WHOLE_A | BGF_EXPLICIT_INLINE | BGF_UPTRS;
memcpy(gset.subdims, subdims, sizeof(SubproblemDim) * 2);
// there is not need in block structure along K
gset.subdims[0].bwidth = gset.subdims[1].bwidth;
subdims = gset.subdims;
/*
* Since tiles are changed dynamically, e. g. in the main tilemul
* loop they are rectangular, but at the second stage both A and B
* tile storages are used for square tiles. One must adjust physical
* vectorization accordindly, so as vector length might not be
* greater than linear size of any tile
*/
memcpy(&extraNew, kextra, sizeof(extraNew));
extraNew.vecLenA = umin(kextra->vecLenA, (unsigned int)subdims[1].y);
extraNew.vecLenB = umin(kextra->vecLenB, (unsigned int)subdims[1].y);
gset.pgran = pgran;
gset.kextra = &extraNew;
initKernelVarNames(&gset.varNames);
// multiplication options
mulOpts.memA = CLMEM_GLOBAL_MEMORY;
mulOpts.memB = CLMEM_GLOBAL_MEMORY;
mulOpts.core = (kextra->flags & KEXTRA_ENABLE_MAD) ? TILEMUL_MAD :
TILEMUL_MULADD;
mulOpts.postFetch = NULL;
mulOpts.flags = kextraToTilemulFlags(CLBLAS_TRSM, kflags);
mulOpts.flags |= TILEMUL_EXTERN_RDECL | TILEMUL_NOT_INC_K;
mulOpts.fctx = createFetchContext();
if (mulOpts.fctx == NULL) {
return -ENOMEM;
}
disableFetchOptLevels(mulOpts.fctx, FOPTLEV_TMP_COORD_PRECOMPUTING);
isInline = (gset.flags & BGF_EXPLICIT_INLINE);
initTiles(&gset, &tileSet, subdims, kflags, dtype,
PRIV_STORAGE_VARIABLE_SET);
ctx = createKgenContext(buf, buflen, true);
if (ctx == NULL) {
destroyFetchContext(mulOpts.fctx);
return -ENOMEM;
}
kgenAddStmt(ctx, "#pragma OPENCL EXTENSION cl_amd_printf : enable\n\n");
b = isDoubleBasedType(dtype);
kgenDeclareUptrs(ctx, b);
if (isComplexType(dtype)) {
genComplexMathOperators(ctx, dtype);
}
if(!isInline) {
genTileInverting(ctx, &gset, &tileSet);
}
if ( extraParams->ldsUse != LDS_NO_USE ) {
SubproblemDim sdims;
DBlockCopyFlags flags;
unsigned int vecLen;
if (!isMatrixAccessColMaj(CLBLAS_TRSM, kflags, MATRIX_B)) {
sdims.x = gset.subdims[1].bwidth * extraParams->unrollingFactor;
sdims.y = gset.subdims[0].x;
}
else {
sdims.x = gset.subdims[0].x;
sdims.y = gset.subdims[1].bwidth * extraParams->unrollingFactor;
}
vecLen = getVecLen(&gset, CLBLAS_TRSM, MATRIX_B);
flags = (vecLen < 4) ? DBLOCK_COPY_NOT_VECTORIZE : 0;
copyDataBlockGen(ctx, &sdims, gset.pgran, dtype,
DBLOCK_GLOBAL_TO_LOCAL, flags);
kgenAddBlankLine(ctx);
kgenGetLastFuncName(copy2LDSFuncName, FUNC_NAME_MAXLEN, ctx);
}
declareTrxmKernel(ctx, dtype, pgran, kflags, CLBLAS_TRSM, "Cached", false,
true);
kgenBeginFuncBody(ctx);
declareLocalVariables(ctx, &gset, &parTile, extraParams);
if (kflags & KEXTRA_A_OFF_NOT_ZERO) {
kgenAddStmt(ctx, "A += offA;\n");
}
genTrxmBMatrShift(ctx, kflags, false);
ptrName = dtypeUPtrField(dtype);
sprintf(tmp, "uB.%s = B;\n\n", ptrName);
kgenAddStmt(ctx, tmp);
// external loop
sprintf(tmp, "for (m0 = 0; m0 < M; m0 += %lu)", subdims[0].y);
kgenBeginBranch(ctx, tmp);
genZeroTile(ctx, &gset.tileCY);
genSetupCoords(ctx, &gset, BLOCK_UPDATE);
kgenAddStmt(ctx, "// Stage 1. Multiply and update with large blocks\n");
gset.tileA = tileSet.rectA;
gset.tileBX = tileSet.origB;
if (!isMatrixUpper(kflags) && tailM) {
addrMode |= FETCH_ADDR_A_CYCLICAL;
setFetchAddrMode(mulOpts.fctx, addrMode);
}
ldsLarge = ((extraParams->ldsUse & LDS_USE_LARGE) != 0);
alignK = subdims[1].bwidth;
if (ldsLarge) {
alignK *= extraParams->unrollingFactor;
}
if (ldsLarge) {
const char *oldCoordB;
FetchAddrMode bamode = addrMode | FETCH_ADDR_K_RELATIVE;
bool withSkew;
withSkew = useSkewedFetchB(&gset);
if (!withSkew) {
bamode |= FETCH_ADDR_B_RELATIVE;
}
else {
bamode |= FETCH_ADDR_B_CYCLICAL;
}
setFetchAddrMode(mulOpts.fctx, bamode);
if (tailN) {
/*
* Conditional branch for those items which hit into
* matrix B with their matrix coordinates
*/
sprintf(tmp, "if ((gid + 1) * %lu < N)", subdims[0].x);
kgenBeginBranch(ctx, tmp);
}
if (isMatrixAccessColMaj(CLBLAS_TRSM, kflags, MATRIX_A)) {
kgenPrintf(ctx, "uA.%s = A + k0 * lda;\n", ptrName);
}
else {
kgenPrintf(ctx, "uA.%s = A + k0;\n", ptrName);
}
if (withSkew) {
unsigned int bwidthOld;
oldCoordB = gset.varNames.coordB;
gset.varNames.coordB = "skewX";
bwidthOld = gset.subdims[0].bwidth;
gset.subdims[0].bwidth = (parTile.trans) ? parTile.nrRows :
parTile.nrCols;
gset.subdims[0].bwidth = bwidthOld;
}
genInternalLoopCtl(ctx, subdims, kflags, alignK, alignK);
genPreloadedTileMul(ctx, &gset, &mulOpts, &parTile, copy2LDSFuncName);
genInternalLoopEnd(ctx); // loop over K
if (withSkew) {
gset.varNames.coordB = oldCoordB;
setFetchAddrMode(mulOpts.fctx, bamode & ~FETCH_ADDR_B_CYCLICAL);
// deliver from skew in the result before proceed to the next stage
genTileCyclicalShift(ctx, &gset);
}
if (tailN) {
kgenEndBranch(ctx, NULL);
kgenBeginBranch(ctx, "else");
}
setFetchAddrMode(mulOpts.fctx, addrMode);
}
if (!ldsLarge || tailN) {
genCheckShiftTailB(ctx, &gset, 0, &tailStatus);
if ((kflags & KEXTRA_TAILS_N_LOWER) && !tailStatus) {
addrMode |= FETCH_ADDR_B_CYCLICAL;
setFetchAddrMode(mulOpts.fctx, addrMode);
}
if (tailN) {
sprintfHitMatrixCond(tmp, MATRIX_B, "if (", ")");
kgenBeginBranch(ctx, tmp);
}
genInternalLoopCtl(ctx, subdims, kflags, subdims[1].bwidth, alignK);
tileMulGen(ctx, &gset, &mulOpts);
genInternalLoopEnd(ctx); // loop over K
if (tailN) {
kgenEndBranch(ctx, NULL);
}
if (extraParams->ldsUse & LDS_USE_LARGE) {
kgenEndBranch(ctx, NULL);
}
}
sprintf(tmp, "uA.%s = A;\n\n", ptrName);
kgenAddStmt(ctx, tmp);
// processing tails along update dimension
if (isMatrixUpper(kflags) &&
((kflags & KEXTRA_TAILS_K_LOWER) ||
(ldsLarge && extraParams->unrolledTail))) {
unsigned int tailChunks;
tailChunks = (extraParams->ldsUse & LDS_USE_LARGE) ?
extraParams->unrolledTail : 1;
if (tailN) {
char hitCond[1024];
sprintfHitMatrixCond(hitCond, MATRIX_B, "(", ")");
sprintf(tmp, "if ((currM + %lu < M) && %s)",
subdims[0].y, hitCond);
}
else {
sprintf(tmp, "if (currM + %lu < M)", subdims[0].y);
}
kgenBeginBranch(ctx, tmp);
if (kflags & KEXTRA_TAILS_K_LOWER) {
setFetchAddrMode(mulOpts.fctx, addrMode | FETCH_ADDR_K_CYCLICAL);
setFetchHandler(&mulOpts, &gset, defaultTilePostFetch, &pfPriv);
}
if (tailChunks > 1) {
mulOpts.flags &= ~TILEMUL_NOT_INC_K;
sprintf(tmp, "for (uint k1 = 0; k1 < %u; k1++)", tailChunks);
kgenBeginBranch(ctx, tmp);
}
addrMode |= FETCH_ADDR_B_CYCLICAL;
setFetchAddrMode(mulOpts.fctx, addrMode);
tileMulGen(ctx, &gset, &mulOpts);
if (tailChunks > 1) {
kgenEndBranch(ctx, NULL);
mulOpts.flags |= TILEMUL_NOT_INC_K;
}
kgenEndBranch(ctx, NULL);
}
gset.tileA = tileSet.squareA;
kgenAddStmt(ctx, "\n/*\n"
" * Stage 2. A part of work items multiply got result on "
"a respective\n"
" * inverted diagonal block, and the remaining ones wait. "
"Then they perform\n"
" * one step of further intermediate result evaluation as "
"multiplying tile by tile.\n"
" * It continues until the whole panel of the "
"matrix A is processed\n"
" */\n");
// one must deal further with square blocks strictly
gset.subdims[0].bwidth = gset.subdims[1].bwidth = gset.subdims[1].y;
sprintf(tmp, "for (m1 = 0; m1 < %lu; m1++)", subdims[0].y / subdims[1].y);
kgenBeginBranch(ctx, tmp);
if (extraParams->ldsUse & LDS_USE_DIAGONAL) {
sprintf(tmp, "const int bid = lid %% %u;\n\n",
l1Pans);
kgenAddStmt(ctx, tmp);
}
/*
* Update the intermediate result multiply on the inverted diagonal tile,
* and write back
*/
genSetupCoords(ctx, &gset, TILE_UPDATE);
sprintfStage2Condition(tmp, &gset, 0);
ret = kgenBeginBranch(ctx, tmp);
upFlags = kextraToUpresFlags(CLBLAS_TRSM, kflags);
upFlags |= tailStatusToUpresFlags(tailStatus);
upFlags |= UPRES_PRIV_DEST | UPRES_WITH_BETA;
genUpdateIntermResult(ctx, &gset, false, upFlags);
kgenAddBlankLine(ctx);
lds_diagonal = ((extraParams->ldsUse & LDS_USE_DIAGONAL) &&
(kflags & (KEXTRA_COLUMN_MAJOR)) == 0 &&
!(tailM || tailN) &&
!(upFlags & UPRES_NO_VECTORIZATION) &&
!isComplexType(kextra->dtype));
/*
* it's needed now to adjust addressing mode of A so as to don't
* exceed the bound of A
*/
if (tailM) {
setFetchAddrMode(mulOpts.fctx,
addrMode | FETCH_ADDR_A_CYCLICAL |
FETCH_ADDR_K_CYCLICAL);
extraNew.flags |= KEXTRA_TAILS_K_LOWER;
}
genMulOnDiagonalTile(ctx, &gset, &tileSet, &mulOpts);
gset.tileBX = tileSet.bStage2;
if (tailM) {
setFetchHandler(&mulOpts, &gset, defaultTilePostFetch, &pfPriv);
}
kgenAddStmt(ctx, "// Write back the given result\n");
upFlags = kextraToUpresFlags(CLBLAS_TRSM, kflags);
upFlags |= tailStatusToUpresFlags(tailStatus);
if (lds_diagonal) {
sprintf(tmp, "tmpB[%%u * %u + bid]", l1Pans);
}
genResultUpdateWithFlags(ctx, CLBLAS_TRSM, &gset, upFlags,
NULL, NULL, lds_diagonal ? tmp : NULL);
kgenEndBranch(ctx, NULL); // multiply on the inverted tile path
kgenAddBarrier(ctx, CLK_GLOBAL_MEM_FENCE);
// continue the tile update
kgenAddBlankLine(ctx);
sprintfStage2Condition(tmp, &gset, 1);
kgenBeginBranch(ctx, tmp);
genCheckShiftTailB(ctx, &gset, 0, &tailStatus);
if (lds_diagonal) {
// TODO: add here storing to LDS as well
}
else {
addrMode |= FETCH_ADDR_B_CYCLICAL;
setFetchAddrMode(mulOpts.fctx, addrMode);
tileMulGen(ctx, &gset, &mulOpts);
}
kgenEndBranch(ctx, NULL); // tile update path
kgenAddBarrier(ctx, CLK_GLOBAL_MEM_FENCE);
kgenEndBranch(ctx, NULL); // second stage loop
if (isMatrixUpper(kflags)) {
sprintf(tmp, "currM -= %lu;\n", subdims[0].y);
kgenAddStmt(ctx, tmp);
}
kgenEndBranch(ctx, NULL); // loop over M
ret = kgenEndFuncBody(ctx);
if (!ret) {
ret = (ssize_t)kgenSourceSize(ctx) + 1;
}
destroyFetchContext(mulOpts.fctx);
destroyKgenContext(ctx);
return (ret < 0) ? -EOVERFLOW : ret;
}