int CrcManager::write_crc_file(char const *filename) // return 0 on failure
{
const size_t msgsize = 100;
char msg[msgsize];
snprintf(msg, msgsize, "%s", symbol_str("calc_crc")); // this may take some time, show the user a status indicator
if (stat_man) stat_man->push(msg,NULL);
int i,total=0;
for (i=0; i<total_files; i++)
{
int failed=0;
get_crc(i,failed);
if (failed)
{
jFILE *fp=new jFILE(get_filename(i),"rb");
if (!fp->open_failure())
{
set_crc(i,crc_file(fp));
total++;
}
delete fp;
} else total++;
if (stat_man)
stat_man->update(i*100/total_files);
}
if (stat_man) stat_man->pop();
jFILE *fp=new jFILE(NET_CRC_FILENAME,"wb");
if (fp->open_failure())
{
delete fp;
return 0;
}
fp->write_uint16(total);
total=0;
for (i=0; i<total_files; i++)
{
uint32_t crc;
int failed=0;
crc=get_crc(i,failed);
if (!failed)
{
fp->write_uint32(crc);
uint8_t len=strlen(get_filename(i))+1;
fp->write_uint8(len);
fp->write(get_filename(i),len);
total++;
}
}
delete fp;
return 1;
}
bool compile(const char *wuid, const char *target, const char *targetCluster)
{
Owned<IConstWUQuery> query = workunit->getQuery();
if (!query)
{
reportError("Workunit does not contain a query", 2);
return false;
}
addTimeStamp(workunit, SSTglobal, NULL, StWhenCompiled);
SCMStringBuffer mainDefinition;
SCMStringBuffer eclQuery;
query->getQueryText(eclQuery);
query->getQueryMainDefinition(mainDefinition);
StringBuffer eclccProgName;
splitDirTail(queryCurrentProcessPath(), eclccProgName);
eclccProgName.append("eclcc");
StringBuffer eclccCmd(" -shared");
if (eclQuery.length())
eclccCmd.append(" -");
if (mainDefinition.length())
eclccCmd.append(" -main ").append(mainDefinition);
eclccCmd.append(" --timings --xml");
eclccCmd.append(" --nostdinc");
if (globals->getPropBool("@enableEclccDali", true))
{
const char *daliServers = globals->queryProp("@daliServers");
if (!daliServers)
daliServers = ".";
eclccCmd.appendf(" -dfs=%s", daliServers);
const char *wuScope = workunit->queryWuScope();
if (!isEmptyString(wuScope))
eclccCmd.appendf(" -scope=%s", wuScope);
eclccCmd.appendf(" -cluster=%s", targetCluster);
SCMStringBuffer token;
workunit->getSecurityToken(token);
if (token.length())
eclccCmd.appendf(" -wuid=%s -token=%s", workunit->queryWuid(), token.str());
}
Owned<IPipeProcess> pipe = createPipeProcess();
pipe->setenv("ECLCCSERVER_THREAD_INDEX", idxStr.str());
Owned<IPropertyTreeIterator> options = globals->getElements("./Option");
ForEach(*options)
{
IPropertyTree &option = options->query();
const char *name = option.queryProp("@name");
const char *value = option.queryProp("@value");
const char *cluster = option.queryProp("@cluster"); // if cluster is set it's specific to a particular target
if (name && (cluster==NULL || cluster[0]==0 || strcmp(cluster, targetCluster)==0))
processOption(name, value, eclccCmd, eclccProgName, *pipe, false);
}
eclccCmd.appendf(" -o%s", wuid);
eclccCmd.appendf(" -platform=%s", target);
eclccCmd.appendf(" --component=%s", queryStatisticsComponentName());
Owned<IStringIterator> debugValues = &workunit->getDebugValues();
ForEach (*debugValues)
{
SCMStringBuffer debugStr, valueStr;
debugValues->str(debugStr);
workunit->getDebugValue(debugStr.str(), valueStr);
processOption(debugStr.str(), valueStr.str(), eclccCmd, eclccProgName, *pipe, true);
}
if (workunit->getResultLimit())
{
eclccCmd.appendf(" -fapplyInstantEclTransformations=1 -fapplyInstantEclTransformationsLimit=%u", workunit->getResultLimit());
}
try
{
cycle_t startCycles = get_cycles_now();
Owned<ErrorReader> errorReader = new ErrorReader(pipe, this);
Owned<AbortWaiter> abortWaiter = new AbortWaiter(pipe, workunit);
eclccCmd.insert(0, eclccProgName);
if (!pipe->run(eclccProgName, eclccCmd, ".", true, false, true, 0, true))
throw makeStringExceptionV(999, "Failed to run eclcc command %s", eclccCmd.str());
errorReader->start();
abortWaiter->start();
try
{
pipe->write(eclQuery.s.length(), eclQuery.s.str());
pipe->closeInput();
}
catch (IException *e)
{
reportError(e);
e->Release();
}
unsigned retcode = pipe->wait();
errorReader->join();
abortWaiter->stop();
if (retcode == 0)
{
StringBuffer realdllname, dllurl;
realdllname.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension);
StringBuffer realdllfilename(dllPath);
realdllfilename.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension);
StringBuffer wuXML;
if (!getWorkunitXMLFromFile(realdllfilename, wuXML))
throw makeStringException(999, "Failed to extract workunit from query dll");
Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit(wuXML);
queryExtendedWU(workunit)->copyWorkUnit(embeddedWU, false, true);
workunit->setIsClone(false);
const char *jobname = embeddedWU->queryJobName();
if (jobname && *jobname) //let ECL win naming job during initial compile
workunit->setJobName(jobname);
if (!workunit->getDebugValueBool("obfuscateOutput", false))
{
Owned<IWUQuery> query = workunit->updateQuery();
query->setQueryText(eclQuery.s.str());
}
createUNCFilename(realdllfilename.str(), dllurl);
unsigned crc = crc_file(realdllfilename.str());
Owned<IWUQuery> query = workunit->updateQuery();
associateLocalFile(query, FileTypeDll, realdllfilename, "Workunit DLL", crc);
queryDllServer().registerDll(realdllname.str(), "Workunit DLL", dllurl.str());
cycle_t elapsedCycles = get_cycles_now() - startCycles;
updateWorkunitTimeStat(workunit, SSTcompilestage, "compile", StTimeElapsed, NULL, cycle_to_nanosec(elapsedCycles));
workunit->commit();
return true;
}
}
catch (IException * e)
{
reportError(e);
e->Release();
}
return false;
}
bool compile(const char *wuid, const char *target, const char *targetCluster)
{
Owned<IConstWUQuery> query = workunit->getQuery();
if (!query)
{
reportError("Workunit does not contain a query", 2);
return false;
}
SCMStringBuffer mainDefinition;
SCMStringBuffer eclQuery;
query->getQueryText(eclQuery);
query->getQueryMainDefinition(mainDefinition);
StringBuffer eclccCmd("eclcc -shared");
if (eclQuery.length())
eclccCmd.append(" -");
if (mainDefinition.length())
eclccCmd.append(" -main ").append(mainDefinition);
Owned<IPropertyTreeIterator> options = globals->getElements("./Option");
ForEach(*options)
{
IPropertyTree &option = options->query();
const char *name = option.queryProp("@name");
const char *value = option.queryProp("@value");
const char *cluster = option.queryProp("@cluster");
if (name && (cluster==NULL || cluster[0]==0 || strcmp(cluster, targetCluster)==0))
{
// options starting '-' are simply passed through to eclcc as name=value
// others are passed as -foption=value
// if cluster is set it's specific to a particular target
eclccCmd.append(" ");
if (name[0]!='-')
eclccCmd.append("-f");
eclccCmd.append(name);
if (value)
eclccCmd.append('=').append(value);
}
}
eclccCmd.appendf(" -o%s", wuid);
eclccCmd.appendf(" -target=%s", target);
Owned<IStringIterator> debugValues = &workunit->getDebugValues();
ForEach (*debugValues)
{
SCMStringBuffer debugStr, valueStr;
debugValues->str(debugStr);
workunit->getDebugValue(debugStr.str(), valueStr);
if (memicmp(debugStr.str(), "eclcc-", 6) == 0)
{
//Allow eclcc-xx-<n> so that multiple values can be passed through for the same named debug symbol
const char * start = debugStr.str() + 6;
const char * dash = strchr(start, '-');
StringAttr optName;
if (dash)
optName.set(start, dash-start);
else
optName.set(start);
if (stricmp(optName, "compileOption") == 0)
eclccCmd.appendf(" -Wc,%s", valueStr.str());
else if (stricmp(optName, "includeLibraryPath") == 0)
eclccCmd.appendf(" -I%s", valueStr.str());
else if (stricmp(optName, "libraryPath") == 0)
eclccCmd.appendf(" -L%s", valueStr.str());
else
eclccCmd.appendf(" -%s=%s", start, valueStr.str());
}
else
eclccCmd.appendf(" -f%s=%s", debugStr.str(), valueStr.str());
}
if (workunit->getResultLimit())
{
eclccCmd.appendf(" -fapplyInstantEclTransformations=1 -fapplyInstantEclTransformationsLimit=%u", workunit->getResultLimit());
}
try
{
Owned<IPipeProcess> pipe = createPipeProcess();
Owned<ErrorReader> errorReader = new ErrorReader(pipe, this);
pipe->run("eclcc", eclccCmd, ".", true, false, true, 0);
errorReader->start();
try
{
pipe->write(eclQuery.s.length(), eclQuery.s.str());
pipe->closeInput();
}
catch (IException *e)
{
reportError(e);
e->Release();
}
unsigned retcode = pipe->wait();
errorReader->join();
if (retcode == 0)
{
StringBuffer realdllname, dllurl;
realdllname.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension);
StringBuffer realdllfilename(dllPath);
realdllfilename.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension);
StringBuffer wuXML;
if (getWorkunitXMLFromFile(realdllfilename, wuXML))
{
Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit();
embeddedWU->loadXML(wuXML);
queryExtendedWU(workunit)->copyWorkUnit(embeddedWU);
Owned<IWUQuery> query = workunit->updateQuery();
query->setQueryText(eclQuery.s.str());
}
createUNCFilename(realdllfilename.str(), dllurl);
unsigned crc = crc_file(realdllfilename.str());
Owned<IWUQuery> query = workunit->updateQuery();
associateLocalFile(query, FileTypeDll, realdllfilename, "Workunit DLL", crc);
queryDllServer().registerDll(realdllname.str(), "Workunit DLL", dllurl.str());
workunit->commit();
return true;
}
}
catch (IException * e)
{
reportError(e);
e->Release();
}
return false;
}
int main(int argc, char *argv[]) {
logger::init();
a2e_log("kernelcacher v%u.%u.%u - %s %s", KERNELCACHER_MAJOR_VERSION, KERNELCACHER_MINOR_VERSION, KERNELCACHER_REVISION_VERSION, KERNELCACHER_BUILT_DATE, KERNELCACHER_BUILT_TIME);
string usage = "usage: kernelcacher /path/to/data/kernels";
if(argc == 1) {
a2e_error("no kernel path specified!\n%s", usage.c_str());
return 0;
}
kernel_path = argv[1];
if(kernel_path.back() != '/') kernel_path.push_back('/');
cache_path = kernel_path.substr(0, kernel_path.rfind('/', kernel_path.length()-2)) + "/cache/";
a2e_debug("caching kernels from \"%s\" to \"%s\" ...", kernel_path, cache_path);
// compile kernels
const string lsl_sm_1x_str = " -DLOCAL_SIZE_LIMIT=512";
const string lsl_sm_20p_str = " -DLOCAL_SIZE_LIMIT=1024";
vector<tuple<string, string, string, std::function<string(const CC_TARGET&)>>> internal_kernels {
{
make_tuple("PARTICLE_INIT", "particle_spawn.cl", "particle_init",
[](const CC_TARGET&) { return " -DA2E_PARTICLE_INIT"; }),
make_tuple("PARTICLE_RESPAWN", "particle_spawn.cl", "particle_respawn",
[](const CC_TARGET&) { return ""; }),
make_tuple("PARTICLE_COMPUTE", "particle_compute.cl", "particle_compute",
[](const CC_TARGET&) { return ""; }),
make_tuple("PARTICLE_SORT_LOCAL", "particle_sort.cl", "bitonicSortLocal",
[&](const CC_TARGET& cc_target) { return (cc_target <= CC_TARGET::SM_13 ? lsl_sm_1x_str : lsl_sm_20p_str); }),
make_tuple("PARTICLE_SORT_MERGE_GLOBAL", "particle_sort.cl", "bitonicMergeGlobal",
[&](const CC_TARGET& cc_target) { return (cc_target <= CC_TARGET::SM_13 ? lsl_sm_1x_str : lsl_sm_20p_str); }),
make_tuple("PARTICLE_SORT_MERGE_LOCAL", "particle_sort.cl", "bitonicMergeLocal",
[&](const CC_TARGET& cc_target) { return (cc_target <= CC_TARGET::SM_13 ? lsl_sm_1x_str : lsl_sm_20p_str); }),
make_tuple("PARTICLE_COMPUTE_DISTANCES", "particle_sort.cl", "compute_distances",
[&](const CC_TARGET& cc_target) { return (cc_target <= CC_TARGET::SM_13 ? lsl_sm_1x_str : lsl_sm_20p_str); }),
}
};
task_counter = internal_kernels.size() + 1;
for(const auto& int_kernel : internal_kernels) {
task::spawn([=]() {
kernel_to_ptx(get<0>(int_kernel),
kernel_path+get<1>(int_kernel),
get<2>(int_kernel),
get<3>(int_kernel));
});
}
//
task::spawn([=]() {
file_io crc_file(cache_path+"CACHECRC", file_io::OPEN_TYPE::WRITE);
if(!crc_file.is_open()) {
a2e_error("couldn't create crc file!");
return;
}
auto& crc_fstream = *crc_file.get_filestream();
const auto kernel_files = core::get_file_list(kernel_path);
for(const auto& kfile : kernel_files) {
if(kfile.first[0] == '.') continue;
stringstream buffer(stringstream::in | stringstream::out);
if(!file_io::file_to_buffer(kernel_path+kfile.first, buffer)) {
a2e_error("failed to read kernel source!");
return;
}
const string src(buffer.str());
const unsigned int crc = crc32(crc32(0L, Z_NULL, 0), (const Bytef*)src.c_str(), (uInt)src.size());
crc_fstream << kfile.first << " " << hex << crc << dec << endl;
}
crc_file.close();
task_counter--;
});
//
while(task_counter != 0) {
SDL_Delay(100);
}
// done!
logger::destroy();
return 0;
}
/*** UNTESTED ***/
int
crc_ccitt(FILE *fin)
{
return crc_file(fin, crcccitt);
}
int
crc_32(FILE *fin)
{
return crc_file(fin, crc32);
}
/*** UNTESTED ***/
int
crc_16(FILE *fin)
{
return crc_file(fin, crc16);
}
bool compile(const char *wuid, const char *target, const char *targetCluster)
{
Owned<IConstWUQuery> query = workunit->getQuery();
if (!query)
{
reportError("Workunit does not contain a query", 2);
return false;
}
SCMStringBuffer mainDefinition;
SCMStringBuffer eclQuery;
query->getQueryText(eclQuery);
query->getQueryMainDefinition(mainDefinition);
StringBuffer eclccProgName("eclcc");
StringBuffer eclccCmd(" -shared");
if (eclQuery.length())
eclccCmd.append(" -");
if (mainDefinition.length())
eclccCmd.append(" -main ").append(mainDefinition);
if (workunit->getDebugValueBool("addTimingToWorkunit", true))
eclccCmd.append(" --timings");
Owned<IPipeProcess> pipe = createPipeProcess();
pipe->setenv("ECLCCSERVER_THREAD_INDEX", idxStr.str());
Owned<IPropertyTreeIterator> options = globals->getElements("./Option");
ForEach(*options)
{
IPropertyTree &option = options->query();
const char *name = option.queryProp("@name");
const char *value = option.queryProp("@value");
const char *cluster = option.queryProp("@cluster"); // if cluster is set it's specific to a particular target
if (name && (cluster==NULL || cluster[0]==0 || strcmp(cluster, targetCluster)==0))
processOption(name, value, eclccCmd, eclccProgName, *pipe, false);
}
eclccCmd.appendf(" -o%s", wuid);
eclccCmd.appendf(" -platform=%s", target);
Owned<IStringIterator> debugValues = &workunit->getDebugValues();
ForEach (*debugValues)
{
SCMStringBuffer debugStr, valueStr;
debugValues->str(debugStr);
workunit->getDebugValue(debugStr.str(), valueStr);
processOption(debugStr.str(), valueStr.str(), eclccCmd, eclccProgName, *pipe, true);
}
if (workunit->getResultLimit())
{
eclccCmd.appendf(" -fapplyInstantEclTransformations=1 -fapplyInstantEclTransformationsLimit=%u", workunit->getResultLimit());
}
try
{
unsigned time = msTick();
Owned<ErrorReader> errorReader = new ErrorReader(pipe, this);
eclccCmd.insert(0, eclccProgName);
pipe->run(eclccProgName, eclccCmd, ".", true, false, true, 0);
errorReader->start();
try
{
pipe->write(eclQuery.s.length(), eclQuery.s.str());
pipe->closeInput();
}
catch (IException *e)
{
reportError(e);
e->Release();
}
unsigned retcode = pipe->wait();
errorReader->join();
if (retcode == 0)
{
StringBuffer realdllname, dllurl;
realdllname.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension);
StringBuffer realdllfilename(dllPath);
realdllfilename.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension);
StringBuffer wuXML;
if (getWorkunitXMLFromFile(realdllfilename, wuXML))
{
Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit();
embeddedWU->loadXML(wuXML);
queryExtendedWU(workunit)->copyWorkUnit(embeddedWU, true);
workunit->setIsClone(false);
SCMStringBuffer jobname;
if (embeddedWU->getJobName(jobname).length()) //let ECL win naming job during initial compile
workunit->setJobName(jobname.str());
Owned<IWUQuery> query = workunit->updateQuery();
query->setQueryText(eclQuery.s.str());
}
createUNCFilename(realdllfilename.str(), dllurl);
unsigned crc = crc_file(realdllfilename.str());
Owned<IWUQuery> query = workunit->updateQuery();
associateLocalFile(query, FileTypeDll, realdllfilename, "Workunit DLL", crc);
queryDllServer().registerDll(realdllname.str(), "Workunit DLL", dllurl.str());
time = msTick()-time;
if (workunit->getDebugValueBool("addTimingToWorkunit", true))
workunit->setTimerInfo("eclccserver: create workunit", NULL, time, 1, 0);
workunit->commit();
return true;
}
}
catch (IException * e)
{
reportError(e);
e->Release();
}
return false;
}
