int CrcManager::write_crc_file(char const *filename) // return 0 on failure { const size_t msgsize = 100; char msg[msgsize]; snprintf(msg, msgsize, "%s", symbol_str("calc_crc")); // this may take some time, show the user a status indicator if (stat_man) stat_man->push(msg,NULL); int i,total=0; for (i=0; i<total_files; i++) { int failed=0; get_crc(i,failed); if (failed) { jFILE *fp=new jFILE(get_filename(i),"rb"); if (!fp->open_failure()) { set_crc(i,crc_file(fp)); total++; } delete fp; } else total++; if (stat_man) stat_man->update(i*100/total_files); } if (stat_man) stat_man->pop(); jFILE *fp=new jFILE(NET_CRC_FILENAME,"wb"); if (fp->open_failure()) { delete fp; return 0; } fp->write_uint16(total); total=0; for (i=0; i<total_files; i++) { uint32_t crc; int failed=0; crc=get_crc(i,failed); if (!failed) { fp->write_uint32(crc); uint8_t len=strlen(get_filename(i))+1; fp->write_uint8(len); fp->write(get_filename(i),len); total++; } } delete fp; return 1; }
bool compile(const char *wuid, const char *target, const char *targetCluster) { Owned<IConstWUQuery> query = workunit->getQuery(); if (!query) { reportError("Workunit does not contain a query", 2); return false; } addTimeStamp(workunit, SSTglobal, NULL, StWhenCompiled); SCMStringBuffer mainDefinition; SCMStringBuffer eclQuery; query->getQueryText(eclQuery); query->getQueryMainDefinition(mainDefinition); StringBuffer eclccProgName; splitDirTail(queryCurrentProcessPath(), eclccProgName); eclccProgName.append("eclcc"); StringBuffer eclccCmd(" -shared"); if (eclQuery.length()) eclccCmd.append(" -"); if (mainDefinition.length()) eclccCmd.append(" -main ").append(mainDefinition); eclccCmd.append(" --timings --xml"); eclccCmd.append(" --nostdinc"); if (globals->getPropBool("@enableEclccDali", true)) { const char *daliServers = globals->queryProp("@daliServers"); if (!daliServers) daliServers = "."; eclccCmd.appendf(" -dfs=%s", daliServers); const char *wuScope = workunit->queryWuScope(); if (!isEmptyString(wuScope)) eclccCmd.appendf(" -scope=%s", wuScope); eclccCmd.appendf(" -cluster=%s", targetCluster); SCMStringBuffer token; workunit->getSecurityToken(token); if (token.length()) eclccCmd.appendf(" -wuid=%s -token=%s", workunit->queryWuid(), token.str()); } Owned<IPipeProcess> pipe = createPipeProcess(); pipe->setenv("ECLCCSERVER_THREAD_INDEX", idxStr.str()); Owned<IPropertyTreeIterator> options = globals->getElements("./Option"); ForEach(*options) { IPropertyTree &option = options->query(); const char *name = option.queryProp("@name"); const char *value = option.queryProp("@value"); const char *cluster = option.queryProp("@cluster"); // if cluster is set it's specific to a particular target if (name && (cluster==NULL || cluster[0]==0 || strcmp(cluster, targetCluster)==0)) processOption(name, value, eclccCmd, eclccProgName, *pipe, false); } eclccCmd.appendf(" -o%s", wuid); eclccCmd.appendf(" -platform=%s", target); eclccCmd.appendf(" --component=%s", queryStatisticsComponentName()); Owned<IStringIterator> debugValues = &workunit->getDebugValues(); ForEach (*debugValues) { SCMStringBuffer debugStr, valueStr; debugValues->str(debugStr); workunit->getDebugValue(debugStr.str(), valueStr); processOption(debugStr.str(), valueStr.str(), eclccCmd, eclccProgName, *pipe, true); } if (workunit->getResultLimit()) { eclccCmd.appendf(" -fapplyInstantEclTransformations=1 -fapplyInstantEclTransformationsLimit=%u", workunit->getResultLimit()); } try { cycle_t startCycles = get_cycles_now(); Owned<ErrorReader> errorReader = new ErrorReader(pipe, this); Owned<AbortWaiter> abortWaiter = new AbortWaiter(pipe, workunit); eclccCmd.insert(0, eclccProgName); if (!pipe->run(eclccProgName, eclccCmd, ".", true, false, true, 0, true)) throw makeStringExceptionV(999, "Failed to run eclcc command %s", eclccCmd.str()); errorReader->start(); abortWaiter->start(); try { pipe->write(eclQuery.s.length(), eclQuery.s.str()); pipe->closeInput(); } catch (IException *e) { reportError(e); e->Release(); } unsigned retcode = pipe->wait(); errorReader->join(); abortWaiter->stop(); if (retcode == 0) { StringBuffer realdllname, dllurl; realdllname.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension); StringBuffer realdllfilename(dllPath); realdllfilename.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension); StringBuffer wuXML; if (!getWorkunitXMLFromFile(realdllfilename, wuXML)) throw makeStringException(999, "Failed to extract workunit from query dll"); Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit(wuXML); queryExtendedWU(workunit)->copyWorkUnit(embeddedWU, false, true); workunit->setIsClone(false); const char *jobname = embeddedWU->queryJobName(); if (jobname && *jobname) //let ECL win naming job during initial compile workunit->setJobName(jobname); if (!workunit->getDebugValueBool("obfuscateOutput", false)) { Owned<IWUQuery> query = workunit->updateQuery(); query->setQueryText(eclQuery.s.str()); } createUNCFilename(realdllfilename.str(), dllurl); unsigned crc = crc_file(realdllfilename.str()); Owned<IWUQuery> query = workunit->updateQuery(); associateLocalFile(query, FileTypeDll, realdllfilename, "Workunit DLL", crc); queryDllServer().registerDll(realdllname.str(), "Workunit DLL", dllurl.str()); cycle_t elapsedCycles = get_cycles_now() - startCycles; updateWorkunitTimeStat(workunit, SSTcompilestage, "compile", StTimeElapsed, NULL, cycle_to_nanosec(elapsedCycles)); workunit->commit(); return true; } } catch (IException * e) { reportError(e); e->Release(); } return false; }
bool compile(const char *wuid, const char *target, const char *targetCluster) { Owned<IConstWUQuery> query = workunit->getQuery(); if (!query) { reportError("Workunit does not contain a query", 2); return false; } SCMStringBuffer mainDefinition; SCMStringBuffer eclQuery; query->getQueryText(eclQuery); query->getQueryMainDefinition(mainDefinition); StringBuffer eclccCmd("eclcc -shared"); if (eclQuery.length()) eclccCmd.append(" -"); if (mainDefinition.length()) eclccCmd.append(" -main ").append(mainDefinition); Owned<IPropertyTreeIterator> options = globals->getElements("./Option"); ForEach(*options) { IPropertyTree &option = options->query(); const char *name = option.queryProp("@name"); const char *value = option.queryProp("@value"); const char *cluster = option.queryProp("@cluster"); if (name && (cluster==NULL || cluster[0]==0 || strcmp(cluster, targetCluster)==0)) { // options starting '-' are simply passed through to eclcc as name=value // others are passed as -foption=value // if cluster is set it's specific to a particular target eclccCmd.append(" "); if (name[0]!='-') eclccCmd.append("-f"); eclccCmd.append(name); if (value) eclccCmd.append('=').append(value); } } eclccCmd.appendf(" -o%s", wuid); eclccCmd.appendf(" -target=%s", target); Owned<IStringIterator> debugValues = &workunit->getDebugValues(); ForEach (*debugValues) { SCMStringBuffer debugStr, valueStr; debugValues->str(debugStr); workunit->getDebugValue(debugStr.str(), valueStr); if (memicmp(debugStr.str(), "eclcc-", 6) == 0) { //Allow eclcc-xx-<n> so that multiple values can be passed through for the same named debug symbol const char * start = debugStr.str() + 6; const char * dash = strchr(start, '-'); StringAttr optName; if (dash) optName.set(start, dash-start); else optName.set(start); if (stricmp(optName, "compileOption") == 0) eclccCmd.appendf(" -Wc,%s", valueStr.str()); else if (stricmp(optName, "includeLibraryPath") == 0) eclccCmd.appendf(" -I%s", valueStr.str()); else if (stricmp(optName, "libraryPath") == 0) eclccCmd.appendf(" -L%s", valueStr.str()); else eclccCmd.appendf(" -%s=%s", start, valueStr.str()); } else eclccCmd.appendf(" -f%s=%s", debugStr.str(), valueStr.str()); } if (workunit->getResultLimit()) { eclccCmd.appendf(" -fapplyInstantEclTransformations=1 -fapplyInstantEclTransformationsLimit=%u", workunit->getResultLimit()); } try { Owned<IPipeProcess> pipe = createPipeProcess(); Owned<ErrorReader> errorReader = new ErrorReader(pipe, this); pipe->run("eclcc", eclccCmd, ".", true, false, true, 0); errorReader->start(); try { pipe->write(eclQuery.s.length(), eclQuery.s.str()); pipe->closeInput(); } catch (IException *e) { reportError(e); e->Release(); } unsigned retcode = pipe->wait(); errorReader->join(); if (retcode == 0) { StringBuffer realdllname, dllurl; realdllname.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension); StringBuffer realdllfilename(dllPath); realdllfilename.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension); StringBuffer wuXML; if (getWorkunitXMLFromFile(realdllfilename, wuXML)) { Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit(); embeddedWU->loadXML(wuXML); queryExtendedWU(workunit)->copyWorkUnit(embeddedWU); Owned<IWUQuery> query = workunit->updateQuery(); query->setQueryText(eclQuery.s.str()); } createUNCFilename(realdllfilename.str(), dllurl); unsigned crc = crc_file(realdllfilename.str()); Owned<IWUQuery> query = workunit->updateQuery(); associateLocalFile(query, FileTypeDll, realdllfilename, "Workunit DLL", crc); queryDllServer().registerDll(realdllname.str(), "Workunit DLL", dllurl.str()); workunit->commit(); return true; } } catch (IException * e) { reportError(e); e->Release(); } return false; }
int main(int argc, char *argv[]) { logger::init(); a2e_log("kernelcacher v%u.%u.%u - %s %s", KERNELCACHER_MAJOR_VERSION, KERNELCACHER_MINOR_VERSION, KERNELCACHER_REVISION_VERSION, KERNELCACHER_BUILT_DATE, KERNELCACHER_BUILT_TIME); string usage = "usage: kernelcacher /path/to/data/kernels"; if(argc == 1) { a2e_error("no kernel path specified!\n%s", usage.c_str()); return 0; } kernel_path = argv[1]; if(kernel_path.back() != '/') kernel_path.push_back('/'); cache_path = kernel_path.substr(0, kernel_path.rfind('/', kernel_path.length()-2)) + "/cache/"; a2e_debug("caching kernels from \"%s\" to \"%s\" ...", kernel_path, cache_path); // compile kernels const string lsl_sm_1x_str = " -DLOCAL_SIZE_LIMIT=512"; const string lsl_sm_20p_str = " -DLOCAL_SIZE_LIMIT=1024"; vector<tuple<string, string, string, std::function<string(const CC_TARGET&)>>> internal_kernels { { make_tuple("PARTICLE_INIT", "particle_spawn.cl", "particle_init", [](const CC_TARGET&) { return " -DA2E_PARTICLE_INIT"; }), make_tuple("PARTICLE_RESPAWN", "particle_spawn.cl", "particle_respawn", [](const CC_TARGET&) { return ""; }), make_tuple("PARTICLE_COMPUTE", "particle_compute.cl", "particle_compute", [](const CC_TARGET&) { return ""; }), make_tuple("PARTICLE_SORT_LOCAL", "particle_sort.cl", "bitonicSortLocal", [&](const CC_TARGET& cc_target) { return (cc_target <= CC_TARGET::SM_13 ? lsl_sm_1x_str : lsl_sm_20p_str); }), make_tuple("PARTICLE_SORT_MERGE_GLOBAL", "particle_sort.cl", "bitonicMergeGlobal", [&](const CC_TARGET& cc_target) { return (cc_target <= CC_TARGET::SM_13 ? lsl_sm_1x_str : lsl_sm_20p_str); }), make_tuple("PARTICLE_SORT_MERGE_LOCAL", "particle_sort.cl", "bitonicMergeLocal", [&](const CC_TARGET& cc_target) { return (cc_target <= CC_TARGET::SM_13 ? lsl_sm_1x_str : lsl_sm_20p_str); }), make_tuple("PARTICLE_COMPUTE_DISTANCES", "particle_sort.cl", "compute_distances", [&](const CC_TARGET& cc_target) { return (cc_target <= CC_TARGET::SM_13 ? lsl_sm_1x_str : lsl_sm_20p_str); }), } }; task_counter = internal_kernels.size() + 1; for(const auto& int_kernel : internal_kernels) { task::spawn([=]() { kernel_to_ptx(get<0>(int_kernel), kernel_path+get<1>(int_kernel), get<2>(int_kernel), get<3>(int_kernel)); }); } // task::spawn([=]() { file_io crc_file(cache_path+"CACHECRC", file_io::OPEN_TYPE::WRITE); if(!crc_file.is_open()) { a2e_error("couldn't create crc file!"); return; } auto& crc_fstream = *crc_file.get_filestream(); const auto kernel_files = core::get_file_list(kernel_path); for(const auto& kfile : kernel_files) { if(kfile.first[0] == '.') continue; stringstream buffer(stringstream::in | stringstream::out); if(!file_io::file_to_buffer(kernel_path+kfile.first, buffer)) { a2e_error("failed to read kernel source!"); return; } const string src(buffer.str()); const unsigned int crc = crc32(crc32(0L, Z_NULL, 0), (const Bytef*)src.c_str(), (uInt)src.size()); crc_fstream << kfile.first << " " << hex << crc << dec << endl; } crc_file.close(); task_counter--; }); // while(task_counter != 0) { SDL_Delay(100); } // done! logger::destroy(); return 0; }
/*** UNTESTED ***/ int crc_ccitt(FILE *fin) { return crc_file(fin, crcccitt); }
int crc_32(FILE *fin) { return crc_file(fin, crc32); }
/*** UNTESTED ***/ int crc_16(FILE *fin) { return crc_file(fin, crc16); }
bool compile(const char *wuid, const char *target, const char *targetCluster) { Owned<IConstWUQuery> query = workunit->getQuery(); if (!query) { reportError("Workunit does not contain a query", 2); return false; } SCMStringBuffer mainDefinition; SCMStringBuffer eclQuery; query->getQueryText(eclQuery); query->getQueryMainDefinition(mainDefinition); StringBuffer eclccProgName("eclcc"); StringBuffer eclccCmd(" -shared"); if (eclQuery.length()) eclccCmd.append(" -"); if (mainDefinition.length()) eclccCmd.append(" -main ").append(mainDefinition); if (workunit->getDebugValueBool("addTimingToWorkunit", true)) eclccCmd.append(" --timings"); Owned<IPipeProcess> pipe = createPipeProcess(); pipe->setenv("ECLCCSERVER_THREAD_INDEX", idxStr.str()); Owned<IPropertyTreeIterator> options = globals->getElements("./Option"); ForEach(*options) { IPropertyTree &option = options->query(); const char *name = option.queryProp("@name"); const char *value = option.queryProp("@value"); const char *cluster = option.queryProp("@cluster"); // if cluster is set it's specific to a particular target if (name && (cluster==NULL || cluster[0]==0 || strcmp(cluster, targetCluster)==0)) processOption(name, value, eclccCmd, eclccProgName, *pipe, false); } eclccCmd.appendf(" -o%s", wuid); eclccCmd.appendf(" -platform=%s", target); Owned<IStringIterator> debugValues = &workunit->getDebugValues(); ForEach (*debugValues) { SCMStringBuffer debugStr, valueStr; debugValues->str(debugStr); workunit->getDebugValue(debugStr.str(), valueStr); processOption(debugStr.str(), valueStr.str(), eclccCmd, eclccProgName, *pipe, true); } if (workunit->getResultLimit()) { eclccCmd.appendf(" -fapplyInstantEclTransformations=1 -fapplyInstantEclTransformationsLimit=%u", workunit->getResultLimit()); } try { unsigned time = msTick(); Owned<ErrorReader> errorReader = new ErrorReader(pipe, this); eclccCmd.insert(0, eclccProgName); pipe->run(eclccProgName, eclccCmd, ".", true, false, true, 0); errorReader->start(); try { pipe->write(eclQuery.s.length(), eclQuery.s.str()); pipe->closeInput(); } catch (IException *e) { reportError(e); e->Release(); } unsigned retcode = pipe->wait(); errorReader->join(); if (retcode == 0) { StringBuffer realdllname, dllurl; realdllname.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension); StringBuffer realdllfilename(dllPath); realdllfilename.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension); StringBuffer wuXML; if (getWorkunitXMLFromFile(realdllfilename, wuXML)) { Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit(); embeddedWU->loadXML(wuXML); queryExtendedWU(workunit)->copyWorkUnit(embeddedWU, true); workunit->setIsClone(false); SCMStringBuffer jobname; if (embeddedWU->getJobName(jobname).length()) //let ECL win naming job during initial compile workunit->setJobName(jobname.str()); Owned<IWUQuery> query = workunit->updateQuery(); query->setQueryText(eclQuery.s.str()); } createUNCFilename(realdllfilename.str(), dllurl); unsigned crc = crc_file(realdllfilename.str()); Owned<IWUQuery> query = workunit->updateQuery(); associateLocalFile(query, FileTypeDll, realdllfilename, "Workunit DLL", crc); queryDllServer().registerDll(realdllname.str(), "Workunit DLL", dllurl.str()); time = msTick()-time; if (workunit->getDebugValueBool("addTimingToWorkunit", true)) workunit->setTimerInfo("eclccserver: create workunit", NULL, time, 1, 0); workunit->commit(); return true; } } catch (IException * e) { reportError(e); e->Release(); } return false; }