int add_mic_status(
dynamic_string *status)
{
COIENGINE engine[MAX_ENGINES];
uint32_t num_engines = 0;
uint32_t i;
struct COI_ENGINE_INFO mic_stat[MAX_ENGINES];
if (COIEngineGetCount(COI_ISA_MIC, &num_engines) != COI_SUCCESS)
{
log_err(-1, __func__, "Mics are present but apparently not configured correctly - can't get count");
return(PBSE_SYSTEM);
}
copy_to_end_of_dynamic_string(status, START_MIC_STATUS);
for (i = 0; i < num_engines; i++)
{
if (COIEngineGetHandle(COI_ISA_MIC, i, &engine[i]) != COI_SUCCESS)
{
snprintf(log_buffer, sizeof(log_buffer), "Can't get handle for mic index %d", (int)i);
log_err(-1, __func__, log_buffer);
copy_to_end_of_dynamic_string(status, END_MIC_STATUS);
return(PBSE_SYSTEM);
}
if (COIEngineGetInfo(engine[i], sizeof(struct COI_ENGINE_INFO), &mic_stat[i]) != COI_SUCCESS)
{
snprintf(log_buffer, sizeof(log_buffer), "Can't get information for mic index %d", (int)i);
log_err(-1, __func__, log_buffer);
copy_to_end_of_dynamic_string(status, END_MIC_STATUS);
return(PBSE_SYSTEM);
}
add_single_mic_info(status, &mic_stat[i]);
}
copy_to_end_of_dynamic_string(status, END_MIC_STATUS);
return(PBSE_NONE);
} /* END add_mic_status() */
int add_mic_status(
dynamic_string *status)
{
COIENGINE engine[MAX_ENGINES];
uint32_t num_engines = 0;
uint32_t i;
struct COI_ENGINE_INFO mic_stat[MAX_ENGINES];
#ifdef NUMA_SUPPORT
/* does this node board have mics configured? */
if (node_boards[numa_index].mic_end_index < 0)
return(PBSE_NONE);
#endif
if (COIEngineGetCount(COI_ISA_MIC, &num_engines) != COI_SUCCESS)
{
log_err(-1, __func__, "Mics are present but apparently not configured correctly - can't get count");
return(PBSE_SYSTEM);
}
copy_to_end_of_dynamic_string(status, START_MIC_STATUS);
#ifdef NUMA_SUPPORT
if (num_engines < node_boards[numa_index].mic_end_index)
{
snprintf(log_buffer, sizeof(log_buffer),
"node board %d is supposed to have mic range %d-%d but there are only %d mics",
numa_index, node_boards[numa_index].mic_start_index,
node_boards[numa_index].mic_end_index, num_engines);
log_err(-1, __func__, log_buffer);
return(PBSE_SYSTEM);
}
for (i = node_boards[numa_index].mic_start_index; i <= node_boards[numa_index].mic_end_index; i++)
#else
for (i = 0; i < num_engines; i++)
#endif
{
if (COIEngineGetHandle(COI_ISA_MIC, i, &engine[i]) != COI_SUCCESS)
{
snprintf(log_buffer, sizeof(log_buffer), "Can't get handle for mic index %d", (int)i);
log_err(-1, __func__, log_buffer);
copy_to_end_of_dynamic_string(status, END_MIC_STATUS);
return(PBSE_SYSTEM);
}
if (COIEngineGetInfo(engine[i], sizeof(struct COI_ENGINE_INFO), &mic_stat[i]) != COI_SUCCESS)
{
snprintf(log_buffer, sizeof(log_buffer), "Can't get information for mic index %d", (int)i);
log_err(-1, __func__, log_buffer);
copy_to_end_of_dynamic_string(status, END_MIC_STATUS);
return(PBSE_SYSTEM);
}
add_single_mic_info(status, &mic_stat[i]);
}
copy_to_end_of_dynamic_string(status, END_MIC_STATUS);
return(PBSE_NONE);
} /* END add_mic_status() */
int add_mic_status(
std::vector<std::string> &status)
{
uint32_t num_engines = 0;
uint32_t i = 0;
#ifdef NUMA_SUPPORT
/* does this node board have mics configured? */
if (node_boards[numa_index].mic_end_index < 0)
return(PBSE_NONE);
#endif
if (COIEngineGetCount(COI_ISA_MIC, &num_engines) != COI_SUCCESS)
{
log_err(-1, __func__, "Mics are present but apparently not configured correctly - can't get count");
return(PBSE_SYSTEM);
}
status.push_back(START_MIC_STATUS);
#ifdef NUMA_SUPPORT
if (num_engines < node_boards[numa_index].mic_end_index)
{
snprintf(log_buffer, sizeof(log_buffer),
"node board %d is supposed to have mic range %d-%d but there are only %d mics",
numa_index, node_boards[numa_index].mic_start_index,
node_boards[numa_index].mic_end_index, num_engines);
log_err(-1, __func__, log_buffer);
return(PBSE_SYSTEM);
}
for (i = node_boards[numa_index].mic_start_index; i <= node_boards[numa_index].mic_end_index; i++)
#else
for (i = 0; i < num_engines; i++)
#endif
{
COIENGINE engine;
struct COI_ENGINE_INFO mic_stat;
memset(&engine, 0, sizeof(engine));
memset(&mic_stat, 0, sizeof(mic_stat));
if (COIEngineGetHandle(COI_ISA_MIC, i, &engine) != COI_SUCCESS)
{
if (down_mics.find(i) == down_mics.end())
{
snprintf(log_buffer, sizeof(log_buffer), "Can't get handle for mic index %d", (int)i);
log_event(PBSEVENT_SYSTEM, PBS_EVENTCLASS_SERVER, __func__, log_buffer);
down_mics.insert(i);
}
continue;
}
if (COIEngineGetInfo(engine, sizeof(struct COI_ENGINE_INFO), &mic_stat) != COI_SUCCESS)
{
snprintf(log_buffer, sizeof(log_buffer), "Can't get information for mic index %d", (int)i);
log_err(-1, __func__, log_buffer);
continue;
}
add_single_mic_info(status, &mic_stat);
}
status.push_back(END_MIC_STATUS);
return(PBSE_NONE);
} /* END add_mic_status() */
int check_for_mics(
uint32_t& num_engines)
{
uint32_t i = 0;
#ifdef NUMA_SUPPORT
/* does this node board have mics configured? */
if (node_boards[numa_index].mic_end_index < 0)
return(PBSE_NONE);
#endif
if (COIEngineGetCount(COI_ISA_MIC, &num_engines) != COI_SUCCESS)
{
log_err(-1, __func__, "Mics are present but apparently not configured correctly - can't get count");
return(PBSE_SYSTEM);
}
#ifdef NUMA_SUPPORT
if (num_engines < node_boards[numa_index].mic_end_index)
{
snprintf(log_buffer, sizeof(log_buffer),
"node board %d is supposed to have mic range %d-%d but there are only %d mics",
numa_index, node_boards[numa_index].mic_start_index,
node_boards[numa_index].mic_end_index, num_engines);
log_err(-1, __func__, log_buffer);
return(PBSE_SYSTEM);
}
for (i = node_boards[numa_index].mic_start_index; i <= node_boards[numa_index].mic_end_index; i++)
#else
for (i = 0; i < num_engines; i++)
#endif
{
int rc;
COIENGINE engine;
struct COI_ENGINE_INFO mic_stat;
std::set<int>::iterator it;
memset(&engine, 0, sizeof(engine));
memset(&mic_stat, 0, sizeof(mic_stat));
rc = COIEngineGetHandle(COI_ISA_MIC, i, &engine);
if (rc != COI_SUCCESS)
{
it = down_mics.find(i);
if (it == down_mics.end())
{
snprintf(log_buffer, sizeof(log_buffer), "Can't get handle for mic index %d", (int)i);
log_event(PBSEVENT_SYSTEM,PBS_EVENTCLASS_SERVER, __func__, log_buffer);
down_mics.insert(i);
}
continue;
}
else
{
it = down_mics.find(i);
if (it != down_mics.end())
{
/* if we made it here we have the mic again. remove it from the down_mics set */
snprintf(log_buffer, sizeof(log_buffer), "handle for mic index %d is back online", (int)i);
log_event(PBSEVENT_SYSTEM,PBS_EVENTCLASS_SERVER, __func__, log_buffer);
down_mics.erase(it);
}
}
}
return(PBSE_NONE);
}
void init(const char* cfg)
{
/* get number of Xeon Phi devices */
uint32_t engines = 0;
COIEngineGetCount( COI_ISA_MIC, &engines );
if (engines == 0) throw std::runtime_error("No Xeon Phi device found.");
/* get engine handle */
COIRESULT result;
result = COIEngineGetHandle( COI_ISA_MIC, 0, &engine );
if (result != COI_SUCCESS)
throw std::runtime_error("Failed to load engine number " + std::stringOf(0) + ": " + COIResultGetName(result));
/* print info of engine */
COI_ENGINE_INFO info;
result = COIEngineGetInfo(engine,sizeof(info),&info);
std::cout << "Found Xeon Phi device with " << info.NumCores << " cores and " << (info.PhysicalMemory/1024/1024) << "MB memory" << std::endl;
/* create process */
const std::string executable = std::string(tutorialName)+"_xeonphi_device";
result = COIProcessCreateFromFile
(engine,
executable.c_str(), // The local path to the sink side binary to launch.
0, NULL, // argc and argv for the sink process.
false, NULL, // Environment variables to set for the sink process.
true, NULL, // Enable the proxy but don't specify a proxy root path.
0, // The amount of memory to reserve for COIBuffers.
NULL, // Path to search for dependencies
&process // The resulting process handle.
);
if (result != COI_SUCCESS)
throw std::runtime_error("Failed to create process " + std::string(executable) +": " + COIResultGetName(result));
/* create pipeline */
COI_CPU_MASK cpuMask;
result = COIPipelineClearCPUMask(&cpuMask);
if (result != COI_SUCCESS)
throw std::runtime_error(std::string("COIPipelineClearCPUMask failed: ") + COIResultGetName(result));
result = COIPipelineSetCPUMask(process,info.NumCores-1,0,&cpuMask);
result = COIPipelineSetCPUMask(process,info.NumCores-1,1,&cpuMask);
result = COIPipelineSetCPUMask(process,info.NumCores-1,2,&cpuMask);
result = COIPipelineSetCPUMask(process,info.NumCores-1,3,&cpuMask);
if (result != COI_SUCCESS)
throw std::runtime_error(std::string("COIPipelineSetCPUMask failed: ") + COIResultGetName(result));
result = COIPipelineCreate(process,cpuMask,0,&pipeline);
if (result != COI_SUCCESS)
throw std::runtime_error(std::string("COIPipelineCreate failed: ") + COIResultGetName(result));
/* get run functions */
const char *fctNameArray[8] = { "run_init", "run_key_pressed", "run_create_mesh", "run_create_hairset", "run_create_scene", "run_pick", "run_render", "run_cleanup" };
result = COIProcessGetFunctionHandles (process, 8, fctNameArray, &runInit);
if (result != COI_SUCCESS)
throw std::runtime_error("COIProcessGetFunctionHandles failed: "+std::string(COIResultGetName(result)));
/* run init runfunction */
InitData parms;
strncpy(parms.cfg,cfg,sizeof(parms.cfg));
result = COIPipelineRunFunction (pipeline, runInit, 0, NULL, NULL, 0, NULL, &parms, sizeof(parms), NULL, 0, NULL);
if (result != COI_SUCCESS)
throw std::runtime_error("COIPipelineRunFunction failed: "+std::string(COIResultGetName(result)));
}
void device_t<COI>::setup(const int device, const int memoryAllocated){
data = new COIDeviceData_t;
OCCA_EXTRACT_DATA(COI, Device);
uint32_t deviceCount;
OCCA_COI_CHECK("Device: Get Count",
COIEngineGetCount(COI_ISA_MIC, &deviceCount));
OCCA_CHECK(device < deviceCount);
OCCA_COI_CHECK("Device: Get Handle",
COIEngineGetHandle(COI_ISA_MIC, device, &data_.deviceID) );
std::stringstream salt;
salt << "COI"
<< occaCOIMain;
std::string cachedBinary = getCachedName("occaCOIMain", salt.str());
struct stat buffer;
bool fileExists = (stat(cachedBinary.c_str(), &buffer) == 0);
if(fileExists)
std::cout << "Found cached binary of [occaCOIMain] in [" << cachedBinary << "]\n";
else{
//---[ Write File ]-----------------
std::string prefix, name;
getFilePrefixAndName(cachedBinary, prefix, name);
const std::string iCachedBinary = prefix + "i_" + name;
if(haveFile(cachedBinary)){
std::cout << "Making [" << iCachedBinary << "]\n";
std::ofstream fs;
fs.open(iCachedBinary.c_str());
fs << occaCOIMain;
fs.close();
std::stringstream command;
command << dev->dHandle->compiler
<< " -o " << cachedBinary
<< " -x c++"
<< ' ' << dev->dHandle->compilerFlags
<< ' ' << iCachedBinary;
const std::string &sCommand = command.str();
std::cout << "Compiling [" << functionName << "]\n" << sCommand << "\n\n";
system(sCommand.c_str());
releaseFile(cachedBinary);
}
else
waitForFile(cachedBinary);
}
// [-] Tentative
std::string SINK_LD_LIBRARY_PATH;
char *c_SINK_LD_LIBRARY_PATH = getenv("SINK_LD_LIBRARY_PATH");
if(c_SINK_LD_LIBRARY_PATH != NULL)
SINK_LD_LIBRARY_PATH = std::string(c_SINK_LD_LIBRARY_PATH);
OCCA_COI_CHECK("Device: Initializing",
COIProcessCreateFromFile(data_.deviceID,
cachedBinary.c_str(),
0 , NULL,
true, NULL,
true, NULL,
memoryAllocated ? memoryAllocated : (4 << 30), // 4 GB
SINK_LD_LIBRARY_PATH.c_str(),
&(data_.chiefID)) );
const char *kernelNames[] = {"occaKernelWith1Argument" , "occaKernelWith2Arguments" , "occaKernelWith3Arguments" ,
"occaKernelWith4Arguments" , "occaKernelWith5Arguments" , "occaKernelWith6Arguments" ,
"occaKernelWith7Arguments" , "occaKernelWith8Arguments" , "occaKernelWith9Arguments" ,
"occaKernelWith10Arguments", "occaKernelWith11Arguments", "occaKernelWith12Arguments",
"occaKernelWith13Arguments", "occaKernelWith14Arguments", "occaKernelWith15Arguments",
"occaKernelWith16Arguments", "occaKernelWith17Arguments", "occaKernelWith18Arguments",
"occaKernelWith19Arguments", "occaKernelWith20Arguments", "occaKernelWith21Arguments",
"occaKernelWith22Arguments", "occaKernelWith23Arguments", "occaKernelWith24Arguments",
"occaKernelWith25Arguments"};
// [-] More hard-coding, if you know what I mean
OCCA_COI_CHECK("Device: Getting Kernel Wrappers",
COIProcessGetFunctionHandles(data_.chiefID,
25,
kernelNames,
data_.kernelWrapper));
}
