void CMeasureNVML<TSkipMs, TVariant>::measure(void *pMsMeasurement, int32_t& rThreadNum) { nvmlReturn_t result; MS_MEASUREMENT_GPU *pMsMeasurementGpu = (MS_MEASUREMENT_GPU *) pMsMeasurement; result = nvmlDeviceGetPowerUsage(mDevice, &(pMsMeasurementGpu->nvml_power_cur)); if (NVML_SUCCESS != result) { mrLog.lock(); mrLog(CLogger::scErr) << "!!! 'nvml thread' (thread #" << rThreadNum << "): Error: no power usage reading possible. (file: " << __FILE__ << ", line: " << __LINE__ << ")" << std::endl; mrLog.unlock(); exit(EXIT_FAILURE); } if(TVariant == VARIANT_FULL) { nvmlMemory_t memory; if(!(mMeasureCounter++ % TSkipMs)) { result = nvmlDeviceGetMemoryInfo(mDevice, &memory); if (NVML_SUCCESS != result) { mrLog.lock(); mrLog(CLogger::scErr) << "!!! 'nvml thread' (thread #" << rThreadNum << "): Error: cannot obtain memory informations. (file: " << __FILE__ << ", line: " << __LINE__ << ")" << std::endl; mrLog.unlock(); exit(EXIT_FAILURE); } pMsMeasurementGpu->nvml_memory_free_cur = (uint32_t)(memory.free >> 10); pMsMeasurementGpu->nvml_memory_used_cur = (uint32_t)(memory.used >> 10); result = nvmlDeviceGetPerformanceState(mDevice, (nvmlPstates_t*)&(pMsMeasurementGpu->internal.nvml_power_state)); if (NVML_SUCCESS != result) { mrLog(CLogger::scErr) << "!!! 'nvml thread' (thread #" << rThreadNum << "): Error: no performance state reading possible. (file: " << __FILE__ << ", line: " << __LINE__ << ")" << std::endl; exit(EXIT_FAILURE); } nvmlTemperatureSensors_t sensorType = NVML_TEMPERATURE_GPU; result = nvmlDeviceGetTemperature(mDevice, sensorType, &(pMsMeasurementGpu->nvml_temperature_cur)); if (NVML_SUCCESS != result) { mrLog.lock(); mrLog(CLogger::scErr) << "!!! 'nvml thread' (thread #" << rThreadNum << "): Error: cannot read temperature. (file: " << __FILE__ << ", line: " << __LINE__ << ")" << std::endl; mrLog.unlock(); exit(EXIT_FAILURE); } result = nvmlDeviceGetClockInfo(mDevice, NVML_CLOCK_SM, &(pMsMeasurementGpu->nvml_clock_sm_cur)); if (NVML_SUCCESS != result) { mrLog.lock(); mrLog(CLogger::scErr) << "!!! 'nvml thread' (thread #" << rThreadNum << "): Error: cannot read frequency. (file: " << __FILE__ << ", line: " << __LINE__ << ")" << std::endl; mrLog.unlock(); exit(EXIT_FAILURE); } result = nvmlDeviceGetClockInfo(mDevice, NVML_CLOCK_MEM, &(pMsMeasurementGpu->nvml_clock_mem_cur)); if (NVML_SUCCESS != result) { mrLog.lock(); mrLog(CLogger::scErr) << "!!! 'nvml thread' (thread #" << rThreadNum << "): Error: cannot read frequency. (file: " << __FILE__ << ", line: " << __LINE__ << ")" << std::endl; mrLog.unlock(); exit(EXIT_FAILURE); } }
unsigned long long getPState( nvmlDevice_t dev ) { unsigned int ret = 0; nvmlPstates_t state = NVML_PSTATE_15; nvmlReturn_t bad; bad = nvmlDeviceGetPerformanceState( dev, &state ); if ( NVML_SUCCESS != bad ) { SUBDBG( "something went wrong %s\n", nvmlErrorString(bad)); } switch ( state ) { case NVML_PSTATE_15: ret++; case NVML_PSTATE_14: ret++; case NVML_PSTATE_13: ret++; case NVML_PSTATE_12: ret++; case NVML_PSTATE_11: ret++; case NVML_PSTATE_10: ret++; case NVML_PSTATE_9: ret++; case NVML_PSTATE_8: ret++; case NVML_PSTATE_7: ret++; case NVML_PSTATE_6: ret++; case NVML_PSTATE_5: ret++; case NVML_PSTATE_4: ret++; case NVML_PSTATE_3: ret++; case NVML_PSTATE_2: ret++; case NVML_PSTATE_1: ret++; case NVML_PSTATE_0: break; case NVML_PSTATE_UNKNOWN: default: /* This should never happen? * The API docs just state Unknown performance state... */ return (unsigned long long) -1; } return (unsigned long long)ret; }
void CMeasureNVML<TSkipMs, TVariant>::init(void) { if(TVariant == VARIANT_FULL) { mrLog() << ">>> 'nvml' (full version)" << std::endl; } else { mrLog() << ">>> 'nvml' (light version)" << std::endl; } nvmlReturn_t result; int32_t rv; char const* args_set_pm[] = {"gpu_management", "-p 1", NULL}; uint32_t device_count; char name[NVML_DEVICE_NAME_BUFFER_SIZE]; nvmlPciInfo_t pci; nvmlEnableState_t mode; std::string modes[2] = {"disabled", "enabled"}; std::stringstream clk_gpu_str; std::stringstream clk_mem_str; nvmlPstates_t power_state; nvmlMemory_t memory; const uint32_t count = 32; uint32_t clk_mem_cnt = count; uint32_t clk_mem[count]; uint32_t clk_mem_max = 0; uint32_t clk_mem_min = 0xffffffff; uint32_t clk_mem_set = 0; uint32_t clk_gpu_min_arr_cnt = count; uint32_t clk_gpu_min_arr[clk_gpu_min_arr_cnt]; uint32_t clk_gpu_min = 0xffffffff; uint32_t clk_gpu_max_arr_cnt = count; uint32_t clk_gpu_max_arr[clk_gpu_max_arr_cnt]; uint32_t clk_gpu_max = 0; uint32_t clk_gpu_set = 0; uint32_t memory_total = 0; result = nvmlInit(); if (NVML_SUCCESS != result) { mrLog(CLogger::scErr) << "!!! 'nvml' (thread main): Error: cannot initialize nvml library. (file: " << __FILE__ << ", line: " << __LINE__ << ")" << std::endl; exit(EXIT_FAILURE); } result = nvmlDeviceGetCount(&device_count); if (NVML_SUCCESS != result) { mrLog(CLogger::scErr) << "!!! 'nvml' (thread main): Error: cannot query device count. (file: " << __FILE__ << ", line: " << __LINE__ << ")" << std::endl; exit(EXIT_FAILURE); } if (device_count > 1) { mrLog(CLogger::scErr) << "!!! 'nvml' (thread main): Error: this software has be rewritten if you want to support more than 1 device. (file: " << __FILE__ << ", line: " << __LINE__ << ")" << std::endl; exit(EXIT_FAILURE); } mrLog() << ">>> 'nvml' (thread main): get gpu device handler..."; mrLog.flush(); result = nvmlDeviceGetHandleByIndex(0, &mDevice); if (NVML_SUCCESS != result) { mrLog(CLogger::scErr) << "!!! 'nvml' (thread main): Error: cannot get device handler. (file: " << __FILE__ << ", line: " << __LINE__ << ")" << std::endl; exit(EXIT_FAILURE); } mrLog() << " done!" << std::endl; result = nvmlDeviceGetName(mDevice, name, NVML_DEVICE_NAME_BUFFER_SIZE); if (NVML_SUCCESS != result) { mrLog(CLogger::scErr) << "!!! 'nvml' (thread main): Error: cannot get device name. (file: " << __FILE__ << ", line: " << __LINE__ << ")" << std::endl; exit(EXIT_FAILURE); } result = nvmlDeviceGetPciInfo(mDevice, &pci); if (NVML_SUCCESS != result) { mrLog(CLogger::scErr) << "!!! 'nvml' (thread main): Error: cannot get pci information. (file: " << __FILE__ << ", line: " << __LINE__ << ")" << std::endl; exit(EXIT_FAILURE); } result = nvmlDeviceGetPowerManagementMode(mDevice, &mode); if (NVML_SUCCESS != result) { mrLog(CLogger::scErr) << "!!! 'nvml' (thread main): Error: no power managment supported. (file: " << __FILE__ << ", line: " << __LINE__ << ")" << std::endl; exit(EXIT_FAILURE); } result = nvmlDeviceGetPerformanceState(mDevice, &power_state); if (NVML_SUCCESS != result) { mrLog(CLogger::scErr) << "!!! 'nvml' (thread main): Error: no performance state reading possible. (file: " << __FILE__ << ", line: " << __LINE__ << ")" << std::endl; exit(EXIT_FAILURE); } result = nvmlDeviceGetSupportedMemoryClocks(mDevice, &clk_mem_cnt, clk_mem); if (NVML_SUCCESS != result) { mrLog(CLogger::scErr) << "!!! 'nvml' (thread main): Error: cannot obtain memory clock. (file: " << __FILE__ << ", line: " << __LINE__ << ")" << std::endl; exit(EXIT_FAILURE); } for (int i=0; i<(int32_t)clk_mem_cnt; ++i) { clk_mem_min = (clk_mem[i]<clk_mem_min) ? clk_mem[i] : clk_mem_min; clk_mem_max = (clk_mem[i]>clk_mem_max) ? clk_mem[i] : clk_mem_max; } result = nvmlDeviceGetSupportedGraphicsClocks(mDevice, clk_mem_min, &clk_gpu_min_arr_cnt, clk_gpu_min_arr); if (NVML_SUCCESS != result) { mrLog(CLogger::scErr) << "!!! 'nvml' (thread main): Error: cannot obtain graphics clock. (file: " << __FILE__ << ", line: " << __LINE__ << ")" << std::endl; exit(EXIT_FAILURE); } for (int32_t i=0; i<(int32_t)clk_gpu_min_arr_cnt; ++i) { clk_gpu_min = (clk_gpu_min_arr[i]<clk_gpu_min) ? clk_gpu_min_arr[i] : clk_gpu_min; } result = nvmlDeviceGetSupportedGraphicsClocks(mDevice, clk_mem_max, &clk_gpu_max_arr_cnt, clk_gpu_max_arr); if (NVML_SUCCESS != result) { mrLog(CLogger::scErr) << "!!! 'nvml' (thread main): Error: cannot obtain graphics clock. (file: " << __FILE__ << ", line: " << __LINE__ << ")" << std::endl; exit(EXIT_FAILURE); } for (int32_t i=0; i<(int32_t)clk_gpu_max_arr_cnt; ++i) { clk_gpu_max = (clk_gpu_max_arr[i]>clk_gpu_max) ? clk_gpu_max_arr[i] : clk_gpu_max; } result = nvmlDeviceGetMemoryInfo(mDevice, &memory); if (NVML_SUCCESS != result) { mrLog.lock(); mrLog(CLogger::scErr) << "!!! 'nvml thread' (thread main): Error: cannot obtain memory informations. (file: " << __FILE__ << ", line: " << __LINE__ << ")" << std::endl; mrLog.unlock(); exit(EXIT_FAILURE); } memory_total = (uint32_t)(memory.total >> 20); rv = exec_gpu_mgmt((char**)args_set_pm); if (rv) { mrLog(CLogger::scErr) << "!!! 'nvml' (thread main): Error: in gpu_management tool. (file: " << __FILE__ << ", line: " << __LINE__ << ")" << std::endl; exit(EXIT_FAILURE); } mrLog() << ">>> 'nvml' (thread main): persistence mode enabled." << std::endl; mrLog() << ">>> 'nvml' (thread main):" << std::endl << " device : " << name << std::endl << " pcie : " << pci.busId << std::endl << " power mgmt mode: " << modes[mode] << std::endl << " power state cur: " << power_state << std::endl << " power state min: " << NVML_PSTATE_15 << std::endl << " power state max: " << NVML_PSTATE_0 << std::endl << " memory total : " << memory_total << " MiB" << std::endl << " avail mem clks : "; for (int i=0; i<(int32_t)clk_mem_cnt; ++i) { if (i<(int32_t)clk_mem_cnt-1) { mrLog() << clk_mem[i] << " MHz, "; } else { mrLog() << clk_mem[i] << " MHz" << std::endl; } } mrLog() << " memory clk min : " << clk_mem_min << " MHz" << std::endl << " avail core clks: "; for (int32_t i=0; i<(int32_t)clk_gpu_min_arr_cnt; ++i) { if (i<(int32_t)clk_gpu_min_arr_cnt-1) { mrLog() << clk_gpu_min_arr[i] << " MHz, "; } else { mrLog() << clk_gpu_min_arr[i] << " MHz" << std::endl; } } mrLog() << " core clk min : " << clk_gpu_min << " MHz" << std::endl; mrLog() << " memory clk max : " << clk_mem_max << " MHz" << std::endl << " avail core clks: "; for (int32_t i=0; i<(int32_t)clk_gpu_max_arr_cnt; ++i) { if (i<(int32_t)clk_gpu_max_arr_cnt-1) { mrLog() << clk_gpu_max_arr[i] << " MHz, "; } else { mrLog() << clk_gpu_max_arr[i] << " MHz" << std::endl; } } mrLog() << " core clk max : " << clk_gpu_max << " MHz" << std::endl; switch (mGpuFrequency) { case GPU_FREQUENCY_MIN: clk_mem_set = clk_mem_min; clk_gpu_set = clk_gpu_min; break; case GPU_FREQUENCY_MAX: clk_mem_set = clk_mem_max; clk_gpu_set = clk_gpu_max; break; case GPU_FREQUENCY_CUR: default: clk_mem_set = 0; clk_gpu_set = 0; result = nvmlDeviceGetClockInfo(mDevice, NVML_CLOCK_MEM, &clk_mem_set); if (NVML_SUCCESS != result) { mrLog.lock(); mrLog(CLogger::scErr) << "!!! 'nvml thread' (thread main): Error: cannot read frequency. (file: " << __FILE__ << ", line: " << __LINE__ << ")" << std::endl; mrLog.unlock(); exit(EXIT_FAILURE); } result = nvmlDeviceGetClockInfo(mDevice, NVML_CLOCK_GRAPHICS, &clk_gpu_set); if (NVML_SUCCESS != result) { mrLog.lock(); mrLog(CLogger::scErr) << "!!! 'nvml thread' (thread main): Error: cannot read frequency. (file: " << __FILE__ << ", line: " << __LINE__ << ")" << std::endl; mrLog.unlock(); exit(EXIT_FAILURE); } break; } if (mGpuFrequency == GPU_FREQUENCY_MIN || mGpuFrequency == GPU_FREQUENCY_MAX) { // In these cases we actually set the GPU frequencies either to the maximum or minimum value. clk_gpu_str << "-c " << clk_gpu_set; clk_mem_str << "-m " << clk_mem_set; char const* args_set_clk[] = {"gpu_management", clk_gpu_str.str().c_str() , clk_mem_str.str().c_str(), NULL}; rv = exec_gpu_mgmt((char**)args_set_clk); if (rv) { mrLog(CLogger::scErr) << "!!! 'nvml' (thread main): Error: in gpu_management tool. (file: " << __FILE__ << ", line: " << __LINE__ << ")" << std::endl; exit(EXIT_FAILURE); } mrLog() << ">>> 'nvml' (thread main): set core clk to " << clk_gpu_set << " MHz and mem clk to " << clk_mem_set << " MHz." << std::endl; } else { // We name the values *_set, but we don't set the frequency. We just print the current GPU frequency. mrLog() << ">>> 'nvml' (thread main): current core clk is " << clk_gpu_set << " MHz and mem clk is " << clk_mem_set << " MHz." << std::endl; } mrLog() << ">>> 'nvml' (thread main): wait for 15s to throttle gpu clocks." << std::endl; sleep(15); mrLog() << ">>> 'nvml' (thread main): initialization done." << std::endl << std::endl; }