static int monitor (hashcat_ctx_t *hashcat_ctx)
{
hashes_t *hashes = hashcat_ctx->hashes;
hwmon_ctx_t *hwmon_ctx = hashcat_ctx->hwmon_ctx;
opencl_ctx_t *opencl_ctx = hashcat_ctx->opencl_ctx;
restore_ctx_t *restore_ctx = hashcat_ctx->restore_ctx;
status_ctx_t *status_ctx = hashcat_ctx->status_ctx;
user_options_t *user_options = hashcat_ctx->user_options;
bool runtime_check = false;
bool remove_check = false;
bool status_check = false;
bool restore_check = false;
bool hwmon_check = false;
bool performance_check = false;
const int sleep_time = 1;
const int temp_threshold = 1; // degrees celcius
const int fan_speed_min = 33; // in percentage
const int fan_speed_max = 100;
const double exec_low = 50.0; // in ms
const double util_low = 90.0; // in percent
if (user_options->runtime)
{
runtime_check = true;
}
if (restore_ctx->enabled == true)
{
restore_check = true;
}
if ((user_options->remove == true) && (hashes->hashlist_mode == HL_MODE_FILE))
{
remove_check = true;
}
if (user_options->status == true)
{
status_check = true;
}
if (hwmon_ctx->enabled == true)
{
hwmon_check = true;
}
if (hwmon_ctx->enabled == true)
{
performance_check = true; // this check simply requires hwmon to work
}
if ((runtime_check == false) && (remove_check == false) && (status_check == false) && (restore_check == false) && (hwmon_check == false) && (performance_check == false))
{
return 0;
}
// these variables are mainly used for fan control
int fan_speed_chgd[DEVICES_MAX];
memset (fan_speed_chgd, 0, sizeof (fan_speed_chgd));
// temperature controller "loopback" values
int temp_diff_old[DEVICES_MAX];
int temp_diff_sum[DEVICES_MAX];
memset (temp_diff_old, 0, sizeof (temp_diff_old));
memset (temp_diff_sum, 0, sizeof (temp_diff_sum));
// timer
hc_time_t last_temp_check_time;
hc_time (&last_temp_check_time);
u32 slowdown_warnings = 0;
u32 performance_warnings = 0;
u32 restore_left = user_options->restore_timer;
u32 remove_left = user_options->remove_timer;
u32 status_left = user_options->status_timer;
while (status_ctx->shutdown_inner == false)
{
sleep (sleep_time);
if (status_ctx->devices_status == STATUS_INIT) continue;
if (hwmon_check == true)
{
hc_thread_mutex_lock (status_ctx->mux_hwmon);
for (u32 device_id = 0; device_id < opencl_ctx->devices_cnt; device_id++)
{
hc_device_param_t *device_param = &opencl_ctx->devices_param[device_id];
if (device_param->skipped == true) continue;
const int rc_throttle = hm_get_throttle_with_device_id (hashcat_ctx, device_id);
if (rc_throttle == -1) continue;
if (rc_throttle > 0)
{
slowdown_warnings++;
if (slowdown_warnings == 1) EVENT_DATA (EVENT_MONITOR_THROTTLE1, &device_id, sizeof (u32));
if (slowdown_warnings == 2) EVENT_DATA (EVENT_MONITOR_THROTTLE2, &device_id, sizeof (u32));
if (slowdown_warnings == 3) EVENT_DATA (EVENT_MONITOR_THROTTLE3, &device_id, sizeof (u32));
}
else
{
slowdown_warnings = 0;
}
}
hc_thread_mutex_unlock (status_ctx->mux_hwmon);
}
if (hwmon_check == true)
{
hc_thread_mutex_lock (status_ctx->mux_hwmon);
hc_time_t temp_check_time;
hc_time (&temp_check_time);
u32 Ta = temp_check_time - last_temp_check_time; // set Ta = sleep_time; is not good enough (see --remove etc)
if (Ta == 0) Ta = 1;
for (u32 device_id = 0; device_id < opencl_ctx->devices_cnt; device_id++)
{
hc_device_param_t *device_param = &opencl_ctx->devices_param[device_id];
if (device_param->skipped == true) continue;
if ((opencl_ctx->devices_param[device_id].device_type & CL_DEVICE_TYPE_GPU) == 0) continue;
const int temperature = hm_get_temperature_with_device_id (hashcat_ctx, device_id);
if (temperature > (int) user_options->gpu_temp_abort)
{
EVENT_DATA (EVENT_MONITOR_TEMP_ABORT, &device_id, sizeof (u32));
myabort (hashcat_ctx);
}
if (hwmon_ctx->hm_device[device_id].fanspeed_get_supported == false) continue;
if (hwmon_ctx->hm_device[device_id].fanspeed_set_supported == false) continue;
const u32 gpu_temp_retain = user_options->gpu_temp_retain;
if (gpu_temp_retain > 0)
{
int temp_cur = temperature;
int temp_diff_new = (int) gpu_temp_retain - temp_cur;
temp_diff_sum[device_id] = temp_diff_sum[device_id] + temp_diff_new;
// calculate Ta value (time difference in seconds between the last check and this check)
last_temp_check_time = temp_check_time;
float Kp = 1.6f;
float Ki = 0.001f;
float Kd = 10.0f;
// PID controller (3-term controller: proportional - Kp, integral - Ki, derivative - Kd)
int fan_diff_required = (int) (Kp * (float)temp_diff_new + Ki * Ta * (float)temp_diff_sum[device_id] + Kd * ((float)(temp_diff_new - temp_diff_old[device_id])) / Ta);
if (abs (fan_diff_required) >= temp_threshold)
{
const int fan_speed_cur = hm_get_fanspeed_with_device_id (hashcat_ctx, device_id);
int fan_speed_level = fan_speed_cur;
if (fan_speed_chgd[device_id] == 0) fan_speed_level = temp_cur;
int fan_speed_new = fan_speed_level - fan_diff_required;
if (fan_speed_new > fan_speed_max) fan_speed_new = fan_speed_max;
if (fan_speed_new < fan_speed_min) fan_speed_new = fan_speed_min;
if (fan_speed_new != fan_speed_cur)
{
int freely_change_fan_speed = (fan_speed_chgd[device_id] == 1);
int fan_speed_must_change = (fan_speed_new > fan_speed_cur);
if ((freely_change_fan_speed == 1) || (fan_speed_must_change == 1))
{
if (device_param->device_vendor_id == VENDOR_ID_AMD)
{
if (hwmon_ctx->hm_adl)
{
hm_set_fanspeed_with_device_id_adl (hashcat_ctx, device_id, fan_speed_new, 1);
}
if (hwmon_ctx->hm_sysfs)
{
hm_set_fanspeed_with_device_id_sysfs (hashcat_ctx, device_id, fan_speed_new);
}
}
else if (device_param->device_vendor_id == VENDOR_ID_NV)
{
if (hwmon_ctx->hm_nvapi)
{
hm_set_fanspeed_with_device_id_nvapi (hashcat_ctx, device_id, fan_speed_new, 1);
}
if (hwmon_ctx->hm_xnvctrl)
{
hm_set_fanspeed_with_device_id_xnvctrl (hashcat_ctx, device_id, fan_speed_new);
}
}
fan_speed_chgd[device_id] = 1;
}
temp_diff_old[device_id] = temp_diff_new;
}
}
}
}
hc_thread_mutex_unlock (status_ctx->mux_hwmon);
}
if (restore_check == true)
{
restore_left--;
if (restore_left == 0)
{
const int rc = cycle_restore (hashcat_ctx);
if (rc == -1) return -1;
restore_left = user_options->restore_timer;
}
}
if ((runtime_check == true) && (status_ctx->runtime_start > 0))
{
const int runtime_left = get_runtime_left (hashcat_ctx);
if (runtime_left <= 0)
{
EVENT_DATA (EVENT_MONITOR_RUNTIME_LIMIT, NULL, 0);
myabort_runtime (hashcat_ctx);
}
}
if (remove_check == true)
{
remove_left--;
if (remove_left == 0)
{
if (hashes->digests_saved != hashes->digests_done)
{
hashes->digests_saved = hashes->digests_done;
const int rc = save_hash (hashcat_ctx);
if (rc == -1) return -1;
}
remove_left = user_options->remove_timer;
}
}
if (status_check == true)
{
status_left--;
if (status_left == 0)
{
hc_thread_mutex_lock (status_ctx->mux_display);
EVENT_DATA (EVENT_MONITOR_STATUS_REFRESH, NULL, 0);
hc_thread_mutex_unlock (status_ctx->mux_display);
status_left = user_options->status_timer;
}
}
if (performance_check == true)
{
int exec_cnt = 0;
int util_cnt = 0;
double exec_total = 0;
double util_total = 0;
hc_thread_mutex_lock (status_ctx->mux_hwmon);
for (u32 device_id = 0; device_id < opencl_ctx->devices_cnt; device_id++)
{
hc_device_param_t *device_param = &opencl_ctx->devices_param[device_id];
if (device_param->skipped == true) continue;
exec_cnt++;
const double exec = status_get_exec_msec_dev (hashcat_ctx, device_id);
exec_total += exec;
const int util = hm_get_utilization_with_device_id (hashcat_ctx, device_id);
if (util == -1) continue;
util_total += (double) util;
util_cnt++;
}
hc_thread_mutex_unlock (status_ctx->mux_hwmon);
double exec_avg = 0;
double util_avg = 0;
if (exec_cnt > 0) exec_avg = exec_total / exec_cnt;
if (util_cnt > 0) util_avg = util_total / util_cnt;
if ((exec_avg > 0) && (exec_avg < exec_low))
{
performance_warnings++;
if (performance_warnings == 10) EVENT_DATA (EVENT_MONITOR_PERFORMANCE_HINT, NULL, 0);
}
if ((util_avg > 0) && (util_avg < util_low))
{
performance_warnings++;
if (performance_warnings == 10) EVENT_DATA (EVENT_MONITOR_PERFORMANCE_HINT, NULL, 0);
}
}
}
// final round of save_hash
if (remove_check == true)
{
if (hashes->digests_saved != hashes->digests_done)
{
const int rc = save_hash (hashcat_ctx);
if (rc == -1) return -1;
}
}
// final round of cycle_restore
if (restore_check == true)
{
const int rc = cycle_restore (hashcat_ctx);
if (rc == -1) return -1;
}
return 0;
}
static int monitor (hashcat_ctx_t *hashcat_ctx)
{
hashes_t *hashes = hashcat_ctx->hashes;
hwmon_ctx_t *hwmon_ctx = hashcat_ctx->hwmon_ctx;
opencl_ctx_t *opencl_ctx = hashcat_ctx->opencl_ctx;
restore_ctx_t *restore_ctx = hashcat_ctx->restore_ctx;
status_ctx_t *status_ctx = hashcat_ctx->status_ctx;
user_options_t *user_options = hashcat_ctx->user_options;
bool runtime_check = false;
bool remove_check = false;
bool status_check = false;
bool restore_check = false;
bool hwmon_check = false;
bool performance_check = false;
const int sleep_time = 1;
const double exec_low = 50.0; // in ms
const double util_low = 90.0; // in percent
if (user_options->runtime)
{
runtime_check = true;
}
if (restore_ctx->enabled == true)
{
restore_check = true;
}
if ((user_options->remove == true) && (hashes->hashlist_mode == HL_MODE_FILE))
{
remove_check = true;
}
if (user_options->status == true)
{
status_check = true;
}
if (hwmon_ctx->enabled == true)
{
hwmon_check = true;
}
if (hwmon_ctx->enabled == true)
{
performance_check = true; // this check simply requires hwmon to work
}
if ((runtime_check == false) && (remove_check == false) && (status_check == false) && (restore_check == false) && (hwmon_check == false) && (performance_check == false))
{
return 0;
}
// timer
u32 slowdown_warnings = 0;
u32 performance_warnings = 0;
u32 restore_left = user_options->restore_timer;
u32 remove_left = user_options->remove_timer;
u32 status_left = user_options->status_timer;
while (status_ctx->shutdown_inner == false)
{
sleep (sleep_time);
if (status_ctx->devices_status == STATUS_INIT) continue;
if (hwmon_ctx->enabled == true)
{
hc_thread_mutex_lock (status_ctx->mux_hwmon);
for (u32 device_id = 0; device_id < opencl_ctx->devices_cnt; device_id++)
{
hc_device_param_t *device_param = &opencl_ctx->devices_param[device_id];
if (device_param->skipped == true) continue;
if ((opencl_ctx->devices_param[device_id].device_type & CL_DEVICE_TYPE_GPU) == 0) continue;
const int temperature = hm_get_temperature_with_device_id (hashcat_ctx, device_id);
if (temperature > (int) user_options->hwmon_temp_abort)
{
EVENT_DATA (EVENT_MONITOR_TEMP_ABORT, &device_id, sizeof (u32));
myabort (hashcat_ctx);
}
}
for (u32 device_id = 0; device_id < opencl_ctx->devices_cnt; device_id++)
{
hc_device_param_t *device_param = &opencl_ctx->devices_param[device_id];
if (device_param->skipped == true) continue;
const int rc_throttle = hm_get_throttle_with_device_id (hashcat_ctx, device_id);
if (rc_throttle == -1) continue;
if (rc_throttle > 0)
{
slowdown_warnings++;
if (slowdown_warnings == 1) EVENT_DATA (EVENT_MONITOR_THROTTLE1, &device_id, sizeof (u32));
if (slowdown_warnings == 2) EVENT_DATA (EVENT_MONITOR_THROTTLE2, &device_id, sizeof (u32));
if (slowdown_warnings == 3) EVENT_DATA (EVENT_MONITOR_THROTTLE3, &device_id, sizeof (u32));
}
else
{
if (slowdown_warnings > 0) slowdown_warnings--;
}
}
hc_thread_mutex_unlock (status_ctx->mux_hwmon);
}
if (restore_check == true)
{
restore_left--;
if (restore_left == 0)
{
const int rc = cycle_restore (hashcat_ctx);
if (rc == -1) return -1;
restore_left = user_options->restore_timer;
}
}
if ((runtime_check == true) && (status_ctx->runtime_start > 0))
{
const int runtime_left = get_runtime_left (hashcat_ctx);
if (runtime_left <= 0)
{
EVENT_DATA (EVENT_MONITOR_RUNTIME_LIMIT, NULL, 0);
myabort_runtime (hashcat_ctx);
}
}
if (remove_check == true)
{
remove_left--;
if (remove_left == 0)
{
if (hashes->digests_saved != hashes->digests_done)
{
hashes->digests_saved = hashes->digests_done;
const int rc = save_hash (hashcat_ctx);
if (rc == -1) return -1;
}
remove_left = user_options->remove_timer;
}
}
if (status_check == true)
{
status_left--;
if (status_left == 0)
{
hc_thread_mutex_lock (status_ctx->mux_display);
EVENT_DATA (EVENT_MONITOR_STATUS_REFRESH, NULL, 0);
hc_thread_mutex_unlock (status_ctx->mux_display);
status_left = user_options->status_timer;
}
}
if (performance_check == true)
{
int exec_cnt = 0;
int util_cnt = 0;
double exec_total = 0;
double util_total = 0;
hc_thread_mutex_lock (status_ctx->mux_hwmon);
for (u32 device_id = 0; device_id < opencl_ctx->devices_cnt; device_id++)
{
hc_device_param_t *device_param = &opencl_ctx->devices_param[device_id];
if (device_param->skipped == true) continue;
exec_cnt++;
const double exec = status_get_exec_msec_dev (hashcat_ctx, device_id);
exec_total += exec;
const int util = hm_get_utilization_with_device_id (hashcat_ctx, device_id);
if (util == -1) continue;
util_total += (double) util;
util_cnt++;
}
hc_thread_mutex_unlock (status_ctx->mux_hwmon);
double exec_avg = 0;
double util_avg = 0;
if (exec_cnt > 0) exec_avg = exec_total / exec_cnt;
if (util_cnt > 0) util_avg = util_total / util_cnt;
if ((exec_avg > 0) && (exec_avg < exec_low))
{
performance_warnings++;
if (performance_warnings == 10) EVENT_DATA (EVENT_MONITOR_PERFORMANCE_HINT, NULL, 0);
}
if ((util_avg > 0) && (util_avg < util_low))
{
performance_warnings++;
if (performance_warnings == 10) EVENT_DATA (EVENT_MONITOR_PERFORMANCE_HINT, NULL, 0);
}
}
// stdin read timeout check
// note: we skip the stdin timeout check if it was disabled with stdin_timeout_abort set to 0
if (user_options->stdin_timeout_abort != 0)
{
if (status_get_progress_done (hashcat_ctx) == 0)
{
if (status_ctx->stdin_read_timeout_cnt > 0)
{
if (status_ctx->stdin_read_timeout_cnt >= user_options->stdin_timeout_abort)
{
EVENT_DATA (EVENT_MONITOR_NOINPUT_ABORT, NULL, 0);
myabort (hashcat_ctx);
status_ctx->shutdown_inner = true;
break;
}
if ((status_ctx->stdin_read_timeout_cnt % STDIN_TIMEOUT_WARN) == 0)
{
EVENT_DATA (EVENT_MONITOR_NOINPUT_HINT, NULL, 0);
}
}
}
}
}
// final round of save_hash
if (remove_check == true)
{
if (hashes->digests_saved != hashes->digests_done)
{
const int rc = save_hash (hashcat_ctx);
if (rc == -1) return -1;
}
}
// final round of cycle_restore
if (restore_check == true)
{
const int rc = cycle_restore (hashcat_ctx);
if (rc == -1) return -1;
}
return 0;
}