/***************** * Local Functions *****************/ static void default_hnp_abort(orte_job_t *jdata) { int rc; /* if we are already in progress, then ignore this call */ if (opal_atomic_trylock(&orte_abort_inprogress_lock)) { /* returns 1 if already locked */ OPAL_OUTPUT_VERBOSE((1, orte_errmgr_base.output, "%s errmgr:default_hnp: abort in progress, ignoring abort on job %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_JOBID_PRINT(jdata->jobid))); return; } OPAL_OUTPUT_VERBOSE((1, orte_errmgr_base.output, "%s errmgr:default_hnp: abort called on job %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_JOBID_PRINT(jdata->jobid))); /* the job aborted - turn off any sensors on this job */ orte_sensor.stop(jdata->jobid); /* set control params to indicate we are terminating */ orte_job_term_ordered = true; orte_enable_recovery = false; /* if it is the daemon job that aborted, then we need * to flag an abnormal term - otherwise, just abort * the job cleanly */ if (ORTE_PROC_MY_NAME->jobid == jdata->jobid) { orte_abnormal_term_ordered = true; } if (0 < jdata->num_non_zero_exit) { /* warn user */ opal_output(orte_clean_output, "-------------------------------------------------------\n" "%s job %s terminated normally, but %d %s. Per user-direction, the job has been aborted.\n" "-------------------------------------------------------", (1 == ORTE_LOCAL_JOBID(jdata->jobid)) ? "Primary" : "Child", (1 == ORTE_LOCAL_JOBID(jdata->jobid)) ? "" : ORTE_LOCAL_JOBID_PRINT(jdata->jobid), jdata->num_non_zero_exit, (1 == jdata->num_non_zero_exit) ? "process returned\na non-zero exit code." : "processes returned\nnon-zero exit codes."); } OPAL_OUTPUT_VERBOSE((1, orte_errmgr_base.output, "%s errmgr:default_hnp: ordering orted termination", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME))); /* tell the plm to terminate the orteds - they will automatically * kill their local procs */ if (ORTE_SUCCESS != (rc = orte_plm.terminate_orteds())) { ORTE_ERROR_LOG(rc); } }
/***************** * Local Functions *****************/ static void default_hnp_abort(orte_job_t *jdata) { int rc; int32_t i32, *i32ptr; /* if we are already in progress, then ignore this call */ if (opal_atomic_trylock(&orte_abort_inprogress_lock)) { /* returns 1 if already locked */ OPAL_OUTPUT_VERBOSE((1, orte_errmgr_base_framework.framework_output, "%s errmgr:default_hnp: abort in progress, ignoring abort on job %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_JOBID_PRINT(jdata->jobid))); return; } OPAL_OUTPUT_VERBOSE((1, orte_errmgr_base_framework.framework_output, "%s errmgr:default_hnp: abort called on job %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_JOBID_PRINT(jdata->jobid))); /* set control params to indicate we are terminating */ orte_job_term_ordered = true; orte_enable_recovery = false; /* if it is the daemon job that aborted, then we need * to flag an abnormal term - otherwise, just abort * the job cleanly */ if (ORTE_PROC_MY_NAME->jobid == jdata->jobid) { orte_abnormal_term_ordered = true; } i32 = 0; i32ptr = &i32; if (orte_get_attribute(&jdata->attributes, ORTE_JOB_NUM_NONZERO_EXIT, (void**)&i32ptr, OPAL_INT32)) { /* warn user */ opal_output(orte_clean_output, "-------------------------------------------------------\n" "%s job %s terminated normally, but %d %s. Per user-direction, the job has been aborted.\n" "-------------------------------------------------------", (1 == ORTE_LOCAL_JOBID(jdata->jobid)) ? "Primary" : "Child", (1 == ORTE_LOCAL_JOBID(jdata->jobid)) ? "" : ORTE_LOCAL_JOBID_PRINT(jdata->jobid), i32, (1 == i32) ? "process returned\na non-zero exit code" : "processes returned\nnon-zero exit codes"); } OPAL_OUTPUT_VERBOSE((1, orte_errmgr_base_framework.framework_output, "%s errmgr:default_hnp: ordering orted termination", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME))); /* tell the plm to terminate the orteds - they will automatically * kill their local procs */ if (ORTE_SUCCESS != (rc = orte_plm.terminate_orteds())) { ORTE_ERROR_LOG(rc); } }
void orte_state_base_check_all_complete(int fd, short args, void *cbdata) { orte_state_caddy_t *caddy = (orte_state_caddy_t*)cbdata; orte_job_t *jdata = caddy->jdata; orte_proc_t *proc; int i; orte_std_cntr_t j; orte_job_t *job; orte_node_t *node; orte_job_map_t *map; orte_std_cntr_t index; bool one_still_alive; orte_vpid_t lowest=0; int32_t i32, *i32ptr; opal_output_verbose(2, orte_state_base_framework.framework_output, "%s state:base:check_job_complete on job %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), (NULL == jdata) ? "NULL" : ORTE_JOBID_PRINT(jdata->jobid)); if (NULL == jdata || jdata->jobid == ORTE_PROC_MY_NAME->jobid) { /* just check to see if the daemons are complete */ OPAL_OUTPUT_VERBOSE((2, orte_state_base_framework.framework_output, "%s state:base:check_job_complete - received NULL job, checking daemons", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME))); goto CHECK_DAEMONS; } else { /* mark the job as terminated, but don't override any * abnormal termination flags */ if (jdata->state < ORTE_JOB_STATE_UNTERMINATED) { jdata->state = ORTE_JOB_STATE_TERMINATED; } } /* tell the IOF that the job is complete */ if (NULL != orte_iof.complete) { orte_iof.complete(jdata); } i32ptr = &i32; if (orte_get_attribute(&jdata->attributes, ORTE_JOB_NUM_NONZERO_EXIT, (void**)&i32ptr, OPAL_INT32) && !orte_abort_non_zero_exit) { if (!orte_report_child_jobs_separately || 1 == ORTE_LOCAL_JOBID(jdata->jobid)) { /* update the exit code */ ORTE_UPDATE_EXIT_STATUS(lowest); } /* warn user */ opal_output(orte_clean_output, "-------------------------------------------------------\n" "While %s job %s terminated normally, %d %s. Further examination may be required.\n" "-------------------------------------------------------", (1 == ORTE_LOCAL_JOBID(jdata->jobid)) ? "the primary" : "child", (1 == ORTE_LOCAL_JOBID(jdata->jobid)) ? "" : ORTE_LOCAL_JOBID_PRINT(jdata->jobid), i32, (1 == i32) ? "process returned\na non-zero exit code." : "processes returned\nnon-zero exit codes."); } OPAL_OUTPUT_VERBOSE((2, orte_state_base_framework.framework_output, "%s state:base:check_job_completed declared job %s terminated with state %s - checking all jobs", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_JOBID_PRINT(jdata->jobid), orte_job_state_to_str(jdata->state))); /* if this job is a continuously operating one, then don't do * anything further - just return here */ if (NULL != jdata && (orte_get_attribute(&jdata->attributes, ORTE_JOB_CONTINUOUS_OP, NULL, OPAL_BOOL) || ORTE_FLAG_TEST(jdata, ORTE_JOB_FLAG_RECOVERABLE))) { goto CHECK_ALIVE; } /* if the job that is being checked is the HNP, then we are * trying to terminate the orteds. In that situation, we * do -not- check all jobs - we simply notify the HNP * that the orteds are complete. Also check special case * if jdata is NULL - we want * to definitely declare the job done if the orteds * have completed, no matter what else may be happening. * This can happen if a ctrl-c hits in the "wrong" place * while launching */ CHECK_DAEMONS: if (jdata == NULL || jdata->jobid == ORTE_PROC_MY_NAME->jobid) { if (0 == orte_routed.num_routes()) { /* orteds are done! */ OPAL_OUTPUT_VERBOSE((2, orte_state_base_framework.framework_output, "%s orteds complete - exiting", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME))); if (NULL == jdata) { jdata = orte_get_job_data_object(ORTE_PROC_MY_NAME->jobid); } ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_DAEMONS_TERMINATED); OBJ_RELEASE(caddy); return; } OBJ_RELEASE(caddy); return; } /* Release the resources used by this job. Since some errmgrs may want * to continue using resources allocated to the job as part of their * fault recovery procedure, we only do this once the job is "complete". * Note that an aborted/killed job -is- flagged as complete and will * therefore have its resources released. We need to do this after * we call the errmgr so that any attempt to restart the job will * avoid doing so in the exact same place as the current job */ if (NULL != jdata->map && jdata->state == ORTE_JOB_STATE_TERMINATED) { map = jdata->map; for (index = 0; index < map->nodes->size; index++) { if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(map->nodes, index))) { continue; } OPAL_OUTPUT_VERBOSE((2, orte_state_base_framework.framework_output, "%s releasing procs from node %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), node->name)); for (i = 0; i < node->procs->size; i++) { if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(node->procs, i))) { continue; } if (proc->name.jobid != jdata->jobid) { /* skip procs from another job */ continue; } node->slots_inuse--; node->num_procs--; OPAL_OUTPUT_VERBOSE((2, orte_state_base_framework.framework_output, "%s releasing proc %s from node %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_NAME_PRINT(&proc->name), node->name)); /* set the entry in the node array to NULL */ opal_pointer_array_set_item(node->procs, i, NULL); /* release the proc once for the map entry */ OBJ_RELEASE(proc); } /* set the node location to NULL */ opal_pointer_array_set_item(map->nodes, index, NULL); /* maintain accounting */ OBJ_RELEASE(node); /* flag that the node is no longer in a map */ ORTE_FLAG_UNSET(node, ORTE_NODE_FLAG_MAPPED); } OBJ_RELEASE(map); jdata->map = NULL; } CHECK_ALIVE: /* now check to see if all jobs are done - trigger notification of this jdata * object when we find it */ one_still_alive = false; for (j=1; j < orte_job_data->size; j++) { if (NULL == (job = (orte_job_t*)opal_pointer_array_get_item(orte_job_data, j))) { /* since we are releasing jdata objects as we * go, we can no longer assume that the job_data * array is left justified */ continue; } /* if this is the job we are checking AND it normally terminated, * then activate the "notify_completed" state - this will release * the job state, but is provided so that the HNP main code can * take alternative actions if desired. If the state is killed_by_cmd, * then go ahead and release it. We cannot release it if it * abnormally terminated as mpirun needs the info so it can * report appropriately to the user * * NOTE: do not release the primary job (j=1) so we * can pretty-print completion message */ if (NULL != jdata && job->jobid == jdata->jobid) { if (jdata->state == ORTE_JOB_STATE_TERMINATED) { OPAL_OUTPUT_VERBOSE((2, orte_state_base_framework.framework_output, "%s state:base:check_job_completed state is terminated - activating notify", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME))); ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_NOTIFY_COMPLETED); one_still_alive = true; } else if (jdata->state == ORTE_JOB_STATE_KILLED_BY_CMD || jdata->state == ORTE_JOB_STATE_NOTIFIED) { OPAL_OUTPUT_VERBOSE((2, orte_state_base_framework.framework_output, "%s state:base:check_job_completed state is killed or notified - cleaning up", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME))); /* release this object, ensuring that the * pointer array internal accounting * is maintained! */ if (1 < j) { if (ORTE_FLAG_TEST(jdata, ORTE_JOB_FLAG_DEBUGGER_DAEMON)) { /* this was a debugger daemon. notify that a debugger has detached */ ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_DEBUGGER_DETACH); } opal_pointer_array_set_item(orte_job_data, j, NULL); /* ensure the array has a NULL */ OBJ_RELEASE(jdata); } } continue; } /* if the job is flagged to not be monitored, skip it */ if (ORTE_FLAG_TEST(job, ORTE_JOB_FLAG_DO_NOT_MONITOR)) { continue; } /* when checking for job termination, we must be sure to NOT check * our own job as it - rather obviously - has NOT terminated! */ if (job->num_terminated < job->num_procs) { /* we have at least one job that is not done yet - we cannot * just return, though, as we need to ensure we cleanout the * job data for the job that just completed */ OPAL_OUTPUT_VERBOSE((2, orte_state_base_framework.framework_output, "%s state:base:check_job_completed job %s is not terminated (%d:%d)", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_JOBID_PRINT(job->jobid), job->num_terminated, job->num_procs)); one_still_alive = true; } else { OPAL_OUTPUT_VERBOSE((2, orte_state_base_framework.framework_output, "%s state:base:check_job_completed job %s is terminated (%d vs %d [%s])", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_JOBID_PRINT(job->jobid), job->num_terminated, job->num_procs, (NULL == jdata) ? "UNKNOWN" : orte_job_state_to_str(jdata->state) )); } } /* if a job is still alive, we just return */ if (one_still_alive) { OPAL_OUTPUT_VERBOSE((2, orte_state_base_framework.framework_output, "%s state:base:check_job_completed at least one job is not terminated", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME))); OBJ_RELEASE(caddy); return; } /* if we get here, then all jobs are done, so terminate */ OPAL_OUTPUT_VERBOSE((2, orte_state_base_framework.framework_output, "%s state:base:check_job_completed all jobs terminated", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME))); /* stop the job timeout event, if set */ if (NULL != orte_mpiexec_timeout) { OBJ_RELEASE(orte_mpiexec_timeout); orte_mpiexec_timeout = NULL; } /* set the exit status to 0 - this will only happen if it * wasn't already set by an error condition */ ORTE_UPDATE_EXIT_STATUS(0); /* order daemon termination - this tells us to cleanup * our local procs as well as telling remote daemons * to die */ orte_plm.terminate_orteds(); OBJ_RELEASE(caddy); }
/* otherwise, if everything matches, just increment the cnt */ OPAL_VALUE_ARRAY_SET_ITEM(&ndreg->cnt, int32_t, num_nodes, cnt+1); } } } if (!found) { /* need to add it */ ndreg = OBJ_NEW(orte_regex_node_t); ndreg->prefix = strdup(prefix); start_sequence(jdata->jobid, node, ndreg, suffix, nodenum); opal_list_append(&nodelist, &ndreg->super); } } /* the regular expression begins with the jobid */ asprintf(&tmp, "LJID=%s", ORTE_LOCAL_JOBID_PRINT(jdata->jobid)); opal_argv_append_nosize(®exargs, tmp); free(tmp); /* next comes the total slots allocated to us */ asprintf(&tmp, "SLOTS=%d", (int)jdata->total_slots_alloc); opal_argv_append_nosize(®exargs, tmp); free(tmp); /* the control flags for this job */ asprintf(&tmp, "CTRLS=%d", (int)jdata->controls); opal_argv_append_nosize(®exargs, tmp); free(tmp); /* the stdin target for the job */ asprintf(&tmp, "STDIN=%d", (int)jdata->stdin_target);
int orte_iof_base_write_output(orte_process_name_t *name, orte_iof_tag_t stream, unsigned char *data, int numbytes, orte_iof_write_event_t *channel) { char starttag[ORTE_IOF_BASE_TAG_MAX], endtag[ORTE_IOF_BASE_TAG_MAX], *suffix; orte_iof_write_output_t *output; int i, j, k, starttaglen, endtaglen, num_buffered; bool endtagged; char qprint[10]; OPAL_OUTPUT_VERBOSE((1, orte_iof_base.iof_output, "%s write:output setting up to write %d bytes to %s for %s on fd %d", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), numbytes, (ORTE_IOF_STDIN & stream) ? "stdin" : ((ORTE_IOF_STDOUT & stream) ? "stdout" : ((ORTE_IOF_STDERR & stream) ? "stderr" : "stddiag")), ORTE_NAME_PRINT(name), (NULL == channel) ? -1 : channel->fd)); /* setup output object */ output = OBJ_NEW(orte_iof_write_output_t); /* write output data to the corresponding tag */ if (ORTE_IOF_STDIN & stream) { /* copy over the data to be written */ if (0 < numbytes) { /* don't copy 0 bytes - we just need to pass * the zero bytes so the fd can be closed * after it writes everything out */ memcpy(output->data, data, numbytes); } output->numbytes = numbytes; goto process; } else if (ORTE_IOF_STDOUT & stream) { /* write the bytes to stdout */ suffix = "stdout"; } else if (ORTE_IOF_STDERR & stream) { /* write the bytes to stderr */ suffix = "stderr"; } else if (ORTE_IOF_STDDIAG & stream) { /* write the bytes to stderr */ suffix = "stddiag"; } else { /* error - this should never happen */ ORTE_ERROR_LOG(ORTE_ERR_VALUE_OUT_OF_BOUNDS); OPAL_OUTPUT_VERBOSE((1, orte_iof_base.iof_output, "%s stream %0x", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), stream)); return ORTE_ERR_VALUE_OUT_OF_BOUNDS; } /* if this is to be xml tagged, create a tag with the correct syntax - we do not allow * timestamping of xml output */ if (orte_xml_output) { snprintf(starttag, ORTE_IOF_BASE_TAG_MAX, "<%s rank=\"%s\">", suffix, ORTE_VPID_PRINT(name->vpid)); snprintf(endtag, ORTE_IOF_BASE_TAG_MAX, "</%s>", suffix); goto construct; } /* if we are to timestamp output, start the tag with that */ if (orte_timestamp_output) { time_t mytime; char *cptr; /* get the timestamp */ time(&mytime); cptr = ctime(&mytime); cptr[strlen(cptr)-1] = '\0'; /* remove trailing newline */ if (orte_tag_output) { /* if we want it tagged as well, use both */ snprintf(starttag, ORTE_IOF_BASE_TAG_MAX, "%s[%s,%s]<%s>:", cptr, ORTE_LOCAL_JOBID_PRINT(name->jobid), ORTE_VPID_PRINT(name->vpid), suffix); } else { /* only use timestamp */ snprintf(starttag, ORTE_IOF_BASE_TAG_MAX, "%s<%s>:", cptr, suffix); } /* no endtag for this option */ memset(endtag, '\0', ORTE_IOF_BASE_TAG_MAX); goto construct; } if (orte_tag_output) { snprintf(starttag, ORTE_IOF_BASE_TAG_MAX, "[%s,%s]<%s>:", ORTE_LOCAL_JOBID_PRINT(name->jobid), ORTE_VPID_PRINT(name->vpid), suffix); /* no endtag for this option */ memset(endtag, '\0', ORTE_IOF_BASE_TAG_MAX); goto construct; } /* if we get here, then the data is not to be tagged - just copy it * and move on to processing */ if (0 < numbytes) { /* don't copy 0 bytes - we just need to pass * the zero bytes so the fd can be closed * after it writes everything out */ memcpy(output->data, data, numbytes); } output->numbytes = numbytes; goto process; construct: starttaglen = strlen(starttag); endtaglen = strlen(endtag); endtagged = false; /* start with the tag */ for (j=0, k=0; j < starttaglen && k < ORTE_IOF_BASE_TAGGED_OUT_MAX; j++) { output->data[k++] = starttag[j]; } /* cycle through the data looking for <cr> * and replace those with the tag */ for (i=0; i < numbytes && k < ORTE_IOF_BASE_TAGGED_OUT_MAX; i++) { if (orte_xml_output) { if ('&' == data[i]) { if (k+5 >= ORTE_IOF_BASE_TAGGED_OUT_MAX) { ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE); goto process; } snprintf(qprint, 10, "&"); for (j=0; j < (int)strlen(qprint) && k < ORTE_IOF_BASE_TAGGED_OUT_MAX; j++) { output->data[k++] = qprint[j]; } } else if ('<' == data[i]) { if (k+4 >= ORTE_IOF_BASE_TAGGED_OUT_MAX) { ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE); goto process; } snprintf(qprint, 10, "<"); for (j=0; j < (int)strlen(qprint) && k < ORTE_IOF_BASE_TAGGED_OUT_MAX; j++) { output->data[k++] = qprint[j]; } } else if ('>' == data[i]) { if (k+4 >= ORTE_IOF_BASE_TAGGED_OUT_MAX) { ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE); goto process; } snprintf(qprint, 10, ">"); for (j=0; j < (int)strlen(qprint) && k < ORTE_IOF_BASE_TAGGED_OUT_MAX; j++) { output->data[k++] = qprint[j]; } } else if (data[i] < 32 || data[i] > 127) { /* this is a non-printable character, so escape it too */ if (k+7 >= ORTE_IOF_BASE_TAGGED_OUT_MAX) { ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE); goto process; } snprintf(qprint, 10, "&#%03d;", (int)data[i]); for (j=0; j < (int)strlen(qprint) && k < ORTE_IOF_BASE_TAGGED_OUT_MAX; j++) { output->data[k++] = qprint[j]; } /* if this was a \n, then we also need to break the line with the end tag */ if ('\n' == data[i] && (k+endtaglen+1) < ORTE_IOF_BASE_TAGGED_OUT_MAX) { /* we need to break the line with the end tag */ for (j=0; j < endtaglen && k < ORTE_IOF_BASE_TAGGED_OUT_MAX-1; j++) { output->data[k++] = endtag[j]; } /* move the <cr> over */ output->data[k++] = '\n'; /* if this isn't the end of the data buffer, add a new start tag */ if (i < numbytes-1 && (k+starttaglen) < ORTE_IOF_BASE_TAGGED_OUT_MAX) { for (j=0; j < starttaglen && k < ORTE_IOF_BASE_TAGGED_OUT_MAX; j++) { output->data[k++] = starttag[j]; endtagged = false; } } else { endtagged = true; } } } else { output->data[k++] = data[i]; } } else { if ('\n' == data[i]) { /* we need to break the line with the end tag */ for (j=0; j < endtaglen && k < ORTE_IOF_BASE_TAGGED_OUT_MAX-1; j++) { output->data[k++] = endtag[j]; } /* move the <cr> over */ output->data[k++] = '\n'; /* if this isn't the end of the data buffer, add a new start tag */ if (i < numbytes-1) { for (j=0; j < starttaglen && k < ORTE_IOF_BASE_TAGGED_OUT_MAX; j++) { output->data[k++] = starttag[j]; endtagged = false; } } else { endtagged = true; } } else { output->data[k++] = data[i]; } } } if (!endtagged) { /* need to add an endtag */ for (j=0; j < endtaglen && k < ORTE_IOF_BASE_TAGGED_OUT_MAX-1; j++) { output->data[k++] = endtag[j]; } output->data[k++] = '\n'; } output->numbytes = k; process: /* add this data to the write list for this fd */ opal_list_append(&channel->outputs, &output->super); /* record how big the buffer is */ num_buffered = opal_list_get_size(&channel->outputs); /* is the write event issued? */ if (!channel->pending) { /* issue it */ OPAL_OUTPUT_VERBOSE((1, orte_iof_base.iof_output, "%s write:output adding write event", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME))); opal_event_add(channel->ev, 0); channel->pending = true; } return num_buffered; }