/*************************************************************************** * Support functions for slurm_allocate_resources_blocking() ***************************************************************************/ static listen_t *_create_allocation_response_socket(char *interface_hostname) { listen_t *listen = NULL; uint16_t *ports; listen = xmalloc(sizeof(listen_t)); if ((ports = slurm_get_srun_port_range())) listen->fd = slurm_init_msg_engine_ports(ports); else listen->fd = slurm_init_msg_engine_port(0); if (listen->fd < 0) { error("slurm_init_msg_engine_port error %m"); return NULL; } if (slurm_get_stream_addr(listen->fd, &listen->address) < 0) { error("slurm_get_stream_addr error %m"); slurm_shutdown_msg_engine(listen->fd); return NULL; } listen->hostname = xstrdup(interface_hostname); /* FIXME - screw it! I can't seem to get the port number through slurm_* functions */ listen->port = ntohs(listen->address.sin_port); fd_set_nonblocking(listen->fd); return listen; }
/* Set up port to handle messages from slurmctld */ slurm_fd_t slurmctld_msg_init(void) { slurm_addr_t slurm_address; uint16_t port; static slurm_fd_t slurmctld_fd = (slurm_fd_t) 0; if (slurmctld_fd) /* May set early for queued job allocation */ return slurmctld_fd; slurmctld_fd = -1; slurmctld_comm_addr.port = 0; if ((slurmctld_fd = slurm_init_msg_engine_port(0)) < 0) { error("slurm_init_msg_engine_port error %m"); exit(error_exit); } if (slurm_get_stream_addr(slurmctld_fd, &slurm_address) < 0) { error("slurm_get_stream_addr error %m"); exit(error_exit); } fd_set_nonblocking(slurmctld_fd); /* hostname is not set, so slurm_get_addr fails slurm_get_addr(&slurm_address, &port, hostname, sizeof(hostname)); */ port = ntohs(slurm_address.sin_port); slurmctld_comm_addr.port = port; debug2("srun PMI messages to port=%u", slurmctld_comm_addr.port); return slurmctld_fd; }
void pty_thread_create(srun_job_t *job) { slurm_addr_t pty_addr; pthread_attr_t attr; if ((job->pty_fd = slurm_init_msg_engine_port(0)) < 0) { error("init_msg_engine_port: %m"); return; } if (slurm_get_stream_addr(job->pty_fd, &pty_addr) < 0) { error("slurm_get_stream_addr: %m"); return; } job->pty_port = ntohs(((struct sockaddr_in) pty_addr).sin_port); debug2("initialized job control port %hu", job->pty_port); slurm_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); if ((pthread_create(&job->pty_id, &attr, &_pty_thread, (void *) job))) { job->pty_id = 0; error("pthread_create(pty_thread): %m"); } slurm_attr_destroy(&attr); }
/*************************************************************************** * Support functions for slurm_allocate_resources_blocking() ***************************************************************************/ static listen_t *_create_allocation_response_socket(char *interface_hostname) { listen_t *listen = NULL; listen = xmalloc(sizeof(listen_t)); /* port "0" lets the operating system pick any port */ if ((listen->fd = slurm_init_msg_engine_port(0)) < 0) { error("slurm_init_msg_engine_port error %m"); return NULL; } if (slurm_get_stream_addr(listen->fd, &listen->address) < 0) { error("slurm_get_stream_addr error %m"); slurm_shutdown_msg_engine(listen->fd); return NULL; } listen->hostname = xstrdup(interface_hostname); /* FIXME - screw it! I can't seem to get the port number through slurm_* functions */ listen->port = ntohs(listen->address.sin_port); fd_set_nonblocking(listen->fd); return listen; }
/* Wait for barrier and get full PMI Keyval space data */ int slurm_get_kvs_comm_set(struct kvs_comm_set **kvs_set_ptr, int pmi_rank, int pmi_size) { int rc, srun_fd, retries = 0, timeout = 0; slurm_msg_t msg_send, msg_rcv; slurm_addr_t slurm_addr, srun_reply_addr; char hostname[64]; uint16_t port; kvs_get_msg_t data; char *env_pmi_ifhn; if (kvs_set_ptr == NULL) return EINVAL; *kvs_set_ptr = NULL; /* initialization */ if ((rc = _get_addr()) != SLURM_SUCCESS) { error("_get_addr: %m"); return rc; } _set_pmi_time(); if (pmi_fd < 0) { if ((pmi_fd = slurm_init_msg_engine_port(0)) < 0) { error("slurm_init_msg_engine_port: %m"); return SLURM_ERROR; } fd_set_blocking(pmi_fd); } if (slurm_get_stream_addr(pmi_fd, &slurm_addr) < 0) { error("slurm_get_stream_addr: %m"); return SLURM_ERROR; } /* hostname is not set here, so slurm_get_addr fails slurm_get_addr(&slurm_addr, &port, hostname, sizeof(hostname)); */ port = ntohs(slurm_addr.sin_port); if ((env_pmi_ifhn = getenv("SLURM_PMI_RESP_IFHN"))) { strncpy(hostname, env_pmi_ifhn, sizeof(hostname)); hostname[sizeof(hostname)-1] = 0; } else gethostname_short(hostname, sizeof(hostname)); data.task_id = pmi_rank; data.size = pmi_size; data.port = port; data.hostname = hostname; slurm_msg_t_init(&msg_send); slurm_msg_t_init(&msg_rcv); msg_send.address = srun_addr; msg_send.msg_type = PMI_KVS_GET_REQ; msg_send.data = &data; /* Send the RPC to the local srun communcation manager. * Since the srun can be sent thousands of messages at * the same time and refuse some connections, retry as * needed. Wait until all key-pairs have been sent by * all tasks then spread out messages by task's rank. * Also increase the message timeout if many tasks * since the srun command can get very overloaded (the * default timeout is 10 secs). */ _delay_rpc(pmi_rank, pmi_size); if (pmi_size > 4000) /* 240 secs */ timeout = slurm_get_msg_timeout() * 24000; else if (pmi_size > 1000) /* 120 secs */ timeout = slurm_get_msg_timeout() * 12000; else if (pmi_size > 100) /* 60 secs */ timeout = slurm_get_msg_timeout() * 6000; else if (pmi_size > 10) /* 20 secs */ timeout = slurm_get_msg_timeout() * 2000; while (slurm_send_recv_rc_msg_only_one(&msg_send, &rc, timeout) < 0) { if (retries++ > MAX_RETRIES) { error("slurm_get_kvs_comm_set: %m"); return SLURM_ERROR; } else debug("get kvs retry %d", retries); _delay_rpc(pmi_rank, pmi_size); } if (rc != SLURM_SUCCESS) { error("slurm_get_kvs_comm_set error_code=%d", rc); return rc; } /* get the message after all tasks reach the barrier */ srun_fd = slurm_accept_msg_conn(pmi_fd, &srun_reply_addr); if (srun_fd < 0) { error("slurm_accept_msg_conn: %m"); return errno; } while ((rc = slurm_receive_msg(srun_fd, &msg_rcv, timeout)) != 0) { if (errno == EINTR) continue; error("slurm_receive_msg: %m"); slurm_close(srun_fd); return errno; } if (msg_rcv.auth_cred) (void)g_slurm_auth_destroy(msg_rcv.auth_cred); if (msg_rcv.msg_type != PMI_KVS_GET_RESP) { error("slurm_get_kvs_comm_set msg_type=%d", msg_rcv.msg_type); slurm_close(srun_fd); return SLURM_UNEXPECTED_MSG_ERROR; } if (slurm_send_rc_msg(&msg_rcv, SLURM_SUCCESS) < 0) error("slurm_send_rc_msg: %m"); slurm_close(srun_fd); *kvs_set_ptr = msg_rcv.data; rc = _forward_comm_set(*kvs_set_ptr); return rc; }