int psl_serv_afu_event(struct AFU_EVENT *event, int port)
{
int cs = -1;
psl_event_reset(event);
event->room = 64;
event->rbp = 0;
struct sockaddr_in ssadr, csadr;
unsigned int csalen = sizeof(csadr);
memset(&ssadr, 0, sizeof(ssadr));
ssadr.sin_family = AF_UNSPEC;
ssadr.sin_addr.s_addr = INADDR_ANY;
ssadr.sin_port = htons(port);
event->sockfd = socket(PF_INET, SOCK_STREAM, 0);
if (event->sockfd < 0) {
perror("socket");
return PSL_BAD_SOCKET;
}
if (bind(event->sockfd, (struct sockaddr *)&ssadr, sizeof(ssadr)) == -1) {
perror("bind");
psl_close_afu_event(event);
return PSL_BAD_SOCKET;
}
char hostname[1024];
hostname[1023] = '\0';
gethostname(hostname, 1023);
printf("AFU Server is waiting for connection on %s:%d\n", hostname,
port);
fflush(stdout);
if (listen(event->sockfd, 10) == -1) {
perror("listen");
psl_close_afu_event(event);
return PSL_BAD_SOCKET;
}
while (cs < 0) {
cs = accept(event->sockfd, (struct sockaddr *)&csadr, &csalen);
if ((cs < 0) && (errno != EINTR)) {
perror("accept");
psl_close_afu_event(event);
return PSL_BAD_SOCKET;
}
}
close(event->sockfd);
event->sockfd = cs;
fcntl(event->sockfd, F_SETFL, O_NONBLOCK);
char clientname[1024];
clientname[1023] = '\0';
getnameinfo((struct sockaddr *)&csadr, sizeof(csadr), clientname, 1024,
NULL, 0, 0);
printf("PSL client connection from %s\n", clientname);
int rc = establish_protocol(event);
printf("Using PSL protocol level : %d.%d.%d\n", event->proto_primary,
event->proto_secondary, event->proto_tertiary);
return rc;
}
void cxl_afu_free(struct cxl_afu_h *afu) {
// Check for valid AFU
if (!afu) return;
// Wait for job_done
while (afu->running) {
// FIXME: Timeout
short_delay();
}
// Reset AFU
status.cmd.code = PSL_JOB_RESET;
status.cmd.addr = 0;
status.cmd.request = AFU_REQUEST;
// Wait for job_done
while (status.cmd.request != AFU_IDLE) {
// FIXME: Timeout
short_delay();
}
// Stop PSL thread
status.psl_state = PSL_DONE;
pthread_join(afu->thread, NULL);
// Shut down socket connection
psl_close_afu_event (status.event);
// Free memory
free (afu);
}
// Initialize and start PSL thread
//
// The return value is encode int a 16-bit value divided into 4 for each
// possible adapter. Then the 4 bits in each adapter represent the 4 possible
// AFUs on an adapter. For example: afu0.0 is 0x8000 and afu3.0 is 0x0008.
uint16_t psl_init(struct psl **head, struct parms *parms, char *id, char *host,
int port, pthread_mutex_t * lock, FILE * dbg_fp)
{
struct psl *psl;
struct job_event *reset;
uint16_t location;
location = 0x8000;
if ((psl = (struct psl *)calloc(1, sizeof(struct psl))) == NULL) {
perror("malloc");
error_msg("Unable to allocation memory for psl");
goto init_fail;
}
psl->timeout = parms->timeout;
if ((strlen(id) != 6) || strncmp(id, "afu", 3) || (id[4] != '.')) {
warn_msg("Invalid afu name: %s", id);
goto init_fail;
}
if ((id[3] < '0') || (id[3] > '3')) {
warn_msg("Invalid afu major: %c", id[3]);
goto init_fail;
}
if ((id[5] < '0') || (id[5] > '3')) {
warn_msg("Invalid afu minor: %c", id[5]);
goto init_fail;
}
psl->dbg_fp = dbg_fp;
psl->major = id[3] - '0';
psl->minor = id[5] - '0';
psl->dbg_id = psl->major << 4;
psl->dbg_id |= psl->minor;
location >>= (4 * psl->major);
location >>= psl->minor;
if ((psl->name = (char *)malloc(strlen(id) + 1)) == NULL) {
perror("malloc");
error_msg("Unable to allocation memory for psl->name");
goto init_fail;
}
strcpy(psl->name, id);
if ((psl->host = (char *)malloc(strlen(host) + 1)) == NULL) {
perror("malloc");
error_msg("Unable to allocation memory for psl->host");
goto init_fail;
}
strcpy(psl->host, host);
psl->port = port;
psl->client = NULL;
psl->idle_cycles = PSL_IDLE_CYCLES;
psl->lock = lock;
// Connect to AFU
psl->afu_event = (struct AFU_EVENT *)malloc(sizeof(struct AFU_EVENT));
if (psl->afu_event == NULL) {
perror("malloc");
goto init_fail;
}
info_msg("Attempting to connect AFU: %s @ %s:%d", psl->name,
psl->host, psl->port);
if (psl_init_afu_event(psl->afu_event, psl->host, psl->port) !=
PSL_SUCCESS) {
warn_msg("Unable to connect AFU: %s @ %s:%d", psl->name,
psl->host, psl->port);
goto init_fail;
}
// DEBUG
debug_afu_connect(psl->dbg_fp, psl->dbg_id);
// Initialize job handler
if ((psl->job = job_init(psl->afu_event, &(psl->state), psl->name,
psl->dbg_fp, psl->dbg_id)) == NULL) {
perror("job_init");
goto init_fail;
}
// Initialize mmio handler
if ((psl->mmio = mmio_init(psl->afu_event, psl->timeout, psl->name,
psl->dbg_fp, psl->dbg_id)) == NULL) {
perror("mmio_init");
goto init_fail;
}
// Initialize cmd handler
if ((psl->cmd = cmd_init(psl->afu_event, parms, psl->mmio,
&(psl->state), psl->name, psl->dbg_fp,
psl->dbg_id))
== NULL) {
perror("cmd_init");
goto init_fail;
}
// Set credits for AFU
if (psl_aux1_change(psl->afu_event, psl->cmd->credits) != PSL_SUCCESS) {
warn_msg("Unable to set credits");
goto init_fail;
}
// Start psl loop thread
if (pthread_create(&(psl->thread), NULL, _psl_loop, psl)) {
perror("pthread_create");
goto init_fail;
}
// Add psl to list
while ((*head != NULL) && ((*head)->major < psl->major)) {
head = &((*head)->_next);
}
while ((*head != NULL) && ((*head)->major == psl->major) &&
((*head)->minor < psl->minor)) {
head = &((*head)->_next);
}
psl->_next = *head;
if (psl->_next != NULL)
psl->_next->_prev = psl;
*head = psl;
// Send reset to AFU
reset = add_job(psl->job, PSL_JOB_RESET, 0L);
while (psl->job->job == reset) { /*infinite loop */
lock_delay(psl->lock);
}
// Read AFU descriptor
psl->state = PSLSE_DESC;
read_descriptor(psl->mmio, psl->lock);
// Finish PSL configuration
psl->state = PSLSE_IDLE;
if (dedicated_mode_support(psl->mmio)) {
// AFU supports Dedicated Mode
psl->max_clients = 1;
}
if (directed_mode_support(psl->mmio)) {
// AFU supports Directed Mode
psl->max_clients = psl->mmio->desc.num_of_processes;
}
if (psl->max_clients == 0) {
error_msg("AFU programming model is invalid");
goto init_fail;
}
psl->client = (struct client **)calloc(psl->max_clients,
sizeof(struct client *));
psl->cmd->client = psl->client;
psl->cmd->max_clients = psl->max_clients;
return location;
init_fail:
if (psl) {
if (psl->afu_event) {
psl_close_afu_event(psl->afu_event);
free(psl->afu_event);
}
if (psl->host)
free(psl->host);
if (psl->name)
free(psl->name);
free(psl);
}
pthread_mutex_unlock(lock);
return 0;
}
// PSL thread loop
static void *_psl_loop(void *ptr)
{
struct psl *psl = (struct psl *)ptr;
struct cmd_event *event, *temp;
int events, i, stopped, reset;
uint8_t ack = PSLSE_DETACH;
stopped = 1;
pthread_mutex_lock(psl->lock);
while (psl->state != PSLSE_DONE) {
// idle_cycles continues to generate clock cycles for some
// time after the AFU has gone idle. Eventually clocks will
// not be presented to an idle AFU to keep simulation
// waveforms from getting huge with no activity cycles.
if (psl->state != PSLSE_IDLE) {
psl->idle_cycles = PSL_IDLE_CYCLES;
if (stopped)
info_msg("Clocking %s", psl->name);
fflush(stdout);
stopped = 0;
}
if (psl->idle_cycles) {
// Clock AFU
psl_signal_afu_model(psl->afu_event);
// Check for events from AFU
events = psl_get_afu_events(psl->afu_event);
// Error on socket
if (events < 0) {
warn_msg("Lost connection with AFU");
break;
}
// Handle events from AFU
if (events > 0)
_handle_afu(psl);
// Drive events to AFU
send_job(psl->job);
send_mmio(psl->mmio);
if (psl->mmio->list == NULL)
psl->idle_cycles--;
} else {
if (!stopped)
info_msg("Stopping clocks to %s", psl->name);
stopped = 1;
lock_delay(psl->lock);
}
// Skip client section if AFU descriptor hasn't been read yet
if (psl->client == NULL) {
lock_delay(psl->lock);
continue;
}
// Check for event from application
reset = 0;
for (i = 0; i < psl->max_clients; i++) {
if (psl->client[i] == NULL)
continue;
if ((psl->client[i]->state == CLIENT_NONE) &&
(psl->client[i]->idle_cycles == 0)) {
put_bytes(psl->client[i]->fd, 1, &ack,
psl->dbg_fp, psl->dbg_id,
psl->client[i]->context);
_free(psl, psl->client[i]);
psl->client[i] = NULL;
reset = 1;
continue;
}
if (psl->state == PSLSE_RESET)
continue;
_handle_client(psl, psl->client[i]);
if (psl->client[i]->idle_cycles) {
psl->client[i]->idle_cycles--;
}
if (client_cmd(psl->cmd, psl->client[i])) {
psl->client[i]->idle_cycles = PSL_IDLE_CYCLES;
}
}
// Send reset to AFU
if (reset == 1) {
psl->cmd->buffer_read = NULL;
event = psl->cmd->list;
while (event != NULL) {
if (reset) {
warn_msg
("Client dropped context before AFU completed");
reset = 0;
}
warn_msg("Dumping command tag=0x%02x",
event->tag);
if (event->data) {
free(event->data);
}
if (event->parity) {
free(event->parity);
}
temp = event;
event = event->_next;
free(temp);
}
psl->cmd->list = NULL;
info_msg("Sending reset to AFU");
add_job(psl->job, PSL_JOB_RESET, 0L);
}
lock_delay(psl->lock);
}
// Disconnect clients
for (i = 0; i < psl->max_clients; i++) {
if ((psl->client != NULL) && (psl->client[i] != NULL)) {
// FIXME: Send warning to clients first?
info_msg("Disconnecting %s context %d", psl->name,
psl->client[i]->context);
close_socket(&(psl->client[i]->fd));
}
}
// DEBUG
debug_afu_drop(psl->dbg_fp, psl->dbg_id);
// Disconnect from simulator, free memory and shut down thread
info_msg("Disconnecting %s @ %s:%d", psl->name, psl->host, psl->port);
if (psl->client)
free(psl->client);
if (psl->_prev)
psl->_prev->_next = psl->_next;
if (psl->_next)
psl->_next->_prev = psl->_prev;
if (psl->cmd) {
free(psl->cmd);
}
if (psl->job) {
free(psl->job);
}
if (psl->mmio) {
free(psl->mmio);
}
if (psl->host)
free(psl->host);
if (psl->afu_event) {
psl_close_afu_event(psl->afu_event);
free(psl->afu_event);
}
if (psl->name)
free(psl->name);
if (*(psl->head) == psl)
*(psl->head) = psl->_next;
pthread_mutex_unlock(psl->lock);
free(psl);
pthread_exit(NULL);
}
// PSL thread loop
static void *_psl_loop(void *ptr)
{
struct psl *psl = (struct psl *)ptr;
struct cmd_event *event, *temp;
int events, i, stopped, reset;
uint8_t ack = PSLSE_DETACH;
stopped = 1;
pthread_mutex_lock(psl->lock);
while (psl->state != PSLSE_DONE) {
// idle_cycles continues to generate clock cycles for some
// time after the AFU has gone idle. Eventually clocks will
// not be presented to an idle AFU to keep simulation
// waveforms from getting huge with no activity cycles.
if (psl->state != PSLSE_IDLE) {
// if we have clients or we are in the reset state, refresh idle_cycles
// so that the afu clock will not be allowed to stop to save afu event simulator cycles
if ((psl->attached_clients > 0) || (psl->state == PSLSE_RESET)) {
psl->idle_cycles = PSL_IDLE_CYCLES;
if (stopped)
info_msg("Clocking %s", psl->name);
fflush(stdout);
stopped = 0;
}
}
if (psl->idle_cycles) {
// Clock AFU
//printf("before psl_signal_afu_model in psl_loop \n");
psl_signal_afu_model(psl->afu_event);
// Check for events from AFU
events = psl_get_afu_events(psl->afu_event);
//printf("after psl_get_afu_events, events is 0x%3x \n", events);
// Error on socket
if (events < 0) {
warn_msg("Lost connection with AFU");
break;
}
// Handle events from AFU
if (events > 0)
_handle_afu(psl);
// Drive events to AFU
send_job(psl->job);
send_pe(psl->job);
send_mmio(psl->mmio);
if (psl->mmio->list == NULL)
psl->idle_cycles--;
} else {
if (!stopped)
info_msg("Stopping clocks to %s", psl->name);
stopped = 1;
lock_delay(psl->lock);
}
// Skip client section if AFU descriptor hasn't been read yet
if (psl->client == NULL) {
lock_delay(psl->lock);
continue;
}
// Check for event from application
reset = 0;
for (i = 0; i < psl->max_clients; i++) {
if (psl->client[i] == NULL)
continue;
if ((psl->client[i]->type == 'd') &&
(psl->client[i]->state == CLIENT_NONE) &&
(psl->client[i]->idle_cycles == 0)) {
// this was the old way of detaching a dedicated process app/afu pair
// we get the detach message, drop the client, and wait for idle cycle to get to 0
put_bytes(psl->client[i]->fd, 1, &ack,
psl->dbg_fp, psl->dbg_id,
psl->client[i]->context);
_free(psl, psl->client[i]);
psl->client[i] = NULL; // aha - this is how we only called _free once the old way
// why do we not free client[i]?
// because this was a short cut pointer
// the *real* client point is in client_list in pslse
reset = 1;
// for m/s devices we need to do this differently and not send a reset...
// _handle_client - creates the llcmd's to term and remove
// send_pe - sends the llcmd pe's to afu one at a time
// _handle_afu calls _handle_aux2
// _handle_aux2 finishes the llcmd pe's when jcack is asserted by afu
// when the remove llcmd is processed, we should put_bytes, _free and set client[i] to NULL
continue;
}
if (psl->state == PSLSE_RESET)
continue;
_handle_client(psl, psl->client[i]);
if (psl->client[i]->idle_cycles) {
psl->client[i]->idle_cycles--;
}
if (client_cmd(psl->cmd, psl->client[i])) {
psl->client[i]->idle_cycles = PSL_IDLE_CYCLES;
}
}
// Send reset to AFU
if (reset == 1) {
psl->cmd->buffer_read = NULL;
event = psl->cmd->list;
while (event != NULL) {
if (reset) {
warn_msg
("Client dropped context before AFU completed");
reset = 0;
}
info_msg("Dumping command tag=0x%02x",
event->tag);
#ifdef PSL9
info_msg("Dumping itag=0x%02x utag=0x%02x type=0x%02x state=0x%02x",
event->itag, event->utag, event->type, event->state);
#endif
if (event->data) {
free(event->data);
}
if (event->parity) {
free(event->parity);
}
temp = event;
event = event->_next;
free(temp);
}
psl->cmd->list = NULL;
info_msg("Sending reset to AFU");
add_job(psl->job, PSL_JOB_RESET, 0L);
}
lock_delay(psl->lock);
}
// Disconnect clients
for (i = 0; i < psl->max_clients; i++) {
if ((psl->client != NULL) && (psl->client[i] != NULL)) {
// FIXME: Send warning to clients first?
info_msg("Disconnecting %s context %d", psl->name,
psl->client[i]->context);
close_socket(&(psl->client[i]->fd));
}
}
// DEBUG
debug_afu_drop(psl->dbg_fp, psl->dbg_id);
// Disconnect from simulator, free memory and shut down thread
info_msg("Disconnecting %s @ %s:%d", psl->name, psl->host, psl->port);
if (psl->client)
free(psl->client);
if (psl->_prev)
psl->_prev->_next = psl->_next;
if (psl->_next)
psl->_next->_prev = psl->_prev;
if (psl->cmd) {
free(psl->cmd);
}
if (psl->job) {
free(psl->job);
}
if (psl->mmio) {
free(psl->mmio);
}
if (psl->host)
free(psl->host);
if (psl->afu_event) {
psl_close_afu_event(psl->afu_event);
free(psl->afu_event);
}
if (psl->name)
free(psl->name);
if (*(psl->head) == psl)
*(psl->head) = psl->_next;
pthread_mutex_unlock(psl->lock);
free(psl);
pthread_exit(NULL);
}