/* Wake up access waiting in a stack's wait list. */
void mod_stack_wakeup_stack(struct mod_stack_t *master_stack)
{
struct mod_stack_t *stack;
int event;
/* No access to wake up */
if (!master_stack->waiting_list_count)
return;
/* Debug */
mem_debug(" %lld %lld 0x%x wake up accesses:", esim_time,
master_stack->id, master_stack->addr);
/* Wake up all coalesced accesses */
while (master_stack->waiting_list_head)
{
stack = master_stack->waiting_list_head;
event = stack->waiting_list_event;
DOUBLE_LINKED_LIST_REMOVE(master_stack, waiting, stack);
stack->master_stack = NULL;
esim_schedule_event(event, stack, 0);
if(( stack->addr >= 0x2F20 && stack->addr<= 0x2F2C))
;// fprintf(stderr, " wake master %x, %x, %d, %lld\n", master_stack->addr, stack->addr , event,esim_time);
mem_debug(" %lld", stack->id);
}
/* Debug */
mem_debug("\n");
}
/* Wake up access waiting in a stack's wait list. */
void mod_stack_wakeup_stack(struct mod_stack_t *master_stack)
{
struct mod_stack_t *stack;
int event;
/* No access to wake up */
if (!master_stack->waiting_list_count)
return;
/* Debug */
mem_debug(" %lld %lld 0x%x wake up accesses:", esim_time,
master_stack->id, master_stack->addr);
//fran
master_stack->coalesced_count = 0;
/* Wake up all coalesced accesses */
while (master_stack->waiting_list_head)
{
stack = master_stack->waiting_list_head;
event = stack->waiting_list_event;
stack->waiting_list_event = 0;
DOUBLE_LINKED_LIST_REMOVE(master_stack, waiting, stack);
stack->state = master_stack->state;
esim_schedule_event(event, stack, 0);
mem_debug(" %lld", stack->id);
}
/* Debug */
mem_debug("\n");
}
void moesi_stack_return(struct moesi_stack_t *stack)
{
int retevent = stack->retevent;
void *retstack = stack->retstack;
repos_free_object(moesi_stack_repos, stack);
esim_schedule_event(retevent, retstack, 0);
}
void net_command_stack_return(struct net_command_stack_t *stack)
{
int retevent = stack->ret_event;
struct net_command_stack_t *retstack = stack->ret_stack;
free(stack);
esim_schedule_event(retevent, retstack, 0);
}
void dram_request_handler (int event, void *data)
{
struct list_t *token_list;
struct request_stack_t *req_stack = data;
struct dram_system_t *system = req_stack->system;
char *request_line = req_stack->request_line;
fprintf(stderr, "request line : %s \n", request_line);
char request[MAX_STRING_SIZE];
long long cycle = esim_domain_cycle(dram_domain_index);
/* Split command in tokens, skip command */
token_list = str_token_list_create(request_line, " ");
assert(list_count(token_list));
long long request_cycle = dram_request_get_llint(token_list,
request_line,"cycle value");
if (request_cycle > cycle)
{
str_token_list_free(token_list);
esim_schedule_event(event, req_stack, request_cycle - cycle);
return;
}
struct dram_request_t *dram_request;
dram_request = dram_request_create();
dram_request_get_type(token_list, request_line, request,
sizeof request);
if (!strcasecmp(request, "READ"))
{
dram_request->type = request_type_read;
}
else if (!strcasecmp(request, "WRITE"))
{
dram_request->type = request_type_write;
}
else
fatal("%s: invalid access type %s.\n\t> %s",
__FUNCTION__, request, request_line);
dram_request->addr = dram_request_get_hex_address(token_list, request_line);
dram_request->system = system;
dram_request->id = system->request_count;
system->request_count++;
list_enqueue(system->dram_request_list, dram_request);
str_token_list_free(token_list);
dram_request_stack_free(req_stack);
}
void mod_stack_return(struct mod_stack_t *stack)
{
int ret_event = stack->ret_event;
void *ret_stack = stack->ret_stack;
/* Wake up dependent accesses */
mod_stack_wakeup_stack(stack);
/* Free */
free(stack);
esim_schedule_event(ret_event, ret_stack, 0);
}
/* Wake up accesses waiting in module wait list. */
void mod_stack_wakeup_mod(struct mod_t *mod)
{
struct mod_stack_t *stack;
int event;
while (mod->waiting_list_head)
{
stack = mod->waiting_list_head;
event = stack->waiting_list_event;
DOUBLE_LINKED_LIST_REMOVE(mod, waiting, stack);
esim_schedule_event(event, stack, 0);
}
}
/* Wake up accesses waiting in a port wait list. */
void mod_stack_wakeup_port(struct mod_port_t *port)
{
struct mod_stack_t *stack;
int event;
while (port->waiting_list_head)
{
stack = port->waiting_list_head;
event = stack->waiting_list_event;
DOUBLE_LINKED_LIST_REMOVE(port, waiting, stack);
esim_schedule_event(event, stack, 0);
}
}
void dir_lock_unlock(struct dir_lock_t *dir_lock)
{
cache_debug(" dir_lock %p - unlock\n", dir_lock);
/* Wake up all waiters */
while (dir_lock->lock_queue) {
esim_schedule_event(dir_lock->lock_queue->lock_event, dir_lock->lock_queue, 1);
cache_debug(" 0x%x access resumed\n", dir_lock->lock_queue->tag);
dir_lock->lock_queue = dir_lock->lock_queue->lock_next;
}
/* Unlock entry */
dir_lock->lock = 0;
}
static void net_config_command_create(struct net_t *net, struct config_t *config, char *section)
{
char *command_line;
char command_var[MAX_STRING_SIZE];
int command_var_id;
/* Checks */
if (net_injection_rate > 0.001)
fatal("Network %s:%s: Using Command section; \n"
"\t option --net-injection-rate should not be used \n",
net->name,section);
/* Read commands */
net_injection_rate = 0;
if (strcmp(net_traffic_pattern, "") &&
(strcmp(net_traffic_pattern, "command")))
fatal("Network %s: Command option doesn't comply with other "
"traffic pattern\n (%s)", net->name,
net_traffic_pattern);
net_traffic_pattern = "command";
command_var_id = 0;
/* Register events for command handler*/
EV_NET_COMMAND = esim_register_event_with_name(net_command_handler,
net_domain_index, "net_command");
EV_NET_COMMAND_RCV = esim_register_event_with_name(net_command_handler,
net_domain_index, "net_command_receive");
while (1)
{
/* Get command */
snprintf(command_var, sizeof command_var, "Command[%d]", command_var_id);
command_line = config_read_string(config, section, command_var, NULL);
if (!command_line)
break;
/* Schedule event to process command */
struct net_stack_t *stack;
stack = net_stack_create(net,ESIM_EV_NONE, NULL);
stack->net = net;
stack->command = xstrdup(command_line);
esim_schedule_event(EV_NET_COMMAND, stack, 0);
/* Next command */
command_var_id++;
}
}
/* Schedule all events waiting in the wakeup list */
void net_buffer_wakeup(struct net_buffer_t *buffer)
{
struct net_buffer_wakeup_t *wakeup;
while (linked_list_count(buffer->wakeup_list))
{
/* Get event/stack */
linked_list_head(buffer->wakeup_list);
wakeup = linked_list_get(buffer->wakeup_list);
linked_list_remove(buffer->wakeup_list);
/* Schedule event */
esim_schedule_event(wakeup->event, wakeup->stack, 0);
free(wakeup);
}
}
/* Return TRUE if a message can be sent to the network. If it cannot be sent,
* return FALSE, and schedule 'retry_event' for the cycle when the check
* should be performed again. This function should not be called if the
* reason why a message cannot be sent is permanent (e.g., no route to
* destination). */
int net_can_send_ev(struct net_t *net, struct net_node_t *src_node,
struct net_node_t *dst_node, int size,
int retry_event, void *retry_stack)
{
struct net_routing_table_t *routing_table = net->routing_table;
struct net_routing_table_entry_t *entry;
struct net_buffer_t *output_buffer;
long long cycle;
/* Get current cycle */
cycle = esim_domain_cycle(net_domain_index);
/* Get output buffer */
entry = net_routing_table_lookup(routing_table, src_node, dst_node);
output_buffer = entry->output_buffer;
/* No route to destination */
if (!output_buffer)
fatal("%s: no route between %s and %s.\n%s",
net->name, src_node->name, dst_node->name,
net_err_no_route);
/* Message is too long */
if (size > output_buffer->size)
fatal("%s: message too long.\n%s", net->name,
net_err_large_message);
/* Output buffer is busy */
if (output_buffer->write_busy >= cycle)
{
esim_schedule_event(retry_event, retry_stack,
output_buffer->write_busy - cycle + 1);
return 0;
}
/* Message does not fit in output buffer */
if (output_buffer->count + size > output_buffer->size)
{
net_buffer_wait(output_buffer, retry_event, retry_stack);
return 0;
}
/* All conditions satisfied, can send */
return 1;
}
void dir_entry_unlock(struct dir_t *dir, int x, int y)
{
struct dir_lock_t *dir_lock;
struct mod_stack_t *stack;
FILE *f;
/* Get lock */
assert(x < dir->xsize && y < dir->ysize);
dir_lock = &dir->dir_lock[x * dir->ysize + y];
/* Wake up first waiter */
if (dir_lock->lock_queue)
{
/* Debug */
f = debug_file(mem_debug_category);
if (f)
{
mem_debug(" A-%lld resumed", dir_lock->lock_queue->id);
if (dir_lock->lock_queue->dir_lock_next)
{
mem_debug(" - {");
for (stack = dir_lock->lock_queue->dir_lock_next; stack;
stack = stack->dir_lock_next)
mem_debug(" A-%lld", stack->id);
mem_debug(" } still waiting");
}
mem_debug("\n");
}
/* Wake up access */
esim_schedule_event(dir_lock->lock_queue->dir_lock_event, dir_lock->lock_queue, 1);
dir_lock->lock_queue = dir_lock->lock_queue->dir_lock_next;
}
/* Trace */
mem_trace("mem.end_access_block cache=\"%s\" access=\"A-%lld\" set=%d way=%d\n",
dir->name, dir_lock->stack_id, x, y);
/* Unlock entry */
dir_lock->lock = 0;
}
static void dram_config_request_create(struct dram_system_t *system, struct config_t *config,
char *section)
{
char *request_line;
char request_var[MAX_STRING_SIZE];
int request_var_id;
/* Read Requests */
request_var_id = 0;
/* Register events for request handler*/
EV_DRAM_REQUEST = esim_register_event_with_name(dram_request_handler,
dram_domain_index, "dram_request");
while (1)
{
/* Get request */
snprintf(request_var, sizeof request_var, "Request[%d]", request_var_id);
request_line = config_read_string(config, section, request_var, NULL);
if (!request_line)
break;
/* Schedule event to process request */
struct request_stack_t *stack;
stack = dram_request_stack_create();
request_line = xstrdup(request_line);
stack->request_line = request_line;
stack->system = system;
esim_schedule_event(EV_DRAM_REQUEST, stack, 0);
/* Next command */
request_var_id++;
}
}
void mod_handler_local_mem_load(int event, void *data)
{
struct mod_stack_t *stack = data;
struct mod_stack_t *new_stack;
struct mod_t *mod = stack->mod;
if (event == EV_MOD_LOCAL_MEM_LOAD)
{
struct mod_stack_t *master_stack;
mem_debug(" %lld %lld 0x%x %s load\n", esim_cycle, stack->id,
stack->addr, mod->name);
mem_trace("mem.new_access name=\"A-%lld\" type=\"load\" "
"state=\"%s:load\" addr=0x%x\n",
stack->id, mod->name, stack->addr);
/* Record access */
mod_access_start(mod, stack, mod_access_load);
/* Coalesce access */
master_stack = mod_can_coalesce(mod, mod_access_load, stack->addr, stack);
if (master_stack)
{
mod->reads++;
mod_coalesce(mod, master_stack, stack);
mod_stack_wait_in_stack(stack, master_stack, EV_MOD_LOCAL_MEM_LOAD_FINISH);
return;
}
esim_schedule_event(EV_MOD_LOCAL_MEM_LOAD_LOCK, stack, 0);
return;
}
if (event == EV_MOD_LOCAL_MEM_LOAD_LOCK)
{
struct mod_stack_t *older_stack;
mem_debug(" %lld %lld 0x%x %s load lock\n", esim_cycle, stack->id,
stack->addr, mod->name);
mem_trace("mem.access name=\"A-%lld\" state=\"%s:load_lock\"\n",
stack->id, mod->name);
/* If there is any older write, wait for it */
older_stack = mod_in_flight_write(mod, stack);
if (older_stack)
{
mem_debug(" %lld wait for write %lld\n",
stack->id, older_stack->id);
mod_stack_wait_in_stack(stack, older_stack, EV_MOD_LOCAL_MEM_LOAD_LOCK);
return;
}
/* If there is any older access to the same address that this access could not
* be coalesced with, wait for it. */
older_stack = mod_in_flight_address(mod, stack->addr, stack);
if (older_stack)
{
mem_debug(" %lld wait for access %lld\n",
stack->id, older_stack->id);
mod_stack_wait_in_stack(stack, older_stack, EV_MOD_LOCAL_MEM_LOAD_LOCK);
return;
}
/* Call find and lock to lock the port */
new_stack = mod_stack_create(stack->id, mod, stack->addr,
EV_MOD_LOCAL_MEM_LOAD_FINISH, stack);
new_stack->read = 1;
esim_schedule_event(EV_MOD_LOCAL_MEM_FIND_AND_LOCK, new_stack, 0);
return;
}
if (event == EV_MOD_LOCAL_MEM_LOAD_FINISH)
{
mem_debug("%lld %lld 0x%x %s load finish\n", esim_cycle, stack->id,
stack->addr, mod->name);
mem_trace("mem.access name=\"A-%lld\" state=\"%s:load_finish\"\n",
stack->id, mod->name);
mem_trace("mem.end_access name=\"A-%lld\"\n", stack->id);
/* Increment witness variable */
if (stack->witness_ptr) {
(*stack->witness_ptr)++;
}
/* Return event queue element into event queue */
if (stack->event_queue && stack->event_queue_item)
linked_list_add(stack->event_queue, stack->event_queue_item);
/* Finish access */
mod_access_finish(mod, stack);
/* Return */
mod_stack_return(stack);
return;
}
abort();
}
void net_command_handler(int event, void *data)
{
struct net_command_stack_t *stack= data;
struct net_t *net = stack->net;
char out_msg[MAX_STRING_SIZE];
char msg_detail[MAX_STRING_SIZE];
char *msg_str = out_msg;
int msg_size = sizeof out_msg;
char *msg_detail_str = msg_detail;
int msg_detail_size = sizeof msg_detail;
int test_failed;
test_failed = 0;
*msg_str = '\0';
*msg_detail_str = '\0';
long long cycle = esim_domain_cycle(net_domain_index);
if (event == EV_NET_COMMAND)
{
char *command_line = stack->command;
char command[MAX_STRING_SIZE];
struct list_t *token_list;
/* Split command in tokens, skip command */
token_list = str_token_list_create(command_line, " ");
assert(list_count(token_list));
long long command_cycle = net_command_get_llint(token_list,
command_line,"cycle value");
if (command_cycle > cycle)
{
str_token_list_free(token_list);
esim_schedule_event(event, stack, command_cycle - cycle);
return;
}
net_command_get_string(token_list, command_line, command,
sizeof command );
if (!strcasecmp(command, "Send"))
{
int msg_size;
long long int msg_id;
struct net_node_t *src_node;
struct net_node_t *dst_node;
/* Getting the Source Node from the command */
src_node = net_command_get_node(net,token_list,
command_line, net_node_end);
/* Getting the Source Node from the command */
dst_node = net_command_get_node(net, token_list,
command_line, net_node_end);
/* Getting the Message Size; or Default values */
msg_size = (int) net_command_get_def(token_list,
command_line,"Message size value",
(long long) net_msg_size);
/* Getting the Message ID; or default value */
msg_id = net_command_get_def(token_list, command_line,
"Message id value",
net->msg_id_counter + 1);
if (msg_id != net->msg_id_counter + 1)
{
fatal("%s: The message id is out of order. "
"\n\t\"You are out o' order. "
"The whole Trial is out o' order \" \n"
"\tBear with us. We are being "
"cute here \n\t> %s",
__FUNCTION__, command_line);
}
stack->msg = net_send_ev(net, src_node, dst_node, msg_size,
EV_NET_COMMAND_RCV, stack);
fprintf(stderr, "\n Message %lld sent at %lld \n\n", msg_id,
cycle);
}
else if (!strcasecmp(command, "Receive"))
{
long long msg_id;
struct net_node_t *dst_node;
struct net_msg_t *msg;
dst_node = net_command_get_node(net, token_list,
command_line, net_node_end);
msg_id = net_command_get_llint(token_list, command_line,
"Message id value");
/* Output */
str_printf(&msg_str, &msg_size,
"CHECK: Cycle %lld: Receive in node %s for message"
" %lld ",cycle, dst_node->name, msg_id);
msg = net_msg_table_get(net, msg_id);
if (!msg)
{
test_failed = 1;
str_printf(&msg_detail_str, &msg_detail_size,
"\t Message is either invalid, "
"not yet send or already received\n");
}
/* Checking : the node indicated in the receive command
* is the destination node for the message*/
else
{
if (dst_node != msg->dst_node)
warning("%s: The node %s in the receive "
"command is \n\tnot intended node "
"%s in the send command\n > %s"
,__FUNCTION__, dst_node->name,
msg->dst_node->name, command_line);
if (msg->node != dst_node)
{
test_failed = 1;
str_printf(&msg_detail_str, &msg_detail_size,
"\t Message expected to be "
"in node %s, but found in %s\n",
dst_node->name, msg->node->name);
}
else if (msg != list_get(msg->buffer->msg_list, 0))
{
test_failed = 1;
str_printf(&msg_detail_str, &msg_detail_size,
"\tMessage expected to be "
"ready for receive in node %s \n\tbut "
"not in the buffer head\n",
dst_node->name);
}
}
/* Output */
fprintf(stderr, ">>> %s - %s\n", out_msg, test_failed ?
"failed" : "passed");
fprintf(stderr, "%s", msg_detail);
net_command_stack_return(stack);
}
else if (!strcasecmp(command, "InBufferCheck"))
{
long long int msg_id;
struct net_msg_t * msg;
struct net_node_t * node;
struct net_buffer_t *buffer;
node = net_command_get_node(net, token_list,
command_line, 0);
msg_id = net_command_get_llint(token_list, command_line,
"invalid message id");
/* Output */
str_printf(&msg_str, &msg_size,
"CHECK: Cycle %lld: message %lld is in input buffer of "
"node %s", cycle, msg_id, node->name);
msg = net_msg_table_get(net, msg_id);
if (!msg)
{
test_failed = 1;
str_printf(&msg_detail_str, &msg_detail_size,
"\t Message is either invalid, "
"not yet send or already received\n");
}
else
{
buffer = msg->buffer;
if (msg->node != node)
{
test_failed = 1;
str_printf(&msg_detail_str, &msg_detail_size,
"\tMessage expected to be in node %s"
"but found in node %s \n",
node->name, msg->node->name);
}
else if (buffer != list_get(node->input_buffer_list,
buffer->index))
{
test_failed = 1;
str_printf(&msg_detail_str, &msg_detail_size,
"\t Message is not in any of input "
"buffers of the node %s\n",
node->name);
}
}
/* Output */
fprintf(stderr, ">>> %s - %s\n", out_msg, test_failed ?
"failed" : "passed");
fprintf(stderr, "%s", msg_detail);
net_command_stack_return(stack);
}
else if (!strcasecmp(command, "OutBufferCheck"))
{
long long int msg_id;
struct net_msg_t * msg;
struct net_node_t * node;
struct net_buffer_t *buffer;
node = net_command_get_node(net, token_list,
command_line, 0);
msg_id = net_command_get_llint(token_list, command_line,
"invalid message id");
/* Output */
str_printf(&msg_str, &msg_size,
"CHECK: Cycle %lld: message %lld is in one of output buffers"
" of node %s", cycle, msg_id, node->name);
msg = net_msg_table_get(net, msg_id);
if (!msg)
{
test_failed = 1;
str_printf(&msg_detail_str, &msg_detail_size,
"\t Message is either invalid, "
"not yet send or already received\n");
}
else
{
buffer = msg->buffer;
if (msg->node != node)
{
test_failed = 1;
str_printf(&msg_detail_str, &msg_detail_size,
"\tMessage expected to be in node %s"
"but found in node %s\n",
node->name, msg->node->name);
}
else if (buffer != list_get(node->output_buffer_list,
buffer->index))
{
test_failed = 1;
str_printf(&msg_detail_str, &msg_detail_size,
"\t Message is not in any of output"
"buffers of the node %s\n", node->name);
}
}
/* Output */
fprintf(stderr, ">>> %s - %s\n", out_msg, test_failed ?
"failed" : "passed");
fprintf(stderr, "%s", msg_detail);
net_command_stack_return(stack);
}
else if (!strcasecmp(command, "NodeCheck"))
{
long long int msg_id;
struct net_msg_t * msg;
struct net_node_t * node;
node = net_command_get_node(net, token_list,
command_line, 0);
msg_id = net_command_get_llint(token_list, command_line,
"invalid message id");
/* Output */
str_printf(&msg_str, &msg_size,
"CHECK: Cycle %lld: message %lld is in node %s", cycle,
msg_id, node->name);
msg = net_msg_table_get(net, msg_id);
if (!msg)
{
test_failed = 1;
str_printf(&msg_detail_str, &msg_detail_size,
"\t Message is either invalid, "
"not yet send or already received\n");
}
else
{
if (msg->node != node)
{
test_failed = 1;
str_printf(&msg_detail_str, &msg_detail_size,
"\tMessage is not in the node %s\n",
node->name);
}
}
/* Output */
fprintf(stderr, ">>> %s - %s\n", out_msg, test_failed ?
"failed" : "passed");
fprintf(stderr, "%s", msg_detail);
net_command_stack_return(stack);
}
else if (!strcasecmp(command, "ExactPosCheck"))
{
long long int msg_id;
struct net_msg_t * msg;
struct net_node_t * node;
struct net_buffer_t *buffer;
node = net_command_get_node(net, token_list,
command_line, 0);
buffer = net_command_get_buffer(node, token_list,
command_line);
msg_id = net_command_get_llint(token_list, command_line,
"invalid message id");
/* Output */
str_printf(&msg_str, &msg_size,
"CHECK: Cycle %lld: message %lld is in buffer %s"
" of node %s", cycle, msg_id, buffer->name, node->name);
/* Checks */
msg = net_msg_table_get(net, msg_id);
if (!msg)
{
test_failed = 1;
str_printf(&msg_detail_str, &msg_detail_size,
"\t Message is either invalid, "
"not yet send or already received\n");
}
else
{
if (msg->buffer != buffer || msg->node != node)
{
test_failed = 1;
str_printf(&msg_detail_str, &msg_detail_size,
"Message expected to be in buffer %s"
"(node %s) \n\tbut found in buffer %s"
"(node %s)\n",buffer->name, node->name,
msg->buffer->name, msg->node->name);
}
}
/* Output */
fprintf(stderr, ">>> %s - %s\n", out_msg, test_failed ?
"failed" : "passed");
fprintf(stderr, "%s", msg_detail);
net_command_stack_return(stack);
}
else
fatal("%s: %s: invalid command.\n\t> %s",
__FUNCTION__, command, command_line);
free(command_line);
str_token_list_free(token_list);
}
else if (event == EV_NET_COMMAND_RCV)
{
struct net_msg_t *msg;
struct net_node_t *dst_node;
msg = stack->msg;
dst_node = msg->dst_node;
assert(dst_node == msg->node);
fprintf(stderr, "\n Message %lld received at %lld \n\n", msg->id, cycle);
net_receive(net, dst_node, msg);
net_command_stack_return(stack);
}
}
void mod_handler_local_mem_store(int event, void *data)
{
struct mod_stack_t *stack = data;
struct mod_stack_t *new_stack;
struct mod_t *mod = stack->mod;
if (event == EV_MOD_LOCAL_MEM_STORE)
{
struct mod_stack_t *master_stack;
mem_debug("%lld %lld 0x%x %s store\n", esim_cycle, stack->id,
stack->addr, mod->name);
mem_trace("mem.new_access name=\"A-%lld\" type=\"store\" "
"state=\"%s:store\" addr=0x%x\n",
stack->id, mod->name, stack->addr);
/* Record access */
mod_access_start(mod, stack, mod_access_store);
/* Coalesce access */
master_stack = mod_can_coalesce(mod, mod_access_store, stack->addr, stack);
if (master_stack)
{
mod->writes++;
mod_coalesce(mod, master_stack, stack);
mod_stack_wait_in_stack(stack, master_stack, EV_MOD_LOCAL_MEM_STORE_FINISH);
/* Increment witness variable */
if (stack->witness_ptr)
(*stack->witness_ptr)++;
return;
}
/* Continue */
esim_schedule_event(EV_MOD_LOCAL_MEM_STORE_LOCK, stack, 0);
return;
}
if (event == EV_MOD_LOCAL_MEM_STORE_LOCK)
{
struct mod_stack_t *older_stack;
mem_debug(" %lld %lld 0x%x %s store lock\n", esim_cycle, stack->id,
stack->addr, mod->name);
mem_trace("mem.access name=\"A-%lld\" state=\"%s:store_lock\"\n",
stack->id, mod->name);
/* If there is any older access, wait for it */
older_stack = stack->access_list_prev;
if (older_stack)
{
mem_debug(" %lld wait for access %lld\n",
stack->id, older_stack->id);
mod_stack_wait_in_stack(stack, older_stack, EV_MOD_LOCAL_MEM_STORE_LOCK);
return;
}
/* Call find and lock */
new_stack = mod_stack_create(stack->id, mod, stack->addr,
EV_MOD_LOCAL_MEM_STORE_FINISH, stack);
new_stack->read = 0;
new_stack->witness_ptr = stack->witness_ptr;
esim_schedule_event(EV_MOD_LOCAL_MEM_FIND_AND_LOCK, new_stack, 0);
/* Set witness variable to NULL so that retries from the same
* stack do not increment it multiple times */
stack->witness_ptr = NULL;
return;
}
if (event == EV_MOD_LOCAL_MEM_STORE_FINISH)
{
mem_debug("%lld %lld 0x%x %s store finish\n", esim_cycle, stack->id,
stack->addr, mod->name);
mem_trace("mem.access name=\"A-%lld\" state=\"%s:store_finish\"\n",
stack->id, mod->name);
mem_trace("mem.end_access name=\"A-%lld\"\n", stack->id);
/* Return event queue element into event queue */
if (stack->event_queue && stack->event_queue_item)
linked_list_add(stack->event_queue, stack->event_queue_item);
/* Finish access */
mod_access_finish(mod, stack);
/* Return */
mod_stack_return(stack);
return;
}
abort();
}
void mod_handler_local_mem_find_and_lock(int event, void *data)
{
struct mod_stack_t *stack = data;
struct mod_stack_t *ret = stack->ret_stack;
struct mod_t *mod = stack->mod;
if (event == EV_MOD_LOCAL_MEM_FIND_AND_LOCK)
{
mem_debug(" %lld %lld 0x%x %s find and lock\n",
esim_cycle, stack->id, stack->addr, mod->name);
mem_trace("mem.access name=\"A-%lld\" state=\"%s:find_and_lock\"\n",
stack->id, mod->name);
/* Get a port */
mod_lock_port(mod, stack, EV_MOD_LOCAL_MEM_FIND_AND_LOCK_PORT);
return;
}
if (event == EV_MOD_LOCAL_MEM_FIND_AND_LOCK_PORT)
{
mem_debug(" %lld %lld 0x%x %s find and lock port\n", esim_cycle, stack->id,
stack->addr, mod->name);
mem_trace("mem.access name=\"A-%lld\" state=\"%s:find_and_lock_port\"\n",
stack->id, mod->name);
/* Set parent stack flag expressing that port has already been locked.
* This flag is checked by new writes to find out if it is already too
* late to coalesce. */
ret->port_locked = 1;
/* Statistics */
mod->accesses++;
if (stack->read)
{
mod->reads++;
mod->effective_reads++;
}
else
{
mod->writes++;
mod->effective_writes++;
/* Increment witness variable when port is locked */
if (stack->witness_ptr)
{
(*stack->witness_ptr)++;
stack->witness_ptr = NULL;
}
}
/* Access latency */
esim_schedule_event(EV_MOD_LOCAL_MEM_FIND_AND_LOCK_ACTION, stack, mod->latency);
return;
}
if (event == EV_MOD_LOCAL_MEM_FIND_AND_LOCK_ACTION)
{
struct mod_port_t *port = stack->port;
assert(port);
mem_debug(" %lld %lld 0x%x %s find and lock action\n", esim_cycle, stack->id,
stack->tag, mod->name);
mem_trace("mem.access name=\"A-%lld\" state=\"%s:find_and_lock_action\"\n",
stack->id, mod->name);
/* Release port */
mod_unlock_port(mod, port, stack);
/* Continue */
esim_schedule_event(EV_MOD_LOCAL_MEM_FIND_AND_LOCK_FINISH, stack, 0);
return;
}
if (event == EV_MOD_LOCAL_MEM_FIND_AND_LOCK_FINISH)
{
mem_debug(" %lld %lld 0x%x %s find and lock finish (err=%d)\n", esim_cycle, stack->id,
stack->tag, mod->name, stack->err);
mem_trace("mem.access name=\"A-%lld\" state=\"%s:find_and_lock_finish\"\n",
stack->id, mod->name);
mod_stack_return(stack);
return;
}
abort();
}
void moesi_handler_read_request(int event, void *data)
{
struct moesi_stack_t *stack = data, *ret = stack->retstack, *newstack;
struct ccache_t *ccache = stack->ccache, *target = stack->target;
uint32_t dir_entry_tag, z;
struct dir_t *dir;
struct dir_entry_t *dir_entry;
if (event == EV_MOESI_READ_REQUEST)
{
struct net_t *net;
int src, dest;
cache_debug(" %lld %lld 0x%x %s read request\n", CYCLE, ID,
stack->addr, ccache->name);
/* Default return values*/
ret->shared = 0;
ret->err = 0;
/* Send request to target */
assert(ccache->next == target || target->next == ccache);
net = ccache->next == target ? ccache->lonet : ccache->hinet;
src = ccache->next == target ? ccache->loid : 0;
dest = ccache->next == target ? 0 : target->loid;
net_send_ev(net, src, dest, 8, EV_MOESI_READ_REQUEST_RECEIVE, stack);
return;
}
if (event == EV_MOESI_READ_REQUEST_RECEIVE)
{
cache_debug(" %lld %lld 0x%x %s read request receive\n", CYCLE, ID,
stack->addr, target->name);
/* Find and lock */
newstack = moesi_stack_create(stack->id, target, stack->addr,
EV_MOESI_READ_REQUEST_ACTION, stack);
newstack->blocking = target->next == ccache;
newstack->read = 1;
newstack->retry = 0;
esim_schedule_event(EV_MOESI_FIND_AND_LOCK, newstack, 0);
return;
}
if (event == EV_MOESI_READ_REQUEST_ACTION)
{
cache_debug(" %lld %lld 0x%x %s read request action\n", CYCLE, ID,
stack->tag, target->name);
/* Check block locking error. If read request is down-up, there should not
* have been any error while locking. */
if (stack->err) {
assert(ccache->next == target);
ret->err = 1;
stack->response = 8;
esim_schedule_event(EV_MOESI_READ_REQUEST_REPLY, stack, 0);
return;
}
esim_schedule_event(ccache->next == target ? EV_MOESI_READ_REQUEST_UPDOWN :
EV_MOESI_READ_REQUEST_DOWNUP, stack, 0);
return;
}
if (event == EV_MOESI_READ_REQUEST_UPDOWN)
{
struct ccache_t *owner;
cache_debug(" %lld %lld 0x%x %s read request updown\n", CYCLE, ID,
stack->tag, target->name);
stack->pending = 1;
if (stack->status) {
/* Status = M/O/E/S
* Check: addr multiple of requester's bsize
* Check: no subblock requested by ccache is already owned by ccache */
assert(stack->addr % ccache->bsize == 0);
dir = ccache_get_dir(target, stack->tag);
for (z = 0; z < dir->zsize; z++) {
dir_entry_tag = stack->tag + z * cache_min_block_size;
if (dir_entry_tag < stack->addr || dir_entry_tag >= stack->addr + ccache->bsize)
continue;
dir_entry = ccache_get_dir_entry(target, stack->set, stack->way, z);
assert(dir_entry->owner != ccache->loid);
}
/* Send read request to owners other than ccache for all subblocks. */
for (z = 0; z < dir->zsize; z++) {
dir_entry = ccache_get_dir_entry(target, stack->set, stack->way, z);
dir_entry_tag = stack->tag + z * cache_min_block_size;
if (!dir_entry->owner) /* no owner */
continue;
if (dir_entry->owner == ccache->loid) /* owner is ccache */
continue;
owner = net_get_node_data(target->hinet, dir_entry->owner);
if (dir_entry_tag % owner->bsize) /* not the first owner subblock */
continue;
/* Send read request */
stack->pending++;
newstack = moesi_stack_create(stack->id, target, dir_entry_tag,
EV_MOESI_READ_REQUEST_UPDOWN_FINISH, stack);
newstack->target = owner;
esim_schedule_event(EV_MOESI_READ_REQUEST, newstack, 0);
}
esim_schedule_event(EV_MOESI_READ_REQUEST_UPDOWN_FINISH, stack, 0);
} else {
/* Status = I */
assert(!dir_entry_group_shared_or_owned(target->dir,
stack->set, stack->way));
newstack = moesi_stack_create(stack->id, target, stack->tag,
EV_MOESI_READ_REQUEST_UPDOWN_MISS, stack);
newstack->target = target->next;
esim_schedule_event(EV_MOESI_READ_REQUEST, newstack, 0);
}
return;
}
if (event == EV_MOESI_READ_REQUEST_UPDOWN_MISS)
{
cache_debug(" %lld %lld 0x%x %s read request updown miss\n", CYCLE, ID,
stack->tag, target->name);
/* Check error */
if (stack->err) {
dir_lock_unlock(stack->dir_lock);
ret->err = 1;
stack->response = 8;
esim_schedule_event(EV_MOESI_READ_REQUEST_REPLY, stack, 0);
return;
}
/* Set block state to excl/shared depending on the return value 'shared'
* that comes from a read request into the next cache level.
* Also set the tag of the block. */
cache_set_block(target->cache, stack->set, stack->way, stack->tag,
stack->shared ? moesi_status_shared : moesi_status_exclusive);
esim_schedule_event(EV_MOESI_READ_REQUEST_UPDOWN_FINISH, stack, 0);
return;
}
if (event == EV_MOESI_READ_REQUEST_UPDOWN_FINISH)
{
int shared;
/* Ignore while pending requests */
assert(stack->pending > 0);
stack->pending--;
if (stack->pending)
return;
cache_debug(" %lld %lld 0x%x %s read request updown finish\n", CYCLE, ID,
stack->tag, target->name);
/* Set owner to 0 for all directory entries not owned by ccache. */
dir = ccache_get_dir(target, stack->tag);
for (z = 0; z < dir->zsize; z++) {
dir_entry = ccache_get_dir_entry(target, stack->set, stack->way, z);
if (dir_entry->owner != ccache->loid)
dir_entry->owner = 0;
}
/* For each subblock requested by ccache, set ccache as sharer, and
* check whether there is other cache sharing it. */
shared = 0;
for (z = 0; z < dir->zsize; z++) {
dir_entry_tag = stack->tag + z * cache_min_block_size;
if (dir_entry_tag < stack->addr || dir_entry_tag >= stack->addr + ccache->bsize)
continue;
dir_entry = ccache_get_dir_entry(target, stack->set, stack->way, z);
dir_entry_set_sharer(dir, dir_entry, ccache->loid);
if (dir_entry->sharers > 1)
shared = 1;
}
/* If no subblock requested by ccache is shared by other cache, set ccache
* as owner of all of them. Otherwise, notify requester that the block is
* shared by setting the 'shared' return value to true. */
ret->shared = shared;
if (!shared) {
for (z = 0; z < dir->zsize; z++) {
dir_entry_tag = stack->tag + z * cache_min_block_size;
if (dir_entry_tag < stack->addr || dir_entry_tag >= stack->addr + ccache->bsize)
continue;
dir_entry = ccache_get_dir_entry(target, stack->set, stack->way, z);
dir_entry->owner = ccache->loid;
}
}
/* Respond with data, update LRU, unlock */
stack->response = ccache->bsize + 8;
if (target->cache)
cache_access_block(target->cache, stack->set, stack->way);
dir_lock_unlock(stack->dir_lock);
esim_schedule_event(EV_MOESI_READ_REQUEST_REPLY, stack, 0);
return;
}
if (event == EV_MOESI_READ_REQUEST_DOWNUP)
{
struct ccache_t *owner;
cache_debug(" %lld %lld 0x%x %s read request downup\n", CYCLE, ID,
stack->tag, target->name);
/* Check: status must not be invalid.
* By default, only one pending request.
* Response depends on status */
assert(stack->status != moesi_status_invalid);
stack->pending = 1;
stack->response = stack->status == moesi_status_exclusive ||
stack->status == moesi_status_shared ?
8 : target->bsize + 8;
/* Send a read request to the owner of each subblock. */
dir = ccache_get_dir(target, stack->tag);
for (z = 0; z < dir->zsize; z++) {
dir_entry_tag = stack->tag + z * cache_min_block_size;
dir_entry = ccache_get_dir_entry(target, stack->set, stack->way, z);
if (!dir_entry->owner) /* no owner */
continue;
owner = net_get_node_data(target->hinet, dir_entry->owner);
if (dir_entry_tag % owner->bsize) /* not the first subblock */
continue;
stack->pending++;
stack->response = target->bsize + 8;
newstack = moesi_stack_create(stack->id, target, dir_entry_tag,
EV_MOESI_READ_REQUEST_DOWNUP_FINISH, stack);
newstack->target = owner;
esim_schedule_event(EV_MOESI_READ_REQUEST, newstack, 0);
}
esim_schedule_event(EV_MOESI_READ_REQUEST_DOWNUP_FINISH, stack, 0);
return;
}
if (event == EV_MOESI_READ_REQUEST_DOWNUP_FINISH)
{
/* Ignore while pending requests */
assert(stack->pending > 0);
stack->pending--;
if (stack->pending)
return;
cache_debug(" %lld %lld 0x%x %s read request downup finish\n", CYCLE, ID,
stack->tag, target->name);
/* Set owner of subblocks to 0. */
dir = ccache_get_dir(target, stack->tag);
for (z = 0; z < dir->zsize; z++) {
dir_entry_tag = stack->tag + z * cache_min_block_size;
dir_entry = ccache_get_dir_entry(target, stack->set, stack->way, z);
dir_entry->owner = 0;
}
/* Set status to S, update LRU, unlock */
cache_set_block(target->cache, stack->set, stack->way, stack->tag,
moesi_status_shared);
cache_access_block(target->cache, stack->set, stack->way);
dir_lock_unlock(stack->dir_lock);
esim_schedule_event(EV_MOESI_READ_REQUEST_REPLY, stack, 0);
return;
}
if (event == EV_MOESI_READ_REQUEST_REPLY)
{
struct net_t *net;
int src, dest;
cache_debug(" %lld %lld 0x%x %s read request reply\n", CYCLE, ID,
stack->tag, target->name);
assert(stack->response);
assert(ccache->next == target || target->next == ccache);
net = ccache->next == target ? ccache->lonet : ccache->hinet;
src = ccache->next == target ? 0 : target->loid;
dest = ccache->next == target ? ccache->loid : 0;
net_send_ev(net, src, dest, stack->response,
EV_MOESI_READ_REQUEST_FINISH, stack);
return;
}
if (event == EV_MOESI_READ_REQUEST_FINISH)
{
cache_debug(" %lld %lld 0x%x %s read request finish\n", CYCLE, ID,
stack->tag, ccache->name);
moesi_stack_return(stack);
return;
}
abort();
}
void dram_controller_schedule_command(struct dram_controller_t *controller)
{
int i, j, k, num_info_per_scheduler, valid;
struct dram_command_t *command;
struct dram_bank_info_t *info;
struct dram_command_scheduler_t *scheduler;
num_info_per_scheduler = controller->dram_num_ranks * controller->dram_num_banks_per_device;
for (i = 0; i < controller->num_physical_channels; i++)
{
/* Get scheduler */
scheduler = list_get(controller->dram_command_scheduler_list, i);
for (j = 0; j < num_info_per_scheduler; j++)
{
switch (controller->scheduling_policy)
{
case rank_bank_round_robin:
/* Locate bank info */
scheduler->last_scheduled_rank_id++;
if (scheduler->last_scheduled_rank_id == controller->dram_num_ranks)
{
scheduler->last_scheduled_rank_id = 0;
scheduler->last_scheduled_bank_id ++;
if (scheduler->last_scheduled_bank_id == controller->dram_num_banks_per_device)
scheduler->last_scheduled_bank_id = 0;
}
break;
case bank_rank_round_robin:
/* Locate bank info */
scheduler->last_scheduled_bank_id++;
if (scheduler->last_scheduled_bank_id == controller->dram_num_banks_per_device)
{
scheduler->last_scheduled_bank_id = 0;
scheduler->last_scheduled_rank_id ++;
if (scheduler->last_scheduled_rank_id == controller->dram_num_ranks)
scheduler->last_scheduled_rank_id = 0;
}
break;
default:
break;
}
info = list_get(controller->dram_bank_info_list, scheduler->last_scheduled_bank_id + scheduler->last_scheduled_rank_id * controller->dram_num_banks_per_device + scheduler->channel_id * controller->dram_num_ranks * controller->dram_num_banks_per_device);
/* Fetch a command from command queue */
command = list_head(info->command_queue);
if (command)
{
/* Check timing */
valid = 1;
for (k = 0; k < DRAM_TIMING_MATRIX_SIZE; k++)
{
if (esim_cycle - info->dram_bank_info_last_scheduled_time_matrix[k] < controller->dram_timing_matrix[command->type][k])
valid = 0;
}
/* If timing is valid, schedule the command */
if (valid)
{
/* Dequeue command */
command = list_dequeue(info->command_queue);
/* Schedule command receive */
esim_schedule_event(EV_DRAM_COMMAND_RECEIVE, command, 0);
/* Update last scheduled time matrix */
info->dram_bank_info_last_scheduled_time_matrix[command->type] = esim_cycle;
}
}
}
}
}
void moesi_handler_evict(int event, void *data)
{
struct moesi_stack_t *stack = data, *ret = stack->retstack, *newstack;
struct ccache_t *ccache = stack->ccache, *target = stack->target;
struct dir_t *dir;
struct dir_entry_t *dir_entry;
uint32_t dir_entry_tag, z;
if (event == EV_MOESI_EVICT)
{
/* Default ret value */
ret->err = 0;
/* Get block info */
ccache_get_block(ccache, stack->set, stack->way, &stack->tag, &stack->status);
assert(stack->status || !dir_entry_group_shared_or_owned(ccache->dir,
stack->set, stack->way));
cache_debug(" %lld %lld 0x%x %s evict (set=%d, way=%d, status=%d)\n", CYCLE, ID,
stack->tag, ccache->name, stack->set, stack->way, stack->status);
/* Save some data */
stack->src_set = stack->set;
stack->src_way = stack->way;
stack->src_tag = stack->tag;
stack->target = target = ccache->next;
/* Send write request to all sharers */
newstack = moesi_stack_create(stack->id, ccache, 0,
EV_MOESI_EVICT_ACTION, stack);
newstack->except = NULL;
newstack->set = stack->set;
newstack->way = stack->way;
esim_schedule_event(EV_MOESI_INVALIDATE, newstack, 0);
return;
}
if (event == EV_MOESI_EVICT_ACTION)
{
cache_debug(" %lld %lld 0x%x %s evict action\n", CYCLE, ID,
stack->tag, ccache->name);
/* status = I */
if (stack->status == moesi_status_invalid) {
esim_schedule_event(EV_MOESI_EVICT_FINISH, stack, 0);
return;
}
/* status = M/O */
if (stack->status == moesi_status_modified ||
stack->status == moesi_status_owned) {
net_send_ev(ccache->lonet, ccache->loid, 0,
ccache->bsize + 8, EV_MOESI_EVICT_RECEIVE, stack);
stack->writeback = 1;
return;
}
/* status = S/E */
net_send_ev(ccache->lonet, ccache->loid, 0, 8,
EV_MOESI_EVICT_RECEIVE, stack);
return;
}
if (event == EV_MOESI_EVICT_RECEIVE) {
cache_debug(" %lld %lld 0x%x %s evict receive\n", CYCLE, ID,
stack->tag, target->name);
/* Find and lock */
newstack = moesi_stack_create(stack->id, target, stack->src_tag,
EV_MOESI_EVICT_WRITEBACK, stack);
newstack->blocking = 0;
newstack->read = 0;
newstack->retry = 0;
esim_schedule_event(EV_MOESI_FIND_AND_LOCK, newstack, 0);
return;
}
if (event == EV_MOESI_EVICT_WRITEBACK)
{
cache_debug(" %lld %lld 0x%x %s evict writeback\n", CYCLE, ID,
stack->tag, target->name);
/* Error locking block */
if (stack->err) {
ret->err = 1;
esim_schedule_event(EV_MOESI_EVICT_REPLY, stack, 0);
return;
}
/* No writeback */
if (!stack->writeback) {
esim_schedule_event(EV_MOESI_EVICT_PROCESS, stack, 0);
return;
}
/* Writeback */
newstack = moesi_stack_create(stack->id, target, 0,
EV_MOESI_EVICT_WRITEBACK_EXCLUSIVE, stack);
newstack->except = ccache;
newstack->set = stack->set;
newstack->way = stack->way;
esim_schedule_event(EV_MOESI_INVALIDATE, newstack, 0);
return;
}
if (event == EV_MOESI_EVICT_WRITEBACK_EXCLUSIVE)
{
cache_debug(" %lld %lld 0x%x %s evict writeback exclusive\n", CYCLE, ID,
stack->tag, target->name);
/* Status = O/S/I */
assert(stack->status != moesi_status_invalid);
if (stack->status == moesi_status_owned || stack->status ==
moesi_status_shared)
{
newstack = moesi_stack_create(stack->id, target, stack->tag,
EV_MOESI_EVICT_WRITEBACK_FINISH, stack);
newstack->target = target->next;
esim_schedule_event(EV_MOESI_WRITE_REQUEST, newstack, 0);
return;
}
/* Status = M/E */
esim_schedule_event(EV_MOESI_EVICT_WRITEBACK_FINISH, stack, 0);
return;
}
if (event == EV_MOESI_EVICT_WRITEBACK_FINISH)
{
cache_debug(" %lld %lld 0x%x %s evict writeback finish\n", CYCLE, ID,
stack->tag, target->name);
/* Error in write request */
if (stack->err) {
ret->err = 1;
dir_lock_unlock(stack->dir_lock);
esim_schedule_event(EV_MOESI_EVICT_REPLY, stack, 0);
return;
}
/* Set tag, status and lru */
if (target->cache) {
cache_set_block(target->cache, stack->set, stack->way, stack->tag,
moesi_status_modified);
cache_access_block(target->cache, stack->set, stack->way);
}
esim_schedule_event(EV_MOESI_EVICT_PROCESS, stack, 0);
return;
}
if (event == EV_MOESI_EVICT_PROCESS)
{
cache_debug(" %lld %lld 0x%x %s evict process\n", CYCLE, ID,
stack->tag, target->name);
/* Remove sharer, owner, and unlock */
dir = ccache_get_dir(target, stack->tag);
for (z = 0; z < dir->zsize; z++) {
dir_entry_tag = stack->tag + z * cache_min_block_size;
if (dir_entry_tag < stack->src_tag || dir_entry_tag >= stack->src_tag + ccache->bsize)
continue;
dir_entry = ccache_get_dir_entry(target, stack->set, stack->way, z);
dir_entry_clear_sharer(dir, dir_entry, ccache->loid);
if (dir_entry->owner == ccache->loid)
dir_entry->owner = 0;
}
dir_lock_unlock(stack->dir_lock);
esim_schedule_event(EV_MOESI_EVICT_REPLY, stack, 0);
return;
}
if (event == EV_MOESI_EVICT_REPLY)
{
cache_debug(" %lld %lld 0x%x %s evict reply\n", CYCLE, ID,
stack->tag, target->name);
net_send_ev(target->hinet, 0, ccache->loid, 8,
EV_MOESI_EVICT_REPLY_RECEIVE, stack);
return;
}
if (event == EV_MOESI_EVICT_REPLY_RECEIVE)
{
cache_debug(" %lld %lld 0x%x %s evict reply receive\n", CYCLE, ID,
stack->tag, ccache->name);
/* Invalidate block if there was no error. */
if (!stack->err)
cache_set_block(ccache->cache, stack->src_set, stack->src_way,
0, moesi_status_invalid);
assert(!dir_entry_group_shared_or_owned(ccache->dir,
stack->src_set, stack->src_way));
esim_schedule_event(EV_MOESI_EVICT_FINISH, stack, 0);
return;
}
if (event == EV_MOESI_EVICT_FINISH)
{
cache_debug(" %lld %lld 0x%x %s evict finish\n", CYCLE, ID,
stack->tag, ccache->name);
moesi_stack_return(stack);
return;
}
abort();
}
void net_event_handler(int event, void *data)
{
struct net_stack_t *stack = data;
struct net_t *net = stack->net;
struct net_routing_table_t *routing_table = net->routing_table;
struct net_msg_t *msg = stack->msg;
struct net_node_t *src_node = msg->src_node;
struct net_node_t *dst_node = msg->dst_node;
struct net_node_t *node = msg->node;
struct net_buffer_t *buffer = msg->buffer;
if (event == EV_NET_SEND)
{
struct net_routing_table_entry_t *entry;
struct net_buffer_t *output_buffer;
/* Debug */
net_debug("msg "
"a=\"send\" "
"net=\"%s\" "
"msg=%lld "
"size=%d "
"src=\"%s\" "
"dst=\"%s\"\n",
net->name,
msg->id,
msg->size,
src_node->name,
dst_node->name);
/* Get output buffer */
entry = net_routing_table_lookup(routing_table, src_node, dst_node);
output_buffer = entry->output_buffer;
if (!output_buffer)
fatal("%s: no route from %s to %s.\n%s", net->name, src_node->name,
dst_node->name, net_err_no_route);
if (output_buffer->write_busy >= esim_cycle)
panic("%s: output buffer busy.\n%s", __FUNCTION__, net_err_can_send);
if (msg->size > output_buffer->size)
panic("%s: message does not fit in buffer.\n%s", __FUNCTION__, net_err_can_send);
if (output_buffer->count + msg->size > output_buffer->size)
panic("%s: output buffer full.\n%s", __FUNCTION__, net_err_can_send);
/* Insert in output buffer (1 cycle latency) */
net_buffer_insert(output_buffer, msg);
output_buffer->write_busy = esim_cycle;
msg->node = src_node;
msg->buffer = output_buffer;
msg->busy = esim_cycle;
/* Schedule next event */
esim_schedule_event(EV_NET_OUTPUT_BUFFER, stack, 1);
}
else if (event == EV_NET_OUTPUT_BUFFER)
{
struct net_link_t *link;
struct net_buffer_t *input_buffer;
int lat;
/* Debug */
net_debug("msg "
"a=\"obuf\" "
"net=\"%s\" "
"msg=%lld "
"node=\"%s\" "
"buf=\"%s\"\n",
net->name,
msg->id,
node->name,
buffer->name);
/* If message is not at buffer head, process later */
assert(list_count(buffer->msg_list));
if (list_get(buffer->msg_list, 0) != msg)
{
net_buffer_wait(buffer, event, stack);
return;
}
/* If source output buffer is busy, wait */
if (buffer->read_busy >= esim_cycle)
{
esim_schedule_event(event, stack, buffer->read_busy - esim_cycle + 1);
return;
}
/* If link is busy, wait */
link = buffer->link;
if (link->busy >= esim_cycle)
{
esim_schedule_event(event, stack, link->busy - esim_cycle + 1);
return;
}
/* If buffer contain the message but doesn't have the shared link in control, wait*/
if (link->virtual_channel > 1)
{
struct net_buffer_t *temp_buffer;
temp_buffer = net_link_arbitrator_vc(link, node);
if (temp_buffer != buffer)
{
net_debug("msg "
"a=\"arbitrator stall\" "
"net=\"%s\" "
"msg=%lld "
"why=\"sched\"\n",
net->name,
msg->id);
esim_schedule_event(event, stack, 1);
return;
}
}
/* If destination input buffer is busy, wait */
assert(buffer == link->src_buffer);
input_buffer = link->dst_buffer;
if (input_buffer->write_busy >= esim_cycle)
{
esim_schedule_event(event, stack, input_buffer->write_busy - esim_cycle + 1);
return;
}
/* If destination input buffer is full, wait */
if (msg->size > input_buffer->size)
fatal("%s: message does not fit in buffer.\n%s",
net->name, net_err_large_message);
if (input_buffer->count + msg->size > input_buffer->size)
{
net_buffer_wait(input_buffer, event, stack);
return;
}
/* Calculate latency and occupy resources */
lat = (msg->size - 1) / link->bandwidth + 1;
assert(lat > 0);
buffer->read_busy = esim_cycle + lat - 1;
link->busy = esim_cycle + lat - 1;
input_buffer->write_busy = esim_cycle + lat - 1;
/* Transfer message to next input buffer */
assert(msg->busy < esim_cycle);
net_buffer_extract(buffer, msg);
net_buffer_insert(input_buffer, msg);
msg->node = input_buffer->node;
msg->buffer = input_buffer;
msg->busy = esim_cycle + lat - 1;
/* Stats */
link->busy_cycles += lat;
link->transferred_bytes += msg->size;
link->transferred_msgs++;
node->bytes_sent += msg->size;
node->msgs_sent++;
input_buffer->node->bytes_received += msg->size;
input_buffer->node->msgs_received++;
/* Schedule next event */
esim_schedule_event(EV_NET_INPUT_BUFFER, stack, lat);
}
else if (event == EV_NET_INPUT_BUFFER)
{
struct net_routing_table_entry_t *entry;
struct net_buffer_t *output_buffer;
int lat;
/* Debug */
net_debug("msg "
"a=\"ibuf\" "
"net=\"%s\" "
"msg=%lld "
"node=\"%s\" "
"buf=\"%s\"\n",
net->name,
msg->id,
node->name,
buffer->name);
/* If message is not at buffer head, process later */
assert(list_count(buffer->msg_list));
if (list_get(buffer->msg_list, 0) != msg)
{
net_debug("msg "
"a=\"stall\" "
"net=\"%s\" "
"msg=%lld "
"why=\"not-head\"\n",
net->name,
msg->id);
net_buffer_wait(buffer, event, stack);
return;
}
/* If this is the destination node, finish */
if (node == msg->dst_node)
{
esim_schedule_event(EV_NET_RECEIVE, stack, 0);
return;
}
/* If source input buffer is busy, wait */
if (buffer->read_busy >= esim_cycle)
{
net_debug("msg "
"a=\"stall\" "
"net=\"%s\" "
"msg=%lld "
"why=\"src-busy\"\n",
net->name,
msg->id);
esim_schedule_event(event, stack, buffer->read_busy - esim_cycle + 1);
return;
}
/* Get output buffer */
entry = net_routing_table_lookup(routing_table, node, dst_node);
output_buffer = entry->output_buffer;
if (!output_buffer)
fatal("%s: no route from %s to %s.\n%s", net->name,
node->name, dst_node->name, net_err_no_route);
/* If destination output buffer is busy, wait */
if (output_buffer->write_busy >= esim_cycle)
{
net_debug("msg "
"a=\"stall\" "
"net=\"%s\" "
"msg=%lld "
"why=\"dst-busy\"\n",
net->name,
msg->id);
esim_schedule_event(event, stack, output_buffer->write_busy - esim_cycle + 1);
return;
}
/* If destination output buffer is full, wait */
if (msg->size > output_buffer->size)
fatal("%s: message does not fit in buffer.\n%s",
net->name, net_err_large_message);
if (output_buffer->count + msg->size > output_buffer->size)
{
net_debug("msg "
"a=\"stall\" "
"net=\"%s\" "
"msg=%lld "
"why=\"dst-full\"\n",
net->name,
msg->id);
net_buffer_wait(output_buffer, event, stack);
return;
}
/* If scheduler says that it is not our turn, try later */
if (net_node_schedule(node, output_buffer) != buffer)
{
net_debug("msg "
"a=\"stall\" "
"net=\"%s\" "
"msg=%lld "
"why=\"sched\"\n",
net->name,
msg->id);
esim_schedule_event(event, stack, 1);
return;
}
/* Calculate latency and occupy resources */
assert(node->kind != net_node_end);
assert(node->bandwidth > 0);
lat = (msg->size - 1) / node->bandwidth + 1;
assert(lat > 0);
buffer->read_busy = esim_cycle + lat - 1;
output_buffer->write_busy = esim_cycle + lat - 1;
/* Transfer message to next output buffer */
assert(msg->busy < esim_cycle);
net_buffer_extract(buffer, msg);
net_buffer_insert(output_buffer, msg);
msg->buffer = output_buffer;
msg->busy = esim_cycle + lat - 1;
/* Schedule next event */
esim_schedule_event(EV_NET_OUTPUT_BUFFER, stack, lat);
}
else if (event == EV_NET_RECEIVE)
{
/* Debug */
net_debug("msg "
"a=\"receive\" "
"net=\"%s\" "
"msg=%lld "
"node=\"%s\"\n",
net->name,
msg->id,
dst_node->name);
/* Stats */
net->transfers++;
net->lat_acc += esim_cycle - msg->send_cycle;
net->msg_size_acc += msg->size;
/* If not return event was specified, free message here */
if (stack->ret_event == ESIM_EV_NONE)
net_receive(net, node, msg);
/* Finish */
net_stack_return(stack);
}
else
{
panic("%s: unknown event", __FUNCTION__);
}
}
void moesi_handler_store(int event, void *data)
{
struct moesi_stack_t *stack = data, *newstack;
struct ccache_t *ccache = stack->ccache;
if (event == EV_MOESI_STORE)
{
cache_debug("%lld %lld 0x%x %s store\n", CYCLE, ID,
stack->addr, ccache->name);
/* Call find and lock */
newstack = moesi_stack_create(stack->id, ccache, stack->addr,
EV_MOESI_STORE_ACTION, stack);
newstack->blocking = 0;
newstack->read = 0;
newstack->retry = stack->retry;
esim_schedule_event(EV_MOESI_FIND_AND_LOCK, newstack, 0);
return;
}
if (event == EV_MOESI_STORE_ACTION)
{
int retry_lat;
cache_debug(" %lld %lld 0x%x %s store action\n", CYCLE, ID,
stack->tag, ccache->name);
/* Error locking */
if (stack->err) {
ccache->write_retries++;
retry_lat = RETRY_LATENCY;
cache_debug(" lock error, retrying in %d cycles\n", retry_lat);
stack->retry = 1;
esim_schedule_event(EV_MOESI_STORE, stack, retry_lat);
return;
}
/* Hit - status=M/E */
if (stack->status == moesi_status_modified ||
stack->status == moesi_status_exclusive)
{
esim_schedule_event(EV_MOESI_STORE_FINISH, stack, 0);
return;
}
/* Miss - status=O/S/I */
newstack = moesi_stack_create(stack->id, ccache, stack->tag,
EV_MOESI_STORE_FINISH, stack);
newstack->target = ccache->next;
esim_schedule_event(EV_MOESI_WRITE_REQUEST, newstack, 0);
return;
}
if (event == EV_MOESI_STORE_FINISH)
{
int retry_lat;
cache_debug("%lld %lld 0x%x %s store finish\n", CYCLE, ID,
stack->tag, ccache->name);
/* Error in write request, unlock block and retry store. */
if (stack->err) {
ccache->write_retries++;
retry_lat = RETRY_LATENCY;
dir_lock_unlock(stack->dir_lock);
cache_debug(" lock error, retrying in %d cycles\n", retry_lat);
stack->retry = 1;
esim_schedule_event(EV_MOESI_STORE, stack, retry_lat);
return;
}
/* Update LRU, tag/status, unlock, and return. */
if (ccache->cache) {
cache_access_block(ccache->cache, stack->set, stack->way);
cache_set_block(ccache->cache, stack->set, stack->way,
stack->tag, moesi_status_modified);
}
dir_lock_unlock(stack->dir_lock);
moesi_stack_return(stack);
return;
}
abort();
}
void moesi_handler_load(int event, void *data)
{
struct moesi_stack_t *stack = data, *newstack;
struct ccache_t *ccache = stack->ccache;
if (event == EV_MOESI_LOAD)
{
cache_debug("%lld %lld 0x%x %s load\n", CYCLE, ID,
stack->addr, ccache->name);
/* Call find and lock */
newstack = moesi_stack_create(stack->id, ccache, stack->addr,
EV_MOESI_LOAD_ACTION, stack);
newstack->blocking = 0;
newstack->read = 1;
newstack->retry = stack->retry;
esim_schedule_event(EV_MOESI_FIND_AND_LOCK, newstack, 0);
return;
}
if (event == EV_MOESI_LOAD_ACTION)
{
int retry_lat;
cache_debug(" %lld %lld 0x%x %s load action\n", CYCLE, ID,
stack->tag, ccache->name);
/* Error locking */
if (stack->err) {
ccache->read_retries++;
retry_lat = RETRY_LATENCY;
cache_debug(" lock error, retrying in %d cycles\n", retry_lat);
stack->retry = 1;
esim_schedule_event(EV_MOESI_LOAD, stack, retry_lat);
return;
}
/* Hit */
if (stack->status) {
esim_schedule_event(EV_MOESI_LOAD_FINISH, stack, 0);
return;
}
/* Miss */
newstack = moesi_stack_create(stack->id, ccache, stack->tag,
EV_MOESI_LOAD_MISS, stack);
newstack->target = ccache->next;
esim_schedule_event(EV_MOESI_READ_REQUEST, newstack, 0);
return;
}
if (event == EV_MOESI_LOAD_MISS)
{
int retry_lat;
cache_debug(" %lld %lld 0x%x %s load miss\n", CYCLE, ID,
stack->tag, ccache->name);
/* Error on read request. Unlock block and retry load. */
if (stack->err) {
ccache->read_retries++;
retry_lat = RETRY_LATENCY;
dir_lock_unlock(stack->dir_lock);
cache_debug(" lock error, retrying in %d cycles\n", retry_lat);
stack->retry = 1;
esim_schedule_event(EV_MOESI_LOAD, stack, retry_lat);
return;
}
/* Set block state to excl/shared depending on return var 'shared'.
* Also set the tag of the block. */
cache_set_block(ccache->cache, stack->set, stack->way, stack->tag,
stack->shared ? moesi_status_shared : moesi_status_exclusive);
/* Continue */
esim_schedule_event(EV_MOESI_LOAD_FINISH, stack, 0);
return;
}
if (event == EV_MOESI_LOAD_FINISH)
{
cache_debug("%lld %lld 0x%x %s load finish\n", CYCLE, ID,
stack->tag, ccache->name);
/* Update LRU, unlock, and return. */
if (ccache->cache)
cache_access_block(ccache->cache, stack->set, stack->way);
dir_lock_unlock(stack->dir_lock);
moesi_stack_return(stack);
return;
}
abort();
}
void moesi_handler_find_and_lock(int event, void *data)
{
struct moesi_stack_t *stack = data, *ret = stack->retstack, *newstack;
struct ccache_t *ccache = stack->ccache;
if (event == EV_MOESI_FIND_AND_LOCK)
{
int hit;
cache_debug(" %lld %lld 0x%x %s find and lock (blocking=%d)\n", CYCLE, ID,
stack->addr, ccache->name, stack->blocking);
/* Default return values */
ret->err = 0;
ret->set = 0;
ret->way = 0;
ret->status = 0;
ret->tag = 0;
/* Look for block. */
hit = ccache_find_block(ccache, stack->addr, &stack->set,
&stack->way, &stack->tag, &stack->status);
if (hit)
cache_debug(" %lld 0x%x %s hit: set=%d, way=%d, status=%d\n", ID,
stack->tag, ccache->name, stack->set, stack->way, stack->status);
/* Stats */
ccache->accesses++;
if (hit)
ccache->hits++;
if (stack->read) {
ccache->reads++;
stack->blocking ? ccache->blocking_reads++ : ccache->non_blocking_reads++;
if (hit)
ccache->read_hits++;
} else {
ccache->writes++;
stack->blocking ? ccache->blocking_writes++ : ccache->non_blocking_writes++;
if (hit)
ccache->write_hits++;
}
if (!stack->retry) {
ccache->no_retry_accesses++;
if (hit)
ccache->no_retry_hits++;
if (stack->read) {
ccache->no_retry_reads++;
if (hit)
ccache->no_retry_read_hits++;
} else {
ccache->no_retry_writes++;
if (hit)
ccache->no_retry_write_hits++;
}
}
/* Miss */
if (!hit) {
assert(!stack->blocking);
assert(ccache != main_memory);
/* Find victim */
stack->way = cache_replace_block(ccache->cache, stack->set);
cache_get_block(ccache->cache, stack->set, stack->way, NULL, &stack->status);
assert(stack->status || !dir_entry_group_shared_or_owned(ccache->dir,
stack->set, stack->way));
cache_debug(" %lld 0x%x %s miss -> lru: set=%d, way=%d, status=%d\n",
ID, stack->tag, ccache->name, stack->set, stack->way, stack->status);
}
/* Lock entry */
stack->dir_lock = ccache_get_dir_lock(ccache, stack->set, stack->way);
if (stack->dir_lock->lock && !stack->blocking) {
cache_debug(" %lld 0x%x %s block already locked: set=%d, way=%d\n",
ID, stack->tag, ccache->name, stack->set, stack->way);
ret->err = 1;
moesi_stack_return(stack);
return;
}
if (!dir_lock_lock(stack->dir_lock, EV_MOESI_FIND_AND_LOCK, stack))
return;
/* Entry is locked. Record the transient tag so that a subsequent lookup
* detects that the block is being brought. */
if (ccache->cache)
cache_set_transient_tag(ccache->cache, stack->set, stack->way, stack->tag);
/* On miss, evict if victim is a valid block. */
if (!hit && stack->status) {
stack->eviction = 1;
newstack = moesi_stack_create(stack->id, ccache, 0,
EV_MOESI_FIND_AND_LOCK_FINISH, stack);
newstack->set = stack->set;
newstack->way = stack->way;
esim_schedule_event(EV_MOESI_EVICT, newstack, ccache->lat);
return;
}
/* Access latency */
esim_schedule_event(EV_MOESI_FIND_AND_LOCK_FINISH, stack, ccache->lat);
return;
}
if (event == EV_MOESI_FIND_AND_LOCK_FINISH)
{
cache_debug(" %lld %lld 0x%x %s find and lock finish (err=%d)\n", CYCLE, ID,
stack->tag, ccache->name, stack->err);
/* If evict produced err, return err */
if (stack->err) {
cache_get_block(ccache->cache, stack->set, stack->way, NULL, &stack->status);
assert(stack->status);
assert(stack->eviction);
ret->err = 1;
dir_lock_unlock(stack->dir_lock);
moesi_stack_return(stack);
return;
}
/* Eviction */
if (stack->eviction) {
ccache->evictions++;
cache_get_block(ccache->cache, stack->set, stack->way, NULL, &stack->status);
assert(!stack->status);
}
/* Return */
ret->err = 0;
ret->set = stack->set;
ret->way = stack->way;
ret->status = stack->status;
ret->tag = stack->tag;
ret->dir_lock = stack->dir_lock;
moesi_stack_return(stack);
return;
}
abort();
}
void moesi_handler_invalidate(int event, void *data)
{
struct moesi_stack_t *stack = data, *newstack;
struct ccache_t *ccache = stack->ccache;
struct dir_t *dir;
struct dir_entry_t *dir_entry;
uint32_t dir_entry_tag, z;
if (event == EV_MOESI_INVALIDATE)
{
int node_count, i;
struct ccache_t *sharer;
/* Get block info */
ccache_get_block(ccache, stack->set, stack->way, &stack->tag, &stack->status);
cache_debug(" %lld %lld 0x%x %s invalidate (set=%d, way=%d, status=%d)\n", CYCLE, ID,
stack->tag, ccache->name, stack->set, stack->way, stack->status);
stack->pending = 1;
/* Send write request to all upper level sharers but ccache */
dir = ccache_get_dir(ccache, stack->tag);
for (z = 0; z < dir->zsize; z++) {
dir_entry_tag = stack->tag + z * cache_min_block_size;
dir_entry = ccache_get_dir_entry(ccache, stack->set, stack->way, z);
node_count = ccache->hinet ? ccache->hinet->end_node_count : 0;
for (i = 1; i < node_count; i++) {
/* Skip non-sharers and 'except' */
if (!dir_entry_is_sharer(dir, dir_entry, i))
continue;
sharer = net_get_node_data(ccache->hinet, i);
if (sharer == stack->except)
continue;
/* Clear sharer and owner */
dir_entry_clear_sharer(dir, dir_entry, i);
if (dir_entry->owner == i)
dir_entry->owner = 0;
/* Send write request upwards if beginning of block */
if (dir_entry_tag % sharer->bsize)
continue;
newstack = moesi_stack_create(stack->id, ccache, dir_entry_tag,
EV_MOESI_INVALIDATE_FINISH, stack);
newstack->target = sharer;
esim_schedule_event(EV_MOESI_WRITE_REQUEST, newstack, 0);
stack->pending++;
}
}
esim_schedule_event(EV_MOESI_INVALIDATE_FINISH, stack, 0);
return;
}
if (event == EV_MOESI_INVALIDATE_FINISH)
{
cache_debug(" %lld %lld 0x%x %s invalidate finish\n", CYCLE, ID,
stack->tag, ccache->name);
/* Ignore while pending */
assert(stack->pending > 0);
stack->pending--;
if (stack->pending)
return;
moesi_stack_return(stack);
return;
}
abort();
}
void net_event_handler(int event, void *data)
{
struct net_stack_t *stack = data;
struct net_t *net = stack->net;
struct net_routing_table_t *routing_table = net->routing_table;
struct net_packet_t *pkt= stack->packet;
struct net_node_t *src_node = pkt->msg->src_node;
struct net_node_t *dst_node = pkt->msg->dst_node;
struct net_node_t *node = pkt->node;
struct net_buffer_t *buffer = pkt->buffer;
long long cycle;
/* Get current cycle */
cycle = esim_domain_cycle(net_domain_index);
if ((net_snap_period != 0) &&
(net->last_recorded_cycle < (cycle/net_snap_period )))
net_bandwidth_snapshot(net, cycle);
if (event == EV_NET_SEND)
{
struct net_routing_table_entry_t *entry;
struct net_buffer_t *output_buffer;
if (net->magicNet)
{
/* Magic Net work-around */
src_node->bytes_sent += pkt->size;
src_node->msgs_sent++;
dst_node->bytes_received += pkt->size;
dst_node->msgs_received++;
pkt->node = dst_node;
esim_schedule_event(EV_NET_RECEIVE, stack,
net->fixed_delay);
}
/* Get output buffer */
entry = net_routing_table_lookup(routing_table, src_node,
dst_node);
output_buffer = entry->output_buffer;
if (!output_buffer)
fatal("%s: no route from %s to %s.\n%s", net->name,
src_node->name, dst_node->name,
net_err_no_route);
if (pkt->msg->size > output_buffer->size)
panic("%s: message does not fit in buffer.\n%s",
__FUNCTION__, net_err_can_send);
if (output_buffer->count + pkt->size > output_buffer->size)
panic("%s: output buffer full.\n%s", __FUNCTION__,
net_err_can_send);
/* Insert in output buffer (1 cycle latency) */
net_buffer_insert(output_buffer, pkt);
output_buffer->write_busy = cycle;
pkt->node = src_node;
pkt->buffer = output_buffer;
pkt->busy = cycle;
/* Schedule next event */
esim_schedule_event(EV_NET_OUTPUT_BUFFER, stack, 1);
}
else if (event == EV_NET_OUTPUT_BUFFER)
{
struct net_buffer_t *input_buffer;
int lat;
/* Debug */
net_debug("msg "
"a=\"obuf\" "
"net=\"%s\" "
"msg-->pkt=%lld-->%d "
"node=\"%s\" "
"buf=\"%s\"\n",
net->name,
pkt->msg->id,
pkt->session_id,
node->name,
buffer->name);
/* If message is not at buffer head, process later */
assert(list_count(buffer->msg_list));
if (list_get(buffer->msg_list, 0) != pkt)
{
net_buffer_wait(buffer, event, stack);
net_debug("msg "
"a=\"stall\" "
"net=\"%s\" "
"msg-->packet=%lld:%d "
"why=\"not output buffer head\"\n",
net->name,
pkt->msg->id,
pkt->session_id);
return;
}
if (buffer->read_busy >= cycle)
{
esim_schedule_event(event, stack,
buffer->read_busy - cycle + 1);
net_debug("msg "
"a=\"stall\" "
"net=\"%s\" "
"msg-->pkt=%lld:%d "
"why=\"output buffer busy\" \n",
net->name,
pkt->msg->id,
pkt->session_id);
return;
}
/* If link is busy, wait */
if (buffer->kind == net_buffer_link)
{
struct net_link_t *link;
assert(buffer->link);
link = buffer->link;
if (link->busy >= cycle)
{
esim_schedule_event(event, stack,
link->busy - cycle + 1);
net_debug("msg "
"a=\"stall\" "
"net=\"%s\" "
"msg-->pkt=%lld:%d "
"why=\"link busy\"\n",
net->name,
pkt->msg->id,
pkt->session_id);
net_trace("net.packet "
"net=\"%s\" "
"name=\"P-%lld:%d\" "
"state=\"%s:%s:link_busy\" "
"stg=\"LB\"\n",
net->name, pkt->msg->id,
pkt->session_id,
node->name,
buffer->name);
return;
}
/* If buffer contain the message but doesn't have the
* shared link in control, wait */
if (link->virtual_channel > 1)
{
struct net_buffer_t *temp_buffer;
temp_buffer = net_link_arbitrator_vc(link, node);
if (temp_buffer != buffer)
{
net_debug("msg "
"a=\"stall\" "
"net=\"%s\" "
"msg-->pkt=%lld:%d "
"why=\"arbitrator sched\"\n",
net->name,
pkt->msg->id,
pkt->session_id);
esim_schedule_event(event, stack, 1);
net_trace("net.packet "
"net=\"%s\" "
"name=\"P-%lld:%d\" "
"state=\"%s:%s:VC_arbitration_fail\" "
"stg=\"VCA\"\n",
net->name, pkt->msg->id,
pkt->session_id,
node->name,
buffer->name);
return;
}
}
/* If destination input buffer is busy, wait */
assert(buffer == link->src_buffer);
input_buffer = link->dst_buffer;
if (input_buffer->write_busy >= cycle)
{
net_debug("msg "
"a=\"stall\" "
"net=\"%s\" "
"msg-->pkt=%lld:%d "
"why=\"input buffer busy\"\n",
net->name, pkt->msg->id,
pkt->session_id);
net_trace("net.packet "
"net=\"%s\" "
"name=\"P-%lld:%d\" "
"state=\"%s:%s:Dest_buffer_busy\" "
"stg=\"DBB\"\n",
net->name, pkt->msg->id,
pkt->session_id,
node->name,
buffer->name);
esim_schedule_event(event, stack,
input_buffer->write_busy - cycle + 1);
return;
}
/* If destination input buffer is full, wait */
if (pkt->size > input_buffer->size)
fatal("%s: packet does not fit in buffer.\n%s",
net->name, net_err_large_message);
if (input_buffer->count + pkt->size >
input_buffer->size)
{
net_debug("msg "
"a=\"stall\" "
"net=\"%s\" "
"msg-->pkt=%lld:%d "
"why=\"input buffer full\"\n",
net->name, pkt->msg->id, pkt->session_id);
net_trace("net.packet "
"net=\"%s\" "
"name=\"P-%lld:%d\" "
"state=\"%s:%s:Dest_buffer_full\" "
"stg=\"DBF\"\n",
net->name, pkt->msg->id,
pkt->session_id,
node->name,
buffer->name);
net_buffer_wait(input_buffer, event, stack);
return;
}
/* Calculate latency and occupy resources */
lat = (pkt->size - 1) / link->bandwidth + 1;
assert(lat > 0);
buffer->read_busy = cycle + lat - 1;
link->busy = cycle + lat - 1;
input_buffer->write_busy = cycle + lat - 1;
/* Transfer message to next input buffer */
assert(pkt->busy < cycle);
net_buffer_extract(buffer, pkt);
net_buffer_insert(input_buffer, pkt);
pkt->node = input_buffer->node;
pkt->buffer = input_buffer;
pkt->busy = cycle + lat - 1;
/* Stats */
link->busy_cycles += lat;
link->transferred_bytes += pkt->size;
link->transferred_msgs++;
net->topology_util_bw += pkt->size;
node->bytes_sent += pkt->size;
node->msgs_sent++;
input_buffer->node->bytes_received += pkt->size;
input_buffer->node->msgs_received++;
net_trace("net.link_transfer net=\"%s\" link=\"%s\" "
"transB=%lld last_size=%d busy=%lld\n",
net->name, link->name,
link->transferred_bytes,
pkt->size, link->busy);
}
else if (buffer->kind == net_buffer_bus)
{
struct net_bus_t *bus, *updated_bus;
struct net_node_t *bus_node;
assert(!buffer->link);
assert(buffer->bus);
bus = buffer->bus;
bus_node = bus->node;
/* before initiating bus transfer we have to figure out what is the
* next input buffer since it is not clear from the
* output buffer */
int input_buffer_detection = 0;
struct net_routing_table_entry_t *entry;
entry = net_routing_table_lookup(routing_table,
pkt->node, pkt->msg->dst_node);
for (int i = 0; i < list_count(bus_node->dst_buffer_list); i++)
{
input_buffer = list_get(bus_node->dst_buffer_list, i);
if (entry->next_node == input_buffer->node)
{
input_buffer_detection = 1;
break;
}
}
if (input_buffer_detection == 0)
fatal("%s: Something went wrong so there is no appropriate input"
"buffer for the route between %s and %s \n", net->name,
pkt->node->name,entry->next_node->name);
/* 1. Check the destination buffer is busy or not */
if (input_buffer->write_busy >= cycle)
{
esim_schedule_event(event, stack,
input_buffer->write_busy - cycle + 1);
net_debug("msg "
"a=\"stall\" "
"net=\"%s\" "
"msg-->pkt=%lld:%d "
"why=\"input busy\"\n",
net->name,
pkt->msg->id,
pkt->session_id);
net_trace("net.packet "
"net=\"%s\" "
"name=\"P-%lld:%d\" "
"state=\"%s:%s:Dest_buffer_busy\" "
"stg=\"DBB\"\n",
net->name, pkt->msg->id,
pkt->session_id,
node->name,
buffer->name);
return;
}
/* 2. Check the destination buffer is full or not */
if (pkt->size > input_buffer->size)
fatal("%s: packet does not fit in buffer.\n%s",
net->name, net_err_large_message);
if (input_buffer->count + pkt->size > input_buffer->size)
{
net_buffer_wait(input_buffer, event, stack);
net_debug("msg "
"a=\"stall\" "
"net=\"%s\" "
"msg-->pkt=%lld:%d "
"why=\"input full\"\n",
net->name, pkt->msg->id,
pkt->session_id);
net_trace("net.packet "
"net=\"%s\" "
"name=\"P-%lld:%d\" "
"state=\"%s:%s:Dest_buffer_full\" "
"stg=\"DBF\"\n",
net->name, pkt->msg->id,
pkt->session_id,
node->name,
buffer->name);
return;
}
/* 3. Make sure if any bus is available; return one
* that is available the fastest */
updated_bus = net_bus_arbitration(bus_node, buffer);
if (updated_bus == NULL)
{
esim_schedule_event(event, stack, 1);
net_debug("msg "
"a=\"stall\" "
"net=\"%s\" "
"msg-->pkt=%lld:%d "
"why=\"bus arbiter\"\n",
net->name, pkt->msg->id,
pkt->session_id);
net_trace("net.packet "
"net=\"%s\" "
"name=\"P-%lld:%d\" "
"state=\"%s:%s:BUS_arbit_fail\" "
"stg=\"BA\"\n",
net->name, pkt->msg->id,
pkt->session_id,
node->name,
buffer->name);
return;
}
/* 4. assign the bus to the buffer. update the
* necessary data ; before here, the bus is not
* assign to anything and is not updated so it can be
* assign to other buffers as well. If this certain
* buffer wins that specific bus_lane the appropriate
* fields will be updated. Contains: bus_lane
* cin_buffer and cout_buffer and busy time as well as
* buffer data itself */
assert(updated_bus);
buffer->bus = updated_bus;
input_buffer->bus = updated_bus;
bus = buffer->bus;
assert(bus);
/* Calculate latency and occupy resources */
/* Wire delay is introduced when the packet is on transit */
lat = bus->fix_delay + ((pkt->size - 1) / bus->bandwidth + 1) ;
assert(lat > 0);
buffer->read_busy = cycle + lat - 1;
bus->busy = cycle + lat - 1;
input_buffer->write_busy = cycle + lat - 1 ;
/* Transfer message to next input buffer */
assert(pkt->busy < cycle);
net_buffer_extract(buffer, pkt);
net_buffer_insert(input_buffer, pkt);
pkt->node = input_buffer->node;
pkt->buffer = input_buffer;
pkt->busy = cycle + lat - 1;
/* Stats */
bus->busy_cycles += lat;
bus->transferred_bytes += pkt->size;
bus->transferred_msgs++;
net->topology_util_bw += pkt->size;
node->bytes_sent += pkt->size;
node->msgs_sent++;
input_buffer->node->bytes_received += pkt->size;
input_buffer->node->msgs_received++;
net_trace("net.bus_transfer net=\"%s\" node=\"%s\" "
"lane_index=%d transB=%lld last_size=%d busy=%lld\n",
net->name, bus->node->name, bus->index,
bus->transferred_bytes, pkt->size, bus->busy);
}
else if (buffer->kind == net_buffer_photonic)
{
struct net_bus_t *bus, *updated_bus;
struct net_node_t *bus_node;
assert(!buffer->link);
assert(buffer->bus);
bus = buffer->bus;
bus_node = bus->node;
/* before 1 and 2 we have to figure out what is the
* next input buffer since it is not clear from the
* output buffer */
int input_buffer_detection = 0;
struct net_routing_table_entry_t *entry;
entry = net_routing_table_lookup(routing_table,
pkt->node, pkt->msg->dst_node);
for (int i = 0; i < list_count(bus_node->dst_buffer_list); i++)
{
input_buffer = list_get(bus_node->dst_buffer_list, i);
if (entry->next_node == input_buffer->node)
{
input_buffer_detection = 1;
break;
}
}
if (input_buffer_detection == 0)
fatal("%s: Something went wrong so there is no appropriate input"
"buffer for the route between %s and %s \n", net->name,
pkt->node->name,entry->next_node->name);
/* 1. Check the destination buffer is busy or not */
if (input_buffer->write_busy > cycle)
{
esim_schedule_event(event, stack,
input_buffer->write_busy - cycle + 1);
net_debug("msg "
"a=\"stall\" "
"net=\"%s\" "
"msg-->pkt=%lld:%d "
"why=\"input busy\"\n",
net->name,
pkt->msg->id,
pkt->session_id);
net_trace("net.packet "
"net=\"%s\" "
"name=\"P-%lld:%d\" "
"state=\"%s:%s:Dest_buffer_busy\" "
"stg=\"DBB\"\n",
net->name, pkt->msg->id,
pkt->session_id,
node->name,
buffer->name);
return;
}
/* 2. Check the destination buffer is full or not */
if (pkt->size > input_buffer->size)
fatal("%s: message does not fit in buffer.\n%s",
net->name, net_err_large_message);
if (input_buffer->count + pkt->size > input_buffer->size)
{
net_buffer_wait(input_buffer, event, stack);
net_debug("msg "
"a=\"stall\" "
"net=\"%s\" "
"msg-->pkt=%lld:%d "
"why=\"input full\"\n",
net->name, pkt->msg->id,
pkt->session_id);
net_trace("net.packet "
"net=\"%s\" "
"name=\"P-%lld:%d\" "
"state=\"%s:%s:Dest_buffer_full\" "
"stg=\"DBF\"\n",
net->name, pkt->msg->id,
pkt->session_id,
node->name,
buffer->name);
return;
}
/* 3. Make sure if any bus is available; return one
* that is available the fastest */
updated_bus = net_photo_link_arbitration(bus_node, buffer);
if (updated_bus == NULL)
{
esim_schedule_event(event, stack, 1);
net_debug("msg "
"a=\"stall\" "
"net=\"%s\" "
"msg-->pkt=%lld:%d "
"why=\"bus arbiter\"\n",
net->name, pkt->msg->id,
pkt->session_id);
net_trace("net.packet "
"net=\"%s\" "
"name=\"P-%lld:%d\" "
"state=\"%s:%s:photonic_arbitration\" "
"stg=\"BA\"\n",
net->name, pkt->msg->id,
pkt->session_id,
node->name,
buffer->name);
return;
}
/* 4. assign the bus to the buffer. update the
* necessary data ; before here, the bus is not
* assign to anything and is not updated so it can be
* assign to other buffers as well. If this certain
* buffer wins that specific bus_lane the appropriate
* fields will be updated. Contains: bus_lane
* cin_buffer and cout_buffer and busy time as well as
* buffer data itself */
assert(updated_bus);
buffer->bus = updated_bus;
input_buffer->bus = updated_bus;
bus = buffer->bus;
assert(bus);
/* Calculate latency and occupy resources */
lat = (pkt->size - 1) / bus->bandwidth + 1;
assert(lat > 0);
buffer->read_busy = cycle + lat - 1;
bus->busy = cycle + lat - 1;
input_buffer->write_busy = cycle + lat - 1;
/* Transfer message to next input buffer */
assert(pkt->busy < cycle);
net_buffer_extract(buffer, pkt);
net_buffer_insert(input_buffer, pkt);
pkt->node = input_buffer->node;
pkt->buffer = input_buffer;
pkt->busy = cycle + lat - 1;
/* Stats */
bus->busy_cycles += lat;
bus->transferred_bytes += pkt->size;
bus->transferred_msgs++;
net->topology_util_bw += pkt->size;
node->bytes_sent += pkt->size;
node->msgs_sent++;
input_buffer->node->bytes_received += pkt->size;
input_buffer->node->msgs_received++;
net_trace("net.photonic_transfer net=\"%s\" node=\"%s\" "
"lane_index=%d transB=%lld last_size=%d busy=%lld\n",
net->name, bus->node->name, bus->index,
bus->transferred_bytes,pkt->size, bus->busy);
net_debug("msg "
"a=\"success photonic transmission\" "
"net=\"%s\" "
"msg-->pkt=%lld:%d "
"through = \" %d\"\n",
net->name, pkt->msg->id,
pkt->session_id, updated_bus->index);
}
/* Schedule next event */
esim_schedule_event(EV_NET_INPUT_BUFFER, stack, lat);
}
else if (event == EV_NET_INPUT_BUFFER)
{
struct net_routing_table_entry_t *entry;
struct net_buffer_t *output_buffer;
int lat;
/* Debug */
net_debug("msg "
"a=\"ibuf\" "
"net=\"%s\" "
"msg-->pkt=%lld:%d "
"node=\"%s\" "
"buf=\"%s\"\n",
net->name,
pkt->msg->id,
pkt->session_id,
node->name,
buffer->name);
/* If message is not at buffer head, process later */
assert(list_count(buffer->msg_list));
if (list_get(buffer->msg_list, 0) != pkt)
{
net_debug("msg "
"a=\"stall\" "
"net=\"%s\" "
"msg-->pkt=%lld:%d"
"why=\"not-head\"\n",
net->name, pkt->msg->id,
pkt->session_id);
net_buffer_wait(buffer, event, stack);
return;
}
/* If this is the destination node, finish */
if (node == pkt->msg->dst_node)
{
esim_schedule_event(EV_NET_RECEIVE, stack, 0);
return;
}
/* If source input buffer is busy, wait */
if (buffer->read_busy >= cycle)
{
net_debug("pkt"
"a=\"stall\" "
"net=\"%s\" "
"msg-->pkt=%lld:%d "
"why=\"src-busy\"\n",
net->name,
pkt->msg->id,
pkt->session_id);
esim_schedule_event(event, stack,
buffer->read_busy - cycle + 1);
return;
}
/* Get output buffer */
entry = net_routing_table_lookup(routing_table, node,
dst_node);
output_buffer = entry->output_buffer;
if (!output_buffer)
fatal("%s: no route from %s to %s.\n%s", net->name,
node->name, dst_node->name, net_err_no_route);
/* If destination output buffer is busy, wait */
if (output_buffer->write_busy >= cycle)
{
net_debug("pkt "
"a=\"stall\" "
"net=\"%s\" "
"msg-->pkt=%lld:%d "
"why=\"dst-busy\"\n",
net->name,
pkt->msg->id,
pkt->session_id);
net_trace("net.packet "
"net=\"%s\" "
"name=\"P-%lld:%d\" "
"state=\"%s:%s:Dest_buffer_busy\" "
"stg=\"DBB\"\n",
net->name, pkt->msg->id,
pkt->session_id,
node->name,
buffer->name);
esim_schedule_event(event, stack,
output_buffer->write_busy - cycle + 1);
return;
}
/* If destination output buffer is full, wait */
if (pkt->size > output_buffer->size)
fatal("%s: packet does not fit in buffer.\n%s",
net->name, net_err_large_message);
if (output_buffer->count + pkt->size > output_buffer->size)
{
net_debug("pkt "
"a=\"stall\" "
"net=\"%s\" "
"msg-->pkt=%lld:%d "
"why=\"dst-full\"\n",
net->name,
pkt->msg->id,
pkt->session_id);
net_trace("net.packet "
"net=\"%s\" "
"name=\"P-%lld:%d\" "
"state=\"%s:%s:Dest_buffer_full\" "
"stg=\"DBF\"\n",
net->name, pkt->msg->id,
pkt->session_id,
node->name,
buffer->name);
net_buffer_wait(output_buffer, event, stack);
return;
}
/* If scheduler says that it is not our turn, try later */
if (net_node_schedule(node, output_buffer) != buffer)
{
net_debug("pkt "
"a=\"stall\" "
"net=\"%s\" "
"msg-->pkt=%lld:%d "
"why=\"sched\"\n",
net->name,
pkt->msg->id,
pkt->session_id);
net_trace("net.packet "
"net=\"%s\" "
"name=\"P-%lld:%d\" "
"state=\"%s:%s:switch_arbit_fail\" "
"stg=\"SA\"\n",
net->name, pkt->msg->id,
pkt->session_id,
node->name,
buffer->name);
esim_schedule_event(event, stack, 1);
return;
}
/* Calculate latency and occupy resources */
assert(node->kind != net_node_end);
assert(node->bandwidth > 0);
lat = (pkt->size - 1) / node->bandwidth + 1;
assert(lat > 0);
buffer->read_busy = cycle + lat - 1;
output_buffer->write_busy = cycle + lat - 1;
/* Transfer message to next output buffer */
assert(pkt->busy < cycle);
net_buffer_extract(buffer, pkt);
net_buffer_insert(output_buffer, pkt);
pkt->buffer = output_buffer;
pkt->busy = cycle + lat - 1;
/* Schedule next event */
esim_schedule_event(EV_NET_OUTPUT_BUFFER, stack, lat);
}
else if (event == EV_NET_RECEIVE)
{
assert (pkt);
struct net_msg_t *msg = pkt->msg;
/* Debug */
net_debug("pkt "
"a=\"receive\" "
"net=\"%s\" "
"msg-->pkt=%lld:%d "
"node=\"%s\"\n",
net->name,
pkt->msg->id,
pkt->session_id,
dst_node->name);
if (net_depacketizer(net, node, pkt) == 1)
{
if (pkt->msg->packet_list_count > 1)
net_trace("net.msg net=\"%s\" name=\"M-%lld\" "
"state=\"%s:depacketize\"\n",
net->name, msg->id, node->name);
if (stack->ret_event == ESIM_EV_NONE)
{
assert (msg);
net_receive(net, node, msg);
}
/* Finish */
net_stack_return(stack);
}
else
/* Freeing packet stack, not the message */
free(stack);
}
else
{
panic("%s: unknown event", __FUNCTION__);
}
}
/* Event handler for EV_MEM_SYSTEM_COMMAND.
* The event data is a string of type 'char *' that needs to be deallocated
* after processing this event. */
void mem_system_command_handler(int event, void *data)
{
struct list_t *token_list;
char *command_line = data;
char command[MAX_STRING_SIZE];
/* Get command */
str_token(command, sizeof command, command_line, 0, " ");
if (!command[0])
fatal("%s: invalid command syntax.\n\t> %s",
__FUNCTION__, command_line);
/* Commands that need to be processed at the end of the simulation
* are ignored here. These are command prefixed with 'CheckXXX'. */
if (!strncasecmp(command, "Check", 5))
{
esim_schedule_end_event(EV_MEM_SYSTEM_END_COMMAND, data);
return;
}
/* Split command in tokens, skip command */
token_list = str_token_list_create(command_line, " ");
assert(list_count(token_list));
str_token_list_shift(token_list);
/* Command 'SetBlock' */
if (!strcasecmp(command, "SetBlock"))
{
struct mod_t *mod;
int set;
int way;
int tag;
int set_check;
int tag_check;
int state;
mod = mem_system_command_get_mod(token_list, command_line);
mem_system_command_get_set_way(token_list, command_line, mod, &set, &way);
tag = mem_system_command_get_hex(token_list, command_line);
state = mem_system_command_get_state(token_list, command_line);
mem_system_command_end(token_list, command_line);
/* Check that module serves address */
if (!mod_serves_address(mod, tag))
fatal("%s: %s: module does not serve address 0x%x.\n\t> %s",
__FUNCTION__, mod->name, tag, command_line);
/* Check that tag goes to specified set */
mod_find_block(mod, tag, &set_check, NULL, &tag_check, NULL);
if (set != set_check)
fatal("%s: %s: tag 0x%x belongs to set %d.\n\t> %s",
__FUNCTION__, mod->name, tag, set_check, command_line);
if (tag != tag_check)
fatal("%s: %s: tag should be multiple of block size.\n\t> %s",
__FUNCTION__, mod->name, command_line);
/* Set tag */
cache_set_block(mod->cache, set, way, tag, state);
}
/* Command 'SetOwner' */
else if (!strcasecmp(command, "SetOwner"))
{
struct mod_t *mod;
struct mod_t *owner;
int set;
int way;
int sub_block;
int owner_index;
/* Get fields */
mod = mem_system_command_get_mod(token_list, command_line);
mem_system_command_get_set_way(token_list, command_line, mod, &set, &way);
sub_block = mem_system_command_get_sub_block(token_list, command_line, mod, set, way);
owner = mem_system_command_get_mod(token_list, command_line);
mem_system_command_end(token_list, command_line);
/* Check that owner is an immediate higher-level module */
if (owner)
{
if (owner->low_net != mod->high_net || !owner->low_net)
fatal("%s: %s is not a higher-level module of %s.\n\t> %s",
__FUNCTION__, owner->name, mod->name, command_line);
}
/* Set owner */
owner_index = owner ? owner->low_net_node->index : -1;
dir_entry_set_owner(mod->dir, set, way, sub_block, owner_index);
}
/* Command 'SetSharers' */
else if (!strcasecmp(command, "SetSharers"))
{
struct mod_t *mod;
struct mod_t *sharer;
int set;
int way;
int sub_block;
/* Get first fields */
mod = mem_system_command_get_mod(token_list, command_line);
mem_system_command_get_set_way(token_list, command_line, mod, &set, &way);
sub_block = mem_system_command_get_sub_block(token_list, command_line, mod, set, way);
/* Get sharers */
mem_system_command_expect(token_list, command_line);
dir_entry_clear_all_sharers(mod->dir, set, way, sub_block);
while (list_count(token_list))
{
/* Get sharer */
sharer = mem_system_command_get_mod(token_list, command_line);
if (!sharer)
continue;
/* Check that sharer is an immediate higher-level module */
if (sharer->low_net != mod->high_net || !sharer->low_net)
fatal("%s: %s is not a higher-level module of %s.\n\t> %s",
__FUNCTION__, sharer->name, mod->name, command_line);
/* Set sharer */
dir_entry_set_sharer(mod->dir, set, way, sub_block, sharer->low_net_node->index);
}
}
/* Command 'Access' */
else if (!strcasecmp(command, "Access"))
{
struct mod_t *mod;
enum mod_access_kind_t access_kind;
unsigned int addr;
long long cycle;
/* Read fields */
mod = mem_system_command_get_mod(token_list, command_line);
cycle = mem_system_command_get_cycle(token_list, command_line);
access_kind = mem_system_command_get_mod_access(token_list, command_line);
addr = mem_system_command_get_hex(token_list, command_line);
/* If command is scheduled for later, exit */
if (cycle > esim_cycle)
{
str_token_list_free(token_list);
esim_schedule_event(EV_MEM_SYSTEM_COMMAND, data, cycle - esim_cycle);
return;
}
/* Access module */
mod_access(mod, access_kind, addr, NULL, NULL, NULL, NULL);
}
/* Command not supported */
else
fatal("%s: %s: invalid command.\n\t> %s",
__FUNCTION__, command, command_line);
/* Free command */
free(command_line);
str_token_list_free(token_list);
}
void moesi_handler_write_request(int event, void *data)
{
struct moesi_stack_t *stack = data, *ret = stack->retstack, *newstack;
struct ccache_t *ccache = stack->ccache, *target = stack->target;
struct dir_t *dir;
struct dir_entry_t *dir_entry;
uint32_t dir_entry_tag, z;
if (event == EV_MOESI_WRITE_REQUEST)
{
struct net_t *net;
int src, dest;
cache_debug(" %lld %lld 0x%x %s write request\n", CYCLE, ID,
stack->addr, ccache->name);
/* Default return values */
ret->err = 0;
/* Send request to target */
assert(ccache->next == target || target->next == ccache);
net = ccache->next == target ? ccache->lonet : ccache->hinet;
src = ccache->next == target ? ccache->loid : 0;
dest = ccache->next == target ? 0 : target->loid;
net_send_ev(net, src, dest, 8, EV_MOESI_WRITE_REQUEST_RECEIVE, stack);
return;
}
if (event == EV_MOESI_WRITE_REQUEST_RECEIVE)
{
cache_debug(" %lld %lld 0x%x %s write request receive\n", CYCLE, ID,
stack->addr, target->name);
/* Find and lock */
newstack = moesi_stack_create(stack->id, target, stack->addr,
EV_MOESI_WRITE_REQUEST_ACTION, stack);
newstack->blocking = target->next == ccache;
newstack->read = 0;
newstack->retry = 0;
esim_schedule_event(EV_MOESI_FIND_AND_LOCK, newstack, 0);
return;
}
if (event == EV_MOESI_WRITE_REQUEST_ACTION)
{
cache_debug(" %lld %lld 0x%x %s write request action\n", CYCLE, ID,
stack->tag, target->name);
/* Check lock error. If write request is down-up, there should
* have been no error. */
if (stack->err) {
assert(ccache->next == target);
ret->err = 1;
stack->response = 8;
esim_schedule_event(EV_MOESI_WRITE_REQUEST_REPLY, stack, 0);
return;
}
/* Invalidate the rest of upper level sharers */
newstack = moesi_stack_create(stack->id, target, 0,
EV_MOESI_WRITE_REQUEST_EXCLUSIVE, stack);
newstack->except = ccache;
newstack->set = stack->set;
newstack->way = stack->way;
esim_schedule_event(EV_MOESI_INVALIDATE, newstack, 0);
return;
}
if (event == EV_MOESI_WRITE_REQUEST_EXCLUSIVE)
{
cache_debug(" %lld %lld 0x%x %s write request exclusive\n", CYCLE, ID,
stack->tag, target->name);
if (ccache->next == target)
esim_schedule_event(EV_MOESI_WRITE_REQUEST_UPDOWN, stack, 0);
else
esim_schedule_event(EV_MOESI_WRITE_REQUEST_DOWNUP, stack, 0);
return;
}
if (event == EV_MOESI_WRITE_REQUEST_UPDOWN)
{
cache_debug(" %lld %lld 0x%x %s write request updown\n", CYCLE, ID,
stack->tag, target->name);
/* status = M/E */
if (stack->status == moesi_status_modified ||
stack->status == moesi_status_exclusive) {
esim_schedule_event(EV_MOESI_WRITE_REQUEST_UPDOWN_FINISH, stack, 0);
return;
}
/* status = O/S/I */
newstack = moesi_stack_create(stack->id, target, stack->tag,
EV_MOESI_WRITE_REQUEST_UPDOWN_FINISH, stack);
newstack->target = target->next;
esim_schedule_event(EV_MOESI_WRITE_REQUEST, newstack, 0);
return;
}
if (event == EV_MOESI_WRITE_REQUEST_UPDOWN_FINISH)
{
cache_debug(" %lld %lld 0x%x %s write request updown finish\n", CYCLE, ID,
stack->tag, target->name);
/* Error in write request to next cache level */
if (stack->err) {
ret->err = 1;
stack->response = 8;
dir_lock_unlock(stack->dir_lock);
esim_schedule_event(EV_MOESI_WRITE_REQUEST_REPLY, stack, 0);
return;
}
/* Check that addr is a multiple of ccache.bsize.
* Set ccache as sharer and owner. */
dir = ccache_get_dir(target, stack->tag);
for (z = 0; z < dir->zsize; z++) {
assert(stack->addr % ccache->bsize == 0);
dir_entry_tag = stack->tag + z * cache_min_block_size;
if (dir_entry_tag < stack->addr || dir_entry_tag >= stack->addr + ccache->bsize)
continue;
dir_entry = ccache_get_dir_entry(target, stack->set, stack->way, z);
dir_entry_set_sharer(dir, dir_entry, ccache->loid);
dir_entry->owner = ccache->loid;
assert(dir_entry->sharers == 1);
}
/* Update LRU, set status: M->M, O/E/S/I->E */
if (target->cache) {
cache_access_block(target->cache, stack->set, stack->way);
if (stack->status != moesi_status_modified)
cache_set_block(target->cache, stack->set, stack->way,
stack->tag, moesi_status_exclusive);
}
/* Unlock, response is the data of the size of the requester's block. */
dir_lock_unlock(stack->dir_lock);
stack->response = ccache->bsize + 8;
esim_schedule_event(EV_MOESI_WRITE_REQUEST_REPLY, stack, 0);
return;
}
if (event == EV_MOESI_WRITE_REQUEST_DOWNUP)
{
cache_debug(" %lld %lld 0x%x %s write request downup\n", CYCLE, ID,
stack->tag, target->name);
/* Compute response, set status to I, unlock */
assert(stack->status != moesi_status_invalid);
assert(!dir_entry_group_shared_or_owned(target->dir, stack->set, stack->way));
stack->response = stack->status == moesi_status_modified || stack->status
== moesi_status_owned ? target->bsize + 8 : 8;
cache_set_block(target->cache, stack->set, stack->way, 0, moesi_status_invalid);
dir_lock_unlock(stack->dir_lock);
esim_schedule_event(EV_MOESI_WRITE_REQUEST_REPLY, stack, 0);
return;
}
if (event == EV_MOESI_WRITE_REQUEST_REPLY)
{
struct net_t *net;
int src, dest;
cache_debug(" %lld %lld 0x%x %s write request reply\n", CYCLE, ID,
stack->tag, target->name);
assert(stack->response);
assert(ccache->next == target || target->next == ccache);
net = ccache->next == target ? ccache->lonet : ccache->hinet;
src = ccache->next == target ? 0 : target->loid;
dest = ccache->next == target ? ccache->loid : 0;
net_send_ev(net, src, dest, stack->response,
EV_MOESI_WRITE_REQUEST_FINISH, stack);
return;
}
if (event == EV_MOESI_WRITE_REQUEST_FINISH)
{
cache_debug(" %lld %lld 0x%x %s write request finish\n", CYCLE, ID,
stack->tag, ccache->name);
moesi_stack_return(stack);
return;
}
abort();
}
