void Topology::printConfig(ostream& out) const
{
assert(m_component_latencies.size() > 0);
out << "--- Begin Topology Print ---" << endl;
out << endl;
out << "Topology print ONLY indicates the _NETWORK_ latency between two machines" << endl;
out << "It does NOT include the latency within the machines" << endl;
out << endl;
for (int m=0; m<MachineType_NUM; m++) {
for (int i=0; i<MachineType_base_count((MachineType)m); i++) {
MachineID cur_mach = {(MachineType)m, i};
out << cur_mach << " Network Latencies" << endl;
for (int n=0; n<MachineType_NUM; n++) {
for (int j=0; j<MachineType_base_count((MachineType)n); j++) {
MachineID dest_mach = {(MachineType)n, j};
if (cur_mach != dest_mach) {
int link_latency = m_component_latencies[MachineType_base_number((MachineType)m)+i][MachineType_base_number(MachineType_NUM)+MachineType_base_number((MachineType)n)+j];
int intermediate_switches = m_component_inter_switches[MachineType_base_number((MachineType)m)+i][MachineType_base_number(MachineType_NUM)+MachineType_base_number((MachineType)n)+j];
out << " " << cur_mach << " -> " << dest_mach << " net_lat: "
<< link_latency+intermediate_switches << endl; // NOTE switches are assumed to have single cycle latency
}
}
}
out << endl;
}
}
out << "--- End Topology Print ---" << endl;
}
bool NetworkInterface::flitisizeMessage(MsgPtr msg_ptr, int vnet)
{
NetworkMessage *net_msg_ptr = dynamic_cast<NetworkMessage*>(msg_ptr.ref());
NetDest net_msg_dest = net_msg_ptr->getInternalDestination();
Vector<NodeID> dest_nodes = net_msg_dest.getAllDest(); // gets all the destinations associated with this message.
int num_flits = (int) ceil((double) MessageSizeType_to_int(net_msg_ptr->getMessageSize())/NetworkConfig::getFlitSize() ); // Number of flits is dependent on the link bandwidth available. This is expressed in terms of bytes/cycle or the flit size
for(int ctr = 0; ctr < dest_nodes.size(); ctr++) // loop because we will be converting all multicast messages into unicast messages
{
int vc = calculateVC(vnet); // this will return a free output virtual channel
if(vc == -1)
{
// did not find a free output vc
return false ;
}
MsgPtr new_msg_ptr = *(msg_ptr.ref());
NodeID destID = dest_nodes[ctr];
NetworkMessage *new_net_msg_ptr = dynamic_cast<NetworkMessage*>(new_msg_ptr.ref());
if(dest_nodes.size() > 1)
{
NetDest personal_dest;
for(int m = 0; m < (int) MachineType_NUM; m++)
{
if((destID >= MachineType_base_number((MachineType) m)) && destID < MachineType_base_number((MachineType) (m+1)))
{
// calculating the NetDest associated with this destination ID
personal_dest.clear();
personal_dest.add((MachineID) {(MachineType) m, (destID - MachineType_base_number((MachineType) m))});
new_net_msg_ptr->getInternalDestination() = personal_dest;
break;
}
}
net_msg_dest.removeNetDest(personal_dest);
net_msg_ptr->getInternalDestination().removeNetDest(personal_dest); // removing the destination from the original message to reflect that a message with this particular destination has been flitisized and an output vc is acquired
}
for(int i = 0; i < num_flits; i++)
{
flit *fl = new flit(i, vc, vnet, num_flits, new_msg_ptr);
m_ni_buffers[vc]->insert(fl);
}
m_out_vc_state[vc]->setState(VC_AB_, g_eventQueue_ptr->getTime());
outNetLink->request_vc_link(vc, new_net_msg_ptr->getInternalDestination(), g_eventQueue_ptr->getTime()); // setting an output vc request for the next hop. It is only when an output vc is acquired at the next hop that this flit will be ready to traverse the link and into the next hop
}
return true ;
}
// make a network as described by the networkFile
void Topology::makeFileSpecified(TopoType topoType)
{
Vector< Vector < SwitchID > > nodePairs; // node pairs extracted from the file
Vector<int> latencies; // link latencies for each link extracted
Vector<int> bw_multis; // bw multipliers for each link extracted
Vector<int> weights; // link weights used to enfore e-cube deadlock free routing
Vector< SwitchID > int_network_switches; // internal switches extracted from the file
Vector<bool> endpointConnectionExist; // used to ensure all endpoints are connected to the network
endpointConnectionExist.setSize(m_nodes);
// initialize endpoint check vector
for (int k = 0; k < endpointConnectionExist.size(); k++) {
endpointConnectionExist[k] = false;
}
string filename = "";
if (g_SIMICS) {
filename = "../ruby/";
}
else
filename = "/curr/ghodrat/cdsc-simulation-platform/trunk/gems-2.1/ruby/";
filename = filename + "network/simple/Network_Files/";
if(topoType == TopoType_Default)
{
if(RubyConfig::networkConfigFile() && strcmp(RubyConfig::networkConfigFile(), "DEFAULT") != 0)
filename = RubyConfig::RubyConfig::networkConfigFile();
else
filename = filename+g_CACHE_DESIGN
+"_Procs-"+int_to_string(RubyConfig::numberOfProcessors())
+"_ProcsPerChip-"+int_to_string(RubyConfig::numberOfProcsPerChip())
+"_L2Banks-"+int_to_string(RubyConfig::numberOfL2Cache())
+"_Memories-"+int_to_string(RubyConfig::numberOfMemories())
#ifdef SIM_NET_PORTS
; if (RubyConfig::numberOfSimicsNetworkPort() > 0) filename = filename
+"_SimPorts-"+int_to_string(RubyConfig::numberOfSimicsNetworkPort())
; filename = filename
#endif
+".txt";
}
else if (topoType == TopoType_Dir2Buf)
{
filename = filename+g_CACHE_DESIGN
+"_Procs-"+int_to_string(RubyConfig::numberOfProcessors())
+"_ProcsPerChip-"+int_to_string(RubyConfig::numberOfProcsPerChip())
+"_L2Banks-"+int_to_string(RubyConfig::numberOfL2Cache())
+"_Memories-"+int_to_string(RubyConfig::numberOfMemories())
#ifdef SIM_NET_PORTS
; if (RubyConfig::numberOfSimicsNetworkPort() > 0) filename = filename
+"_SimPorts-"+int_to_string(RubyConfig::numberOfSimicsNetworkPort())
; filename = filename
#endif
+"_Dir2Buf"
+".txt";
}
else
{
}
cout << "Opening network file: " << filename << std::endl;
ifstream networkFile( filename.c_str() , ios::in);
if (!networkFile.is_open()) {
cerr << "Error: Could not open network file: " << filename << endl;
cerr << "Probably no network file exists for " << RubyConfig::numberOfProcessors()
<< " processors and " << RubyConfig::numberOfProcsPerChip() << " procs per chip " << endl;
exit(1);
}
string line = "";
while (!networkFile.eof()) {
Vector < SwitchID > nodes;
nodes.setSize(2);
int latency = -1; // null latency
int weight = -1; // null weight
int bw_multiplier = DEFAULT_BW_MULTIPLIER; // default multiplier incase the network file doesn't define it
int i = 0; // node pair index
int varsFound = 0; // number of varsFound on the line
int internalNodes = 0; // used to determine if the link is between 2 internal nodes
std::getline(networkFile, line, '\n');
string varStr = string_split(line, ' ');
// parse the current line in the file
while (varStr != "") {
string label = string_split(varStr, ':');
// valid node labels
if (label == "ext_node" || label == "int_node") {
ASSERT(i < 2); // one link between 2 switches per line
varsFound++;
bool isNewIntSwitch = true;
if (label == "ext_node") { // input link to node
MachineType machine = string_to_MachineType(string_split(varStr, ':'));
string nodeStr = string_split(varStr, ':');
if (string_split(varStr, ':') == "bank") {
nodes[i] = MachineType_base_number(machine)
+ atoi(nodeStr.c_str())
+ atoi((string_split(varStr, ':')).c_str())*RubyConfig::numberOfChips();
} else {
nodes[i] = MachineType_base_number(machine)
+ atoi(nodeStr.c_str());
}
// in nodes should be numbered 0 to m_nodes-1
ASSERT(nodes[i] >= 0 && nodes[i] < m_nodes);
isNewIntSwitch = false;
endpointConnectionExist[nodes[i]] = true;
}
if (label == "int_node") { // interior node
nodes[i] = atoi((string_split(varStr, ':')).c_str())+m_nodes*2;
// in nodes should be numbered >= m_nodes*2
ASSERT(nodes[i] >= m_nodes*2);
for (int k = 0; k < int_network_switches.size(); k++) {
if (int_network_switches[k] == nodes[i]) {
isNewIntSwitch = false;
}
}
if (isNewIntSwitch) { // if internal switch
m_number_of_switches++;
int_network_switches.insertAtBottom(nodes[i]);
}
internalNodes++;
}
i++;
} else if (label == "link_latency") {
latency = atoi((string_split(varStr, ':')).c_str());
varsFound++;
} else if (label == "bw_multiplier") { // not necessary, defaults to DEFAULT_BW_MULTIPLIER
bw_multiplier = atoi((string_split(varStr, ':')).c_str());
} else if (label == "link_weight") { // not necessary, defaults to link_latency
weight = atoi((string_split(varStr, ':')).c_str());
} else if (label == "processors") {
ASSERT(atoi((string_split(varStr, ':')).c_str()) == RubyConfig::numberOfProcessors());
} else if (label == "bw_unit") {
ASSERT(atoi((string_split(varStr, ':')).c_str()) == g_endpoint_bandwidth);
} else if (label == "procs_per_chip") {
ASSERT(atoi((string_split(varStr, ':')).c_str()) == RubyConfig::numberOfProcsPerChip());
} else if (label == "L2banks") {
ASSERT(atoi((string_split(varStr, ':')).c_str()) == RubyConfig::numberOfL2Cache());
} else if (label == "memories") {
ASSERT(atoi((string_split(varStr, ':')).c_str()) == RubyConfig::numberOfMemories());
#ifdef SIM_NET_PORTS
} else if (label == "simicsPorts") {
ASSERT(atoi((string_split(varStr, ':')).c_str()) == RubyConfig::numberOfSimicsNetworkPort());
#endif
} else {
cerr << "Error: Unexpected Identifier: " << label << endl;
exit(1);
}
varStr = string_split(line, ' ');
}
if (varsFound == 3) { // all three necessary link variables where found so add the link
nodePairs.insertAtBottom(nodes);
latencies.insertAtBottom(latency);
if (weight != -1) {
weights.insertAtBottom(weight);
} else {
weights.insertAtBottom(latency);
}
bw_multis.insertAtBottom(bw_multiplier);
Vector < SwitchID > otherDirectionNodes;
otherDirectionNodes.setSize(2);
otherDirectionNodes[0] = nodes[1];
if (internalNodes == 2) { // this is an internal link
otherDirectionNodes[1] = nodes[0];
} else {
otherDirectionNodes[1] = nodes[0]+m_nodes;
}
nodePairs.insertAtBottom(otherDirectionNodes);
latencies.insertAtBottom(latency);
if (weight != -1) {
weights.insertAtBottom(weight);
} else {
weights.insertAtBottom(latency);
}
bw_multis.insertAtBottom(bw_multiplier);
} else {
if (varsFound != 0) { // if this is not a valid link, then no vars should have been found
cerr << "Error in line: " << line << endl;
exit(1);
}
}
} // end of file
// makes sure all enpoints are connected in the soon to be created network
for (int k = 0; k < endpointConnectionExist.size(); k++) {
if (endpointConnectionExist[k] == false) {
cerr << "Error: Unconnected Endpoint: " << k << endl;
exit(1);
}
}
//cerr << "Num nodes "<<m_nodes<<"\n";
//cerr << "Node pairs "<<nodePairs.size()<<"\n";
ASSERT(nodePairs.size() == latencies.size() && latencies.size() == bw_multis.size() && latencies.size() == weights.size())
for (int k = 0; k < nodePairs.size(); k++) {
ASSERT(nodePairs[k].size() == 2);
addLink(nodePairs[k][0], nodePairs[k][1], latencies[k], bw_multis[k], weights[k]);
}
networkFile.close();
}
void Topology::makeSwitchesPerChip(Vector< Vector < SwitchID > > &nodePairs, Vector<int> &latencies, Vector<int> &bw_multis, int numberOfChipSwitches)
{
Vector < SwitchID > nodes; // temporary buffer
nodes.setSize(2);
Vector<bool> endpointConnectionExist; // used to ensure all endpoints are connected to the network
endpointConnectionExist.setSize(m_nodes);
// initialize endpoint check vector
for (int k = 0; k < endpointConnectionExist.size(); k++) {
endpointConnectionExist[k] = false;
}
Vector<int> componentCount;
componentCount.setSize(MachineType_NUM);
for (MachineType mType = MachineType_FIRST; mType < MachineType_NUM; ++mType) {
componentCount[mType] = 0;
}
// components to/from network links
for (int chip = 0; chip < RubyConfig::numberOfChips(); chip++) {
for (MachineType mType = MachineType_FIRST; mType < MachineType_NUM; ++mType) {
for (int component = 0; component < MachineType_chip_count(mType, chip); component++) {
int latency = -1;
int bw_multiplier = -1; // internal link bw multiplier of the global bandwidth
if (mType != MachineType_Directory) {
latency = ON_CHIP_LINK_LATENCY; // internal link latency
bw_multiplier = 10; // internal link bw multiplier of the global bandwidth
} else {
latency = NETWORK_LINK_LATENCY; // local memory latency
bw_multiplier = 1; // local memory link bw multiplier of the global bandwidth
}
nodes[0] = MachineType_base_number(mType)+componentCount[mType];
nodes[1] = chip+m_nodes*2; // this is the chip's internal switch id #
// insert link
nodePairs.insertAtBottom(nodes);
latencies.insertAtBottom(latency);
//bw_multis.insertAtBottom(bw_multiplier);
bw_multis.insertAtBottom(componentCount[mType]+MachineType_base_number((MachineType)mType));
// opposite direction link
Vector < SwitchID > otherDirectionNodes;
otherDirectionNodes.setSize(2);
otherDirectionNodes[0] = nodes[1];
otherDirectionNodes[1] = nodes[0]+m_nodes;
nodePairs.insertAtBottom(otherDirectionNodes);
latencies.insertAtBottom(latency);
bw_multis.insertAtBottom(bw_multiplier);
assert(!endpointConnectionExist[nodes[0]]);
endpointConnectionExist[nodes[0]] = true;
componentCount[mType]++;
}
}
}
// make sure all enpoints are connected in the soon to be created network
for (int k = 0; k < endpointConnectionExist.size(); k++) {
if (endpointConnectionExist[k] == false) {
cerr << "Error: Unconnected Endpoint: " << k << endl;
exit(1);
}
}
// secondary check to ensure we saw the correct machine counts
for (MachineType mType = MachineType_FIRST; mType < MachineType_NUM; ++mType) {
assert(componentCount[mType] == MachineType_base_count((MachineType)mType));
}
}
