void Topology::init()
{
assert(m_nodes >= MachineType_NUM);
if (m_nodes == MachineType_NUM) {
SwitchID id = newSwitchID();
for (int machine=0; machine<MachineType_NUM; machine++) {
addLink(machine, id, g_param_ptr->NETWORK_LINK_LATENCY());
addLink(id, m_nodes+machine, g_param_ptr->NETWORK_LINK_LATENCY());
}
return;
}
Vector<SwitchID> network_in_switches;
Vector<SwitchID> network_out_switches;
for (int node = 0; node < m_nodes; node++) {
network_in_switches.insertAtBottom(node); // numbered [0...m_nodes-1]
network_out_switches.insertAtBottom(node+m_nodes); // numbered [m_nodes-1...m_nodes+m_nodes-1]
}
// topology-specific set-up
switch (g_param_ptr->NETWORK_TOPOLOGY()) {
case TopologyType_TORUS_2D:
make2DTorus(network_in_switches, network_out_switches);
break;
case TopologyType_HIERARCHICAL_SWITCH:
makeHierarchicalSwitch(network_in_switches, network_out_switches, g_param_ptr->FAN_OUT_DEGREE());
break;
case TopologyType_CROSSBAR:
makeHierarchicalSwitch(network_in_switches, network_out_switches, 1024);
break;
default:
ERROR_MSG("Unexpected typology type")
}
}
void Topology::make2DTorus(Vector<SwitchID> network_in_switches, Vector<SwitchID> network_out_switches)
{
int lengthOfSide = (int)sqrt((double)m_nodes);
Vector<SwitchID> torusSwitches;
for(int i=0; i<m_nodes; i++){
SwitchID new_switch = newSwitchID();
torusSwitches.insertAtBottom(new_switch);
}
for(int i=0; i<m_nodes; i++){
SwitchID new_switch = torusSwitches[i];
// add links between switch and node
SwitchID port_into_network = network_in_switches[i];
SwitchID port_from_network = network_out_switches[i];
addLink(port_into_network, new_switch, g_param_ptr->NETWORK_LINK_LATENCY());
// Latency of 1 because we're not crossing a chip boundary (on-chip switch to processor)
// Bandwidth multiplier of 10 so this on-chip link to the processor isn't the bandwidth bottleneck
addLink(new_switch, port_from_network, 1, 10);
// left
SwitchID leftNeighbor;
if(new_switch%lengthOfSide == 0){
leftNeighbor = new_switch - 1 + lengthOfSide;
} else {
leftNeighbor = new_switch - 1;
}
addLink(new_switch, leftNeighbor, g_param_ptr->NETWORK_LINK_LATENCY());
// right
SwitchID rightNeighbor;
if((new_switch + 1)%lengthOfSide == 0){
rightNeighbor = new_switch + 1 - lengthOfSide;
} else {
rightNeighbor = new_switch + 1;
}
addLink(new_switch, rightNeighbor, g_param_ptr->NETWORK_LINK_LATENCY());
// top
SwitchID topNeighbor;
if(new_switch - lengthOfSide < 2*m_nodes){
topNeighbor = new_switch - lengthOfSide + (lengthOfSide*lengthOfSide);
} else {
topNeighbor = new_switch - lengthOfSide;
}
addLink(new_switch, topNeighbor, g_param_ptr->NETWORK_LINK_LATENCY());
// bottom
SwitchID bottomNeighbor;
if(new_switch + lengthOfSide >= 3*m_nodes){ // sorin: bad bug if this is a > instead of a >=
bottomNeighbor = new_switch + lengthOfSide - (lengthOfSide*lengthOfSide);
} else {
bottomNeighbor = new_switch + lengthOfSide;
}
addLink(new_switch, bottomNeighbor, g_param_ptr->NETWORK_LINK_LATENCY());
}
}
void Topology::init()
#endif
{
if (m_nodes == 1) {
SwitchID id = newSwitchID();
addLink(0, id, NETWORK_LINK_LATENCY);
addLink(id, 1, NETWORK_LINK_LATENCY);
return;
}
// topology-specific set-up
TopologyType topology = string_to_TopologyType(g_NETWORK_TOPOLOGY);
switch (topology) {
case TopologyType_TORUS_2D:
make2DTorus();
break;
case TopologyType_HIERARCHICAL_SWITCH:
makeHierarchicalSwitch(FAN_OUT_DEGREE);
break;
case TopologyType_CROSSBAR:
makeHierarchicalSwitch(1024);
break;
case TopologyType_PT_TO_PT:
makePtToPt();
break;
case TopologyType_FILE_SPECIFIED:
makeFileSpecified(topoType);
break;
default:
ERROR_MSG("Unexpected typology type")
}
cout << "setupNet value = " << setupNet << endl;
#ifdef SIM_MEMORY
#ifdef CS_NOC
if(!setupNet)
{
cout << "SimicsMemoryInterface::SetMemoryPortCount " << RubyConfig::numberOfMemories() << endl;
SimicsMemoryInterface::SetMemoryPortCount(RubyConfig::numberOfMemories());
}
#else
SimicsMemoryInterface::SetMemoryPortCount(RubyConfig::numberOfMemories());
#endif
#endif
// initialize component latencies record
m_component_latencies.setSize(0);
m_component_inter_switches.setSize(0);
}
void Topology::makeHierarchicalSwitch(Vector<SwitchID> network_in_switches, Vector<SwitchID> network_out_switches, int fan_out_degree)
{
// Make a row of switches with only one input. This extra row makes
// sure the links out of the nodes have latency and limited
// bandwidth.
Vector<SwitchID> last_level;
int counter = 0;
for (int i=0; i<(network_in_switches.size()/MachineType_NUM); i++) {
SwitchID new_switch = newSwitchID();
last_level.insertAtBottom(new_switch);
// Add a link per machine
for (int machine=0; machine<MachineType_NUM; machine++) {
SwitchID port_into_network = network_in_switches[counter];
counter++;
addLink(port_into_network, new_switch, g_param_ptr->NETWORK_LINK_LATENCY());
}
}
Vector<SwitchID> next_level;
while(last_level.size() > 1) {
for (int i=0; i<last_level.size(); i++) {
if ((i % fan_out_degree) == 0) {
next_level.insertAtBottom(newSwitchID());
}
// Add this link to the last switch we created
addLink(last_level[i], next_level[next_level.size()-1], g_param_ptr->NETWORK_LINK_LATENCY());
}
// Make the current level the last level to get ready for next
// iteration
last_level = next_level;
next_level.clear();
}
// Make the root switch
SwitchID root_switch = last_level[0];
// Build the down network from the endpoints to the root
// Make a row of switches for endpoints
last_level.clear();
counter = 0;
for (int j=0; j<(network_out_switches.size()/MachineType_NUM); j++) {
SwitchID new_switch = newSwitchID();
last_level.insertAtBottom(new_switch);
// Add a link per machine
for (int machine=0; machine<MachineType_NUM; machine++) {
SwitchID port_from_network = network_out_switches[counter];
counter++;
addLink(new_switch, port_from_network, 1, 10);
}
}
// Build the down network from the endpoints to the root
next_level.clear();
while(last_level.size() != 1) {
// A level of switches
NodeID node_counter = 0;
for (int i=0; i<last_level.size(); i++) {
if ((i % fan_out_degree) == 0) {
if (last_level.size() > fan_out_degree) {
next_level.insertAtBottom(newSwitchID());
} else {
next_level.insertAtBottom(root_switch);
}
}
// Add this link to the last switch we created
addLink(next_level[next_level.size()-1], last_level[i], g_param_ptr->NETWORK_LINK_LATENCY());
}
// Make the current level the last level to get ready for next
// iteration
last_level = next_level;
next_level.clear();
}
}
// one internal node per chip, point to point links between chips
void Topology::makePtToPt()
{
Vector< Vector < SwitchID > > nodePairs; // node pairs extracted from the file
Vector<int> latencies; // link latencies for each link extracted
Vector<int> bw_multis; // bw multipliers for each link extracted
Vector < SwitchID > nodes;
nodes.setSize(2);
// number of inter-chip switches
int numberOfChipSwitches = m_nodes/MachineType_base_level(MachineType_NUM);
// two switches per machine node grouping
// one intra-chip switch and one inter-chip switch per chip
for(int i=0; i<numberOfChipSwitches; i++){
SwitchID new_switch = newSwitchID();
new_switch = newSwitchID();
}
makeSwitchesPerChip(nodePairs, latencies, bw_multis, numberOfChipSwitches);
// connect intra-chip switch to inter-chip switch
for (int chip = 0; chip < RubyConfig::numberOfChips(); chip++) {
int latency = ON_CHIP_LINK_LATENCY; // internal link latency
int bw_multiplier = 10; // external link bw multiplier of the global bandwidth
nodes[0] = chip+m_nodes*2;
nodes[1] = chip+m_nodes*2+RubyConfig::numberOfChips();
// insert link
nodePairs.insertAtBottom(nodes);
latencies.insertAtBottom(latency);
bw_multis.insertAtBottom(bw_multiplier);
// opposite direction link
Vector < SwitchID > otherDirectionNodes;
otherDirectionNodes.setSize(2);
otherDirectionNodes[0] = nodes[1];
otherDirectionNodes[1] = nodes[0];
nodePairs.insertAtBottom(otherDirectionNodes);
latencies.insertAtBottom(latency);
bw_multis.insertAtBottom(bw_multiplier);
}
// point-to-point network between chips
for (int chip = 0; chip < RubyConfig::numberOfChips(); chip++) {
for (int other_chip = chip+1; other_chip < RubyConfig::numberOfChips(); other_chip++) {
int latency = NETWORK_LINK_LATENCY; // external link latency
int bw_multiplier = 1; // external link bw multiplier of the global bandwidth
nodes[0] = chip+m_nodes*2+RubyConfig::numberOfChips();
nodes[1] = other_chip+m_nodes*2+RubyConfig::numberOfChips();
// insert link
nodePairs.insertAtBottom(nodes);
latencies.insertAtBottom(latency);
bw_multis.insertAtBottom(bw_multiplier);
// opposite direction link
Vector < SwitchID > otherDirectionNodes;
otherDirectionNodes.setSize(2);
otherDirectionNodes[0] = nodes[1];
otherDirectionNodes[1] = nodes[0];
nodePairs.insertAtBottom(otherDirectionNodes);
latencies.insertAtBottom(latency);
bw_multis.insertAtBottom(bw_multiplier);
}
}
// add links
ASSERT(nodePairs.size() == latencies.size() && latencies.size() == bw_multis.size())
for (int k = 0; k < nodePairs.size(); k++) {
ASSERT(nodePairs[k].size() == 2);
addLink(nodePairs[k][0], nodePairs[k][1], latencies[k], bw_multis[k]);
}
}
// hierarchical switch topology
void Topology::makeHierarchicalSwitch(int fan_out_degree)
{
// Make a row of switches with only one input. This extra row makes
// sure the links out of the nodes have latency and limited
// bandwidth.
// number of inter-chip switches, i.e. the last row of switches
Vector<SwitchID> last_level;
for (int i=0; i<m_nodes; i++) {
SwitchID new_switch = newSwitchID(); // internal switch id #
addLink(i, new_switch, NETWORK_LINK_LATENCY);
last_level.insertAtBottom(new_switch);
}
// Create Hierarchical Switches
// start from the bottom level and work up to root
Vector<SwitchID> next_level;
while(last_level.size() > 1) {
for (int i=0; i<last_level.size(); i++) {
if ((i % fan_out_degree) == 0) {
next_level.insertAtBottom(newSwitchID());
}
// Add this link to the last switch we created
addLink(last_level[i], next_level[next_level.size()-1], NETWORK_LINK_LATENCY);
}
// Make the current level the last level to get ready for next
// iteration
last_level = next_level;
next_level.clear();
}
SwitchID root_switch = last_level[0];
Vector<SwitchID> out_level;
for (int i=0; i<m_nodes; i++) {
out_level.insertAtBottom(m_nodes+i);
}
// Build the down network from the endpoints to the root
while(out_level.size() != 1) {
// A level of switches
for (int i=0; i<out_level.size(); i++) {
if ((i % fan_out_degree) == 0) {
if (out_level.size() > fan_out_degree) {
next_level.insertAtBottom(newSwitchID());
} else {
next_level.insertAtBottom(root_switch);
}
}
// Add this link to the last switch we created
addLink(next_level[next_level.size()-1], out_level[i], NETWORK_LINK_LATENCY);
}
// Make the current level the last level to get ready for next
// iteration
out_level = next_level;
next_level.clear();
}
}
void Topology::make2DTorus()
{
Vector< Vector < SwitchID > > nodePairs; // node pairs extracted from the file
Vector<int> latencies; // link latencies for each link extracted
Vector<int> bw_multis; // bw multipliers for each link extracted
Vector < SwitchID > nodes; // temporary buffer
nodes.setSize(2);
// number of inter-chip switches
int numberOfTorusSwitches = m_nodes/MachineType_base_level(MachineType_NUM);
// one switch per machine node grouping
Vector<SwitchID> torusSwitches;
for(int i=0; i<numberOfTorusSwitches; i++){
SwitchID new_switch = newSwitchID();
torusSwitches.insertAtBottom(new_switch);
}
makeSwitchesPerChip(nodePairs, latencies, bw_multis, numberOfTorusSwitches);
int lengthOfSide = (int)sqrt((double)numberOfTorusSwitches);
// Now connect the inter-chip torus links
int latency = NETWORK_LINK_LATENCY; // external link latency
int bw_multiplier = 1; // external link bw multiplier of the global bandwidth
for(int i=0; i<numberOfTorusSwitches; i++){
nodes[0] = torusSwitches[i]; // current switch
// left
if(nodes[0]%lengthOfSide == 0){ // determine left neighbor
nodes[1] = nodes[0] - 1 + lengthOfSide;
} else {
nodes[1] = nodes[0] - 1;
}
nodePairs.insertAtBottom(nodes);
latencies.insertAtBottom(latency);
bw_multis.insertAtBottom(bw_multiplier);
// right
if((nodes[0] + 1)%lengthOfSide == 0){ // determine right neighbor
nodes[1] = nodes[0] + 1 - lengthOfSide;
} else {
nodes[1] = nodes[0] + 1;
}
nodePairs.insertAtBottom(nodes);
latencies.insertAtBottom(latency);
bw_multis.insertAtBottom(bw_multiplier);
// top
if(nodes[0] - lengthOfSide < 2*m_nodes){ // determine if node is on the top
nodes[1] = nodes[0] - lengthOfSide + (lengthOfSide*lengthOfSide);
} else {
nodes[1] = nodes[0] - lengthOfSide;
}
nodePairs.insertAtBottom(nodes);
latencies.insertAtBottom(latency);
bw_multis.insertAtBottom(bw_multiplier);
// bottom
if(nodes[0] + lengthOfSide >= 2*m_nodes+numberOfTorusSwitches){ // determine if node is on the bottom
// sorin: bad bug if this is a > instead of a >=
nodes[1] = nodes[0] + lengthOfSide - (lengthOfSide*lengthOfSide);
} else {
nodes[1] = nodes[0] + lengthOfSide;
}
nodePairs.insertAtBottom(nodes);
latencies.insertAtBottom(latency);
bw_multis.insertAtBottom(bw_multiplier);
}
// add links
ASSERT(nodePairs.size() == latencies.size() && latencies.size() == bw_multis.size())
for (int k = 0; k < nodePairs.size(); k++) {
ASSERT(nodePairs[k].size() == 2);
addLink(nodePairs[k][0], nodePairs[k][1], latencies[k], bw_multis[k]);
}
}