void BcmPort::init(bool warmBoot) {
bool up = false;
if (warmBoot) {
// Get port status from HW on warm boot.
// All ports are initially down on a cold boot.
int linkStatus;
opennsl_port_link_status_get(unit_, port_, &linkStatus);
up = (linkStatus == OPENNSL_PORT_LINK_STATUS_UP);
} else {
// In open source code, we don't have any guarantees for the
// state of the port at startup. Bringing them down guarantees
// that things are in a known state.
//
// We should only be doing this on cold boot, since warm booting
// should be initializing the state for us.
auto rv = opennsl_port_enable_set(unit_, port_, false);
bcmCheckError(rv, "failed to set port to known state: ", port_);
}
setPortStatus(up);
// Linkscan should be enabled if the port is enabled already
auto linkscan = isEnabled() ? OPENNSL_LINKSCAN_MODE_SW :
OPENNSL_LINKSCAN_MODE_NONE;
auto rv = opennsl_linkscan_mode_set(unit_, port_, linkscan);
bcmCheckError(rv, "failed to set initial linkscan mode on port ", port_);
}
void BcmPort::disable(const std::shared_ptr<Port>& swPort) {
if (!isEnabled()) {
// Already disabled
return;
}
auto pbmp = getPbmp();
for (auto entry : swPort->getVlans()) {
auto rv = opennsl_vlan_port_remove(unit_, entry.first, pbmp);
bcmCheckError(rv, "failed to remove disabled port ",
swPort->getID(), " from VLAN ", entry.first);
}
// Disable packet and byte counter statistic collection.
auto rv = opennsl_port_stat_enable_set(unit_, gport_, false);
bcmCheckError(rv, "Unexpected error disabling counter DMA on port ",
swPort->getID());
// Disable linkscan
rv = opennsl_linkscan_mode_set(unit_, port_, OPENNSL_LINKSCAN_MODE_NONE);
bcmCheckError(rv, "Failed to disable linkscan on port ", swPort->getID());
rv = opennsl_port_enable_set(unit_, port_, false);
bcmCheckError(rv, "failed to disable port ", swPort->getID());
}
void BcmPort::setIngressVlan(const shared_ptr<Port>& swPort) {
opennsl_vlan_t currVlan;
auto rv = opennsl_port_untagged_vlan_get(unit_, port_, &currVlan);
bcmCheckError(rv, "failed to get ingress VLAN for port ", swPort->getID());
opennsl_vlan_t bcmVlan = swPort->getIngressVlan();
if (bcmVlan != currVlan) {
rv = opennsl_port_untagged_vlan_set(unit_, port_, bcmVlan);
bcmCheckError(rv, "failed to set ingress VLAN for port ",
swPort->getID(), " to ", swPort->getIngressVlan());
}
}
cfg::PortSpeed BcmPort::getSpeed() const {
int curSpeed{0};
auto rv = opennsl_port_speed_get(unit_, port_, &curSpeed);
bcmCheckError(
rv, "Failed to get current speed for port ", port_);
return cfg::PortSpeed(curSpeed);
}
void BcmPort::init(bool warmBoot) {
bool up = false;
if (warmBoot) {
// Get port status from HW on warm boot.
// All ports are initially down on a cold boot.
int linkStatus;
opennsl_port_link_status_get(unit_, port_, &linkStatus);
up = (linkStatus == OPENNSL_PORT_LINK_STATUS_UP);
} else {
// In open source code, we don't have any guarantees for the
// state of the port at startup. Bringing them down guarantees
// that things are in a known state.
//
// We should only be doing this on cold boot, since warm booting
// should be initializing the state for us.
auto rv = opennsl_port_enable_set(unit_, port_, false);
bcmCheckError(rv, "failed to set port to known state: ", port_);
}
// Notify platform port of initial state/speed
getPlatformPort()->linkSpeedChanged(getSpeed());
getPlatformPort()->linkStatusChanged(up, isEnabled());
enableLinkscan();
}
void BcmUnit::detach() {
attached_.store(false, std::memory_order_release);
// Clean up SDK state, without touching the hardware
auto rv = _opennsl_shutdown(unit_);
bcmCheckError(rv, "failed to clean up BCM state during warm boot shutdown");
}
BcmPort::BcmPort(BcmSwitch* hw, opennsl_port_t port,
BcmPlatformPort* platformPort)
: hw_(hw),
port_(port),
platformPort_(platformPort),
unit_(hw->getUnit()) {
// Obtain the gport handle from the port handle.
int rv = opennsl_port_gport_get(unit_, port_, &gport_);
bcmCheckError(rv, "Failed to get gport for BCM port ", port_);
disablePause();
// Initialize our stats data structures
auto statMap = fbData->getStatMap();
const auto expType = stats::AVG;
outQueueLen_ = statMap->getLockAndStatItem(statName("out_queue_length"),
&expType);
auto histMap = fbData->getHistogramMap();
stats::ExportedHistogram pktLenHist(1, 0, kInPktLengthStats.size());
inPktLengths_ = histMap->getOrCreateUnlocked(statName("in_pkt_lengths"),
&pktLenHist);
outPktLengths_ = histMap->getOrCreateUnlocked(statName("out_pkt_lengths"),
&pktLenHist);
setConfiguredMaxSpeed();
VLOG(2) << "created BCM port:" << port_ << ", gport:" << gport_
<< ", FBOSS PortID:" << platformPort_->getPortID();
}
void BcmPort::enable(const std::shared_ptr<Port>& swPort) {
if (isEnabled()) {
// Port is already enabled, don't need to do anything
return;
}
auto pbmp = getPbmp();
opennsl_pbmp_t emptyPortList;
OPENNSL_PBMP_CLEAR(emptyPortList);
int rv;
for (auto entry : swPort->getVlans()) {
if (!entry.second.tagged) {
rv = opennsl_vlan_port_add(unit_, entry.first, pbmp, pbmp);
} else {
rv = opennsl_vlan_port_add(unit_, entry.first, pbmp, emptyPortList);
}
bcmCheckError(rv, "failed to add enabled port ",
swPort->getID(), " to VLAN ", entry.first);
}
// Drop packets to/from this port that are tagged with a VLAN that this
// port isn't a member of.
rv = opennsl_port_vlan_member_set(unit_, port_,
OPENNSL_PORT_VLAN_MEMBER_INGRESS |
OPENNSL_PORT_VLAN_MEMBER_EGRESS);
bcmCheckError(rv, "failed to set VLAN filtering on port ",
swPort->getID());
// Set the speed and ingress vlan before enabling
program(swPort);
// Enable packet and byte counter statistic collection.
rv = opennsl_port_stat_enable_set(unit_, gport_, true);
if (rv != OPENNSL_E_EXISTS) {
// Don't throw an error if counter collection is already enabled
bcmCheckError(rv, "Unexpected error enabling counter DMA on port ",
swPort->getID());
}
// Enable linkscan
rv = opennsl_linkscan_mode_set(unit_, port_, OPENNSL_LINKSCAN_MODE_SW);
bcmCheckError(rv, "Failed to enable linkscan on port ", swPort->getID());
rv = opennsl_port_enable_set(unit_, port_, true);
bcmCheckError(rv, "failed to enable port ", swPort->getID());
}
bool BcmPort::isUp() {
if (!isEnabled()) {
return false;
}
int linkStatus;
auto rv = opennsl_port_link_status_get(hw_->getUnit(), port_, &linkStatus);
bcmCheckError(rv, "could not find if the port ", port_, " is up or down...");
return linkStatus == OPENNSL_PORT_LINK_STATUS_UP;
}
void BcmRoute::programLpmRoute(opennsl_if_t egressId,
const RouteNextHopEntry& fwd) {
opennsl_l3_route_t rt;
initL3RouteT(&rt);
rt.l3a_intf = egressId;
if (fwd.getNextHopSet().size() > 1) { // multipath
rt.l3a_flags |= OPENNSL_L3_MULTIPATH;
}
bool addRoute = false;
const auto warmBootCache = hw_->getWarmBootCache();
auto vrfAndPfx2RouteCitr = warmBootCache->findRoute(vrf_, prefix_, len_);
if (vrfAndPfx2RouteCitr != warmBootCache->vrfAndPrefix2Route_end()) {
// Lambda to compare if the routes are equivalent and thus we need to
// do nothing
auto equivalent =
[=] (const opennsl_l3_route_t& newRoute,
const opennsl_l3_route_t& existingRoute) {
// Compare flags (primarily MULTIPATH vs non MULTIPATH
// and egress id.
return existingRoute.l3a_flags == newRoute.l3a_flags &&
existingRoute.l3a_intf == newRoute.l3a_intf;
};
if (!equivalent(rt, vrfAndPfx2RouteCitr->second)) {
VLOG (3) << "Updating route for : " << prefix_ << "/"
<< static_cast<int>(len_) << " in vrf : " << vrf_;
// This is a change
rt.l3a_flags |= OPENNSL_L3_REPLACE;
addRoute = true;
} else {
VLOG(3) << " Route for : " << prefix_ << "/" << static_cast<int>(len_)
<< " in vrf : " << vrf_ << " already exists";
}
} else {
addRoute = true;
}
if (addRoute) {
if (vrfAndPfx2RouteCitr == warmBootCache->vrfAndPrefix2Route_end()) {
VLOG (3) << "Adding route for : " << prefix_ << "/"
<< static_cast<int>(len_) << " in vrf : " << vrf_;
}
if (added_) {
rt.l3a_flags |= OPENNSL_L3_REPLACE;
}
auto rc = opennsl_l3_route_add(hw_->getUnit(), &rt);
bcmCheckError(rc, "failed to create a route entry for ", prefix_, "/",
static_cast<int>(len_), " @ ", fwd, " @egress ", egressId);
VLOG(3) << "created a route entry for " << prefix_.str() << "/"
<< static_cast<int>(len_) << " @egress " << egressId
<< " with " << fwd;
}
if (vrfAndPfx2RouteCitr != warmBootCache->vrfAndPrefix2Route_end()) {
warmBootCache->programmed(vrfAndPfx2RouteCitr);
}
}
void BcmUnit::attach() {
if (attached_.load(std::memory_order_acquire)) {
throw FbossError("unit ", unit_, " already initialized");
}
auto rv = opennsl_switch_event_register(unit_, switchEventCallback, this);
bcmCheckError(rv, "unable to register switch event callback for unit ",
unit_);
attached_.store(true, std::memory_order_release);
}
BcmPort::BcmPort(BcmSwitch* hw, opennsl_port_t port,
BcmPlatformPort* platformPort)
: hw_(hw),
port_(port),
platformPort_(platformPort),
unit_(hw->getUnit()) {
// Obtain the gport handle from the port handle.
int rv = opennsl_port_gport_get(unit_, port_, &gport_);
bcmCheckError(rv, "Failed to get gport for BCM port ", port_);
// Initialize our stats data structures
reinitPortStats();
VLOG(2) << "created BCM port:" << port_ << ", gport:" << gport_
<< ", FBOSS PortID:" << platformPort_->getPortID();
}
void BcmPort::setSpeed(const shared_ptr<Port>& swPort) {
int ret;
cfg::PortSpeed desiredPortSpeed;
if (swPort->getSpeed() == cfg::PortSpeed::DEFAULT) {
int speed;
ret = opennsl_port_speed_max(unit_, port_, &speed);
bcmCheckError(ret, "failed to get max speed for port", swPort->getID());
desiredPortSpeed = cfg::PortSpeed(speed);
} else {
desiredPortSpeed = swPort->getSpeed();
}
int desiredSpeed = static_cast<int>(desiredPortSpeed);
// Unnecessarily updating BCM port speed actually causes
// the port to flap, even if this should be a noop, so check current
// speed before making speed related changes. Doing so fixes
// the interface flaps we were seeing during warm boots
int curSpeed = static_cast<int>(getSpeed());
// If the port is down or disabled its safe to update mode and speed to
// desired values
bool portUp = isUp();
// Update to correct mode and speed settings if the port is down/disabled
// or if the speed changed. Ideally we would like to always update to the
// desired mode and speed. However these changes are disruptive, in that
// they cause a port flap. So to avoid that, we don't update to desired
// mode if the port is UP and running at the desired speed. Speed changes
// though are applied to UP ports as well, since running at wrong (lower than
// desired) speed is pretty dangerous, and can trigger non obvious outages.
//
// Another practical reason for not updating to the desired mode on ports that
// are UP is that there is at least one bug whereby SDK thinks that the ports
// are in a different mode than they actually are. We are tracking that
// separately. Once that is resolved, we can do a audit to see that if all
// ports are in desired mode settings, we can make mode changes a first
// class citizen as well.
if (!portUp || curSpeed != desiredSpeed) {
opennsl_port_if_t desiredMode = getDesiredInterfaceMode(desiredPortSpeed,
swPort->getID(),
swPort->getName());
// Check whether we have the correct interface set
opennsl_port_if_t curMode;
ret = opennsl_port_interface_get(unit_, port_, &curMode);
bcmCheckError(ret,
"Failed to get current interface setting for port ",
swPort->getID());
if (curMode != desiredMode) {
// Changes to the interface setting only seem to take effect on the next
// call to opennsl_port_speed_set()
ret = opennsl_port_interface_set(unit_, port_, desiredMode);
bcmCheckError(
ret, "failed to set interface type for port ", swPort->getID());
}
if (portUp) {
// Changing the port speed causes traffic disruptions, but not doing
// it would cause inconsistency. Warn the user.
LOG(WARNING) << "Changing port speed on up port. This will "
<< "disrupt traffic. Port: " << swPort->getName()
<< " id: " << swPort->getID();
}
// Note that we call speed_set even if the speed is already set
// properly and port is down. This is because speed_set
// reinitializes the MAC layer of the port and allows us to pick
// up changes in interface mode and finalize flex port
// transitions. We ensure that the port is down for these
// potentially unnecessary calls, as otherwise this will cause
// port flaps on ports where link is up.
ret = opennsl_port_speed_set(unit_, port_, desiredSpeed);
bcmCheckError(
ret,
"failed to set speed to ",
desiredSpeed,
" from ",
curSpeed,
", on port ",
swPort->getID());
getPlatformPort()->linkSpeedChanged(desiredPortSpeed);
}
}
void BcmPort::enableLinkscan() {
int rv = opennsl_linkscan_mode_set(unit_, port_, OPENNSL_LINKSCAN_MODE_SW);
bcmCheckError(rv, "Failed to enable linkscan on port ", port_);
}
bool BcmPort::isEnabled() {
int enabled;
auto rv = opennsl_port_enable_get(unit_, port_, &enabled);
bcmCheckError(rv, "Failed to determine if port is already disabled");
return static_cast<bool>(enabled);
}
void BcmPort::setSpeed(const shared_ptr<Port>& swPort) {
if (isEnabled()) {
LOG(ERROR) << "Cannot set port speed while the port is enabled. Port: "
<< swPort->getName() << " id: " << swPort->getID();
return;
}
int ret;
cfg::PortSpeed desiredPortSpeed;
if (swPort->getSpeed() == cfg::PortSpeed::DEFAULT) {
int speed;
ret = opennsl_port_speed_max(unit_, port_, &speed);
bcmCheckError(ret, "failed to get max speed for port", swPort->getID());
desiredPortSpeed = cfg::PortSpeed(speed);
} else {
desiredPortSpeed = swPort->getSpeed();
}
opennsl_port_if_t desiredMode = getDesiredInterfaceMode(desiredPortSpeed,
swPort->getID(),
swPort->getName());
opennsl_port_if_t curMode;
ret = opennsl_port_interface_get(unit_, port_, &curMode);
bcmCheckError(ret,
"Failed to get current interface setting for port ",
swPort->getID());
bool updateSpeed = false;
if (curMode != desiredMode) {
ret = opennsl_port_interface_set(unit_, port_, desiredMode);
bcmCheckError(
ret, "failed to set interface type for port ", swPort->getID());
// Changes to the interface setting only seem to take effect on the next
// call to opennsl_port_speed_set(). Therefore make sure we update the
// speed below, even if the speed is already at the desired setting.
updateSpeed = true;
}
int desiredSpeed = static_cast<int>(desiredPortSpeed);
// Unnecessarily updating BCM port speed actually causes
// the port to flap, even if this should be a noop, so check current
// speed before making speed related changes. Doing so fixes
// the interface flaps we were seeing during warm boots
if (!updateSpeed && desiredMode != OPENNSL_PORT_IF_KR4) {
int curSpeed;
ret = opennsl_port_speed_get(unit_, port_, &curSpeed);
bcmCheckError(
ret, "Failed to get current speed for port ", swPort->getID());
updateSpeed |= (curSpeed != desiredSpeed);
}
if (updateSpeed) {
if (desiredMode == OPENNSL_PORT_IF_KR4) {
// We don't need to set speed when mode is KR4, since ports in KR4 mode
// do autonegotiation to figure out the speed.
setKR4Ability();
} else {
ret = opennsl_port_speed_set(unit_, port_, desiredSpeed);
bcmCheckError(ret,
"failed to set speed, ",
desiredSpeed,
", to port ",
swPort->getID());
}
}
}