int main(int argc, char *argv[])
{
struct nl_sock *h[1025];
int i;
h[0] = nl_handle_alloc();
printf("Created handle with port 0x%x\n",
nl_socket_get_local_port(h[0]));
nl_handle_destroy(h[0]);
h[0] = nl_handle_alloc();
printf("Created handle with port 0x%x\n",
nl_socket_get_local_port(h[0]));
nl_handle_destroy(h[0]);
for (i = 0; i < 1025; i++) {
h[i] = nl_handle_alloc();
if (h[i] == NULL)
nl_perror("Unable to allocate socket");
else
printf("Created handle with port 0x%x\n",
nl_socket_get_local_port(h[i]));
}
return 0;
}
static int usnic_is_nlreply_expected(struct usnic_nl_sk *unlsk,
struct nlmsghdr *nlm_hdr)
{
if (nlm_hdr->nlmsg_pid != nl_socket_get_local_port(unlsk->nlh)
|| nlm_hdr->nlmsg_seq != unlsk->seq) {
usnic_err("Not an expected reply msg pid: %u local pid: %u msg seq: %u expected seq: %u\n",
nlm_hdr->nlmsg_pid,
nl_socket_get_local_port(unlsk->nlh),
nlm_hdr->nlmsg_seq, unlsk->seq);
return 0;
}
return 1;
}
static int usnic_nl_send_query(struct usnic_nl_sk *unlsk, struct nl_msg *msg,
int protocol, int flag)
{
int ret, retry;
struct nlmsghdr *nlhdr;
nlhdr = nlmsg_hdr(msg);
while (1) {
nlhdr->nlmsg_pid = nl_socket_get_local_port(unlsk->nlh);
nlhdr->nlmsg_seq = ++unlsk->seq;
nlmsg_set_proto(msg, protocol);
nlhdr->nlmsg_flags = flag;
/* Sometimes nl_send() can fail simply because the
* kernel is temporarily out of resources, and we
* should just try again. libnl1 and libnl3 handle
* this case a little differently, so use the
* USD_NL_SEND() macro to hide the differences. If
* retry comes back as true, then sleep a little and
* try again. */
USD_NL_SEND(unlsk->nlh, msg, ret, retry);
if (retry) {
usleep(5);
continue;
}
break;
}
return ret;
}
static void nl_socket_free(struct nl_sock* handle) {
uint32_t port = nl_socket_get_local_port(handle);
port >>= 22;
g_portbitmap[port / 32] &= ~(1 << (port % 32));
nl_handle_destroy(handle);
}
/**
* virNetlinkEventServiceLocalPid:
*
* Returns the nl_pid value that was used to bind() the netlink socket
* used by the netlink event service, or -1 on error (netlink
* guarantees that this value will always be > 0).
*/
int virNetlinkEventServiceLocalPid(void)
{
if (!(server && server->netlinknh)) {
netlinkError(VIR_ERR_INTERNAL_ERROR, "%s",
_("netlink event service not running"));
return -1;
}
return (int)nl_socket_get_local_port(server->netlinknh);
}
/**
* virNetlinkEventServiceLocalPid:
*
* @protocol: netlink protocol
*
* Returns the nl_pid value that was used to bind() the netlink socket
* used by the netlink event service, or -1 on error (netlink
* guarantees that this value will always be > 0).
*/
int virNetlinkEventServiceLocalPid(unsigned int protocol)
{
if (protocol >= MAX_LINKS)
return -EINVAL;
if (!(server[protocol] && server[protocol]->netlinknh)) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("netlink event service not running"));
return -1;
}
return (int)nl_socket_get_local_port(server[protocol]->netlinknh);
}
static int rtnl_send_ack_disable(struct usnic_rtnl_sk *unlsk, struct nl_msg *msg)
{
struct nlmsghdr *nlhdr;
nlhdr = nlmsg_hdr(msg);
nlhdr->nlmsg_pid = nl_socket_get_local_port(unlsk->nlh);
nlhdr->nlmsg_seq = ++unlsk->seq;
nlmsg_set_proto(msg, NETLINK_ROUTE);
nlhdr->nlmsg_flags |= NLM_F_REQUEST;
return nl_send(unlsk->nlh, msg);
}
int main(int argc, char *argv[])
{
struct nl_handle *h;
int i;
for (i = 0; i < 1025; i++) {
h = nl_handle_alloc();
printf("Created handle with port 0x%x\n",
nl_socket_get_local_port(h));
}
return 0;
}
static int usnic_nl_send_query(struct usnic_nl_sk *unlsk, struct nl_msg *msg,
int protocol, int flag)
{
struct nlmsghdr *nlhdr;
nlhdr = nlmsg_hdr(msg);
nlhdr->nlmsg_pid = nl_socket_get_local_port(unlsk->nlh);
nlhdr->nlmsg_seq = ++unlsk->seq;
nlmsg_set_proto(msg, protocol);
nlhdr->nlmsg_flags = flag;
return nl_send(unlsk->nlh, msg);
}
static int
event_msg_recv (struct nl_msg *msg, void *arg)
{
struct nl_sock *nlh = arg;
struct nlmsghdr *hdr = nlmsg_hdr (msg);
struct ucred *creds = nlmsg_get_creds (msg);
const struct sockaddr_nl *snl;
guint32 local_port;
gboolean accept_msg = FALSE;
/* Only messages sent from the kernel */
if (!creds || creds->uid != 0) {
nm_log_dbg (LOGD_HW, "ignoring netlink message from UID %d",
creds ? creds->uid : -1);
return NL_SKIP;
}
snl = nlmsg_get_src (msg);
g_assert (snl);
/* Accept any messages from the kernel */
if (hdr->nlmsg_pid == 0 || snl->nl_pid == 0)
accept_msg = TRUE;
/* And any multicast message directed to our netlink PID, since multicast
* currently requires CAP_ADMIN to use.
*/
local_port = nl_socket_get_local_port (nlh);
if ((hdr->nlmsg_pid == local_port) && snl->nl_groups)
accept_msg = TRUE;
if (accept_msg == FALSE) {
nm_log_dbg (LOGD_HW, "ignoring netlink message from PID %d (local PID %d, multicast %d)",
hdr->nlmsg_pid,
local_port,
(hdr->nlmsg_flags & NLM_F_MULTI));
return NL_SKIP;
}
return NL_OK;
}
static int rtnl_raw_parse_cb(struct nl_msg *msg, void *arg)
{
struct nl_lookup_arg *lookup_arg = (struct nl_lookup_arg *)arg;
struct usnic_rtnl_sk *unlsk = lookup_arg->unlsk;
struct nlmsghdr *nlm_hdr = nlmsg_hdr(msg);
struct rtmsg *rtm;
struct nlattr *tb[RTA_MAX + 1];
int found = 0;
int err;
#if WANT_DEBUG_MSGS
nl_msg_dump(msg, stderr);
#endif /* WANT_DEBUG_MSGS */
lookup_arg->nh_addr = 0;
lookup_arg->found = 0;
lookup_arg->replied = 0;
lookup_arg->msg_count++;
if (nlm_hdr->nlmsg_pid != nl_socket_get_local_port(unlsk->nlh)
|| nlm_hdr->nlmsg_seq != unlsk->seq) {
usnic_err("Not an expected reply msg pid: %u local pid: %u "
"msg seq: %u expected seq: %u\n",
nlm_hdr->nlmsg_pid, nl_socket_get_local_port(unlsk->nlh),
nlm_hdr->nlmsg_seq, unlsk->seq);
return NL_SKIP;
}
lookup_arg->replied = 1;
if (nlm_hdr->nlmsg_type == NLMSG_ERROR) {
struct nlmsgerr *e = (struct nlmsgerr *)nlmsg_data(nlm_hdr);
if (nlm_hdr->nlmsg_len >= (__u32)nlmsg_msg_size(sizeof(*e))) {
usnic_err("Received a netlink error message %d\n",
e->error);
}
else {
usnic_err("Received a truncated netlink error message\n");
}
return NL_STOP;
}
if (nlm_hdr->nlmsg_type != RTM_NEWROUTE) {
usnic_err("Received an invalid route request reply message\n");
return NL_STOP;
}
rtm = nlmsg_data(nlm_hdr);
if (rtm->rtm_family != AF_INET) {
usnic_err("RTM message contains invalid AF family\n");
return NL_STOP;
}
init_route_policy(route_policy);
err = nlmsg_parse(nlm_hdr, sizeof(struct rtmsg), tb, RTA_MAX,
route_policy);
if (err < 0) {
usnic_err("nlmsg parse error %d\n", err);
return NL_STOP;
}
if (tb[RTA_OIF]) {
if (nla_get_u32(tb[RTA_OIF]) == (uint32_t)lookup_arg->oif)
found = 1;
else
usnic_err("Retrieved route has a different outgoing interface %d (expected %d)\n",
nla_get_u32(tb[RTA_OIF]),
lookup_arg->oif);
}
if (found && tb[RTA_METRICS]) {
lookup_arg->metric = (int)nla_get_u32(tb[RTA_METRICS]);
}
if (found && tb[RTA_GATEWAY])
lookup_arg->nh_addr = nla_get_u32(tb[RTA_GATEWAY]);
lookup_arg->found = found;
return NL_STOP;
}
void
ind_ovs_port_added(uint32_t port_no, const char *ifname, of_mac_addr_t mac_addr)
{
indigo_error_t err;
if (ind_ovs_ports[port_no]) {
return;
}
struct ind_ovs_port *port = calloc(1, sizeof(*port));
if (port == NULL) {
LOG_ERROR("failed to allocate port");
return;
}
strncpy(port->ifname, ifname, sizeof(port->ifname));
port->dp_port_no = port_no;
port->mac_addr = mac_addr;
aim_ratelimiter_init(&port->upcall_log_limiter, 1000*1000, 5, NULL);
aim_ratelimiter_init(&port->pktin_limiter, PORT_PKTIN_INTERVAL, PORT_PKTIN_BURST_SIZE, NULL);
pthread_mutex_init(&port->quiesce_lock, NULL);
pthread_cond_init(&port->quiesce_cvar, NULL);
port->notify_socket = ind_ovs_create_nlsock();
if (port->notify_socket == NULL) {
goto cleanup_port;
}
if (nl_socket_set_nonblocking(port->notify_socket) < 0) {
LOG_ERROR("failed to set netlink socket nonblocking");
goto cleanup_port;
}
struct nl_msg *msg = ind_ovs_create_nlmsg(ovs_vport_family, OVS_VPORT_CMD_SET);
nla_put_u32(msg, OVS_VPORT_ATTR_PORT_NO, port_no);
nla_put_u32(msg, OVS_VPORT_ATTR_UPCALL_PID,
nl_socket_get_local_port(port->notify_socket));
err = ind_ovs_transact(msg);
if (err < 0) {
LOG_ERROR("datapath failed to configure port %s", ifname);
goto cleanup_port;
}
if (!ind_ovs_get_interface_flags(ifname, &port->ifflags)) {
/* Bring interface up if not already */
if (!(port->ifflags & IFF_UP)) {
port->ifflags |= IFF_UP;
(void) ind_ovs_set_interface_flags(ifname, port->ifflags);
}
} else {
/* Not a netdev, fake the interface flags */
port->ifflags = IFF_UP;
}
/* Ensure port is fully populated before publishing it. */
__sync_synchronize();
ind_ovs_ports[port_no] = port;
if ((err = port_status_notify(port_no, OF_PORT_CHANGE_REASON_ADD)) < 0) {
LOG_WARN("failed to notify controller of port addition");
/* Can't cleanup the port because it's already visible to other
* threads. */
}
ind_ovs_upcall_register(port);
LOG_INFO("Added port %s", port->ifname);
ind_ovs_kflow_invalidate_all();
return;
cleanup_port:
assert(ind_ovs_ports[port_no] == NULL);
if (port->notify_socket) {
nl_socket_free(port->notify_socket);
}
free(port);
}
/**
* Transmit Netlink message (taking IO vector)
* @arg sk Netlink socket (required)
* @arg msg Netlink message to be sent (required)
* @arg iov IO vector to be sent (required)
* @arg iovlen Number of struct iovec to be sent (required)
*
* This function is identical to nl_send() except that instead of taking a
* `struct nl_msg` object it takes an IO vector. Please see the description
* of `nl_send()`.
*
* @callback This function triggers the `NL_CB_MSG_OUT` callback.
*
* @see nl_send()
*
* @return Number of bytes sent on success or a negative error code.
*
* @lowlevel
*/
int nl_send_iovec(struct nl_sock *sk, struct nl_msg *msg, struct iovec *iov, unsigned iovlen)
{
struct sockaddr_nl *dst;
struct ucred *creds;
struct msghdr hdr = {
.msg_name = (void *) &sk->s_peer,
.msg_namelen = sizeof(struct sockaddr_nl),
.msg_iov = iov,
.msg_iovlen = iovlen,
};
/* Overwrite destination if specified in the message itself, defaults
* to the peer address of the socket.
*/
dst = nlmsg_get_dst(msg);
if (dst->nl_family == AF_NETLINK)
hdr.msg_name = dst;
/* Add credentials if present. */
creds = nlmsg_get_creds(msg);
if (creds != NULL) {
char buf[CMSG_SPACE(sizeof(struct ucred))];
struct cmsghdr *cmsg;
hdr.msg_control = buf;
hdr.msg_controllen = sizeof(buf);
cmsg = CMSG_FIRSTHDR(&hdr);
cmsg->cmsg_level = SOL_SOCKET;
cmsg->cmsg_type = SCM_CREDENTIALS;
cmsg->cmsg_len = CMSG_LEN(sizeof(struct ucred));
memcpy(CMSG_DATA(cmsg), creds, sizeof(struct ucred));
}
return nl_sendmsg(sk, msg, &hdr);
}
/**
* Transmit Netlink message
* @arg sk Netlink socket (required)
* @arg msg Netlink message (required)
*
* Transmits the Netlink message `msg` over the Netlink socket using the
* `sendmsg()` system call. This function is based on `nl_send_iovec()` but
* takes care of initializing a `struct iovec` based on the `msg` object.
*
* The message is addressed to the peer as specified in the socket by either
* the nl_socket_set_peer_port() or nl_socket_set_peer_groups() function.
* The peer address can be overwritten by specifying an address in the `msg`
* object using nlmsg_set_dst().
*
* If present in the `msg`, credentials set by the nlmsg_set_creds() function
* are added to the control buffer of the message.
*
* @par Overwriting Capability:
* Calls to this function can be overwritten by providing an alternative using
* the nl_cb_overwrite_send() function.
*
* @callback This function triggers the `NL_CB_MSG_OUT` callback.
*
* @attention
* Unlike `nl_send_auto()`, this function does *not* finalize the message in
* terms of automatically adding needed flags or filling out port numbers.
*
* @see nl_send_auto()
* @see nl_send_iovec()
* @see nl_socket_set_peer_port()
* @see nl_socket_set_peer_groups()
* @see nlmsg_set_dst()
* @see nlmsg_set_creds()
* @see nl_cb_overwrite_send()
*
* @return Number of bytes sent on success or a negative error code.
*/
int nl_send(struct nl_sock *sk, struct nl_msg *msg)
{
struct nl_cb *cb = sk->s_cb;
if (cb->cb_send_ow)
return cb->cb_send_ow(sk, msg);
else {
struct iovec iov = {
.iov_base = (void *) nlmsg_hdr(msg),
.iov_len = nlmsg_hdr(msg)->nlmsg_len,
};
return nl_send_iovec(sk, msg, &iov, 1);
}
}
/**
* Finalize Netlink message
* @arg sk Netlink socket (required)
* @arg msg Netlink message (required)
*
* This function finalizes a Netlink message by completing the message with
* desirable flags and values depending on the socket configuration.
*
* - If not yet filled out, the source address of the message (`nlmsg_pid`)
* will be set to the local port number of the socket.
* - If not yet specified, the next available sequence number is assigned
* to the message (`nlmsg_seq`).
* - If not yet specified, the protocol field of the message will be set to
* the protocol field of the socket.
* - The `NLM_F_REQUEST` Netlink message flag will be set.
* - The `NLM_F_ACK` flag will be set if Auto-ACK mode is enabled on the
* socket.
*/
void nl_complete_msg(struct nl_sock *sk, struct nl_msg *msg)
{
struct nlmsghdr *nlh;
nlh = nlmsg_hdr(msg);
if (nlh->nlmsg_pid == NL_AUTO_PORT)
nlh->nlmsg_pid = nl_socket_get_local_port(sk);
if (nlh->nlmsg_seq == NL_AUTO_SEQ)
nlh->nlmsg_seq = sk->s_seq_next++;
if (msg->nm_protocol == -1)
msg->nm_protocol = sk->s_proto;
nlh->nlmsg_flags |= NLM_F_REQUEST;
if (!(sk->s_flags & NL_NO_AUTO_ACK))
nlh->nlmsg_flags |= NLM_F_ACK;
}
/**
* Finalize and transmit Netlink message
* @arg sk Netlink socket (required)
* @arg msg Netlink message (required)
*
* Finalizes the message by passing it to `nl_complete_msg()` and transmits it
* by passing it to `nl_send()`.
*
* @callback This function triggers the `NL_CB_MSG_OUT` callback.
*
* @see nl_complete_msg()
* @see nl_send()
*
* @return Number of bytes sent or a negative error code.
*/
int nl_send_auto(struct nl_sock *sk, struct nl_msg *msg)
{
nl_complete_msg(sk, msg);
return nl_send(sk, msg);
}
/**
* Finalize and transmit Netlink message and wait for ACK or error message
* @arg sk Netlink socket (required)
* @arg msg Netlink message (required)
*
* Passes the `msg` to `nl_send_auto()` to finalize and transmit it. Frees the
* message and waits (sleeps) for the ACK or error message to be received.
*
* @attention
* Disabling Auto-ACK (nl_socket_disable_auto_ack()) will cause this function
* to return immediately after transmitting the message. However, the peer may
* still be returning an error message in response to the request. It is the
* responsibility of the caller to handle such messages.
*
* @callback This function triggers the `NL_CB_MSG_OUT` callback.
*
* @attention
* This function frees the `msg` object after transmitting it by calling
* `nlmsg_free()`.
*
* @see nl_send_auto().
* @see nl_wait_for_ack()
*
* @return 0 on success or a negative error code.
*/
int nl_send_sync(struct nl_sock *sk, struct nl_msg *msg)
{
int err;
err = nl_send_auto(sk, msg);
nlmsg_free(msg);
if (err < 0)
return err;
return wait_for_ack(sk);
}
/* Create a socket to netlink interface_t */
static int
netlink_socket(nl_handle_t *nl, int flags, int group, ...)
{
int ret;
va_list gp;
memset(nl, 0, sizeof (*nl));
#ifdef _HAVE_LIBNL3_
/* We need to keep libnl3 in step with our netlink socket creation. */
nl->sk = nl_socket_alloc();
if ( nl->sk == NULL ) {
log_message(LOG_INFO, "Netlink: Cannot allocate netlink socket" );
return -1;
}
ret = nl_connect(nl->sk, NETLINK_ROUTE);
if (ret != 0) {
log_message(LOG_INFO, "Netlink: Cannot open netlink socket : (%d)", ret);
return -1;
}
/* Unfortunately we can't call nl_socket_add_memberships() with variadic arguments
* from a variadic argument list passed to us
*/
va_start(gp, group);
while (group != 0) {
if (group < 0) {
va_end(gp);
return -1;
}
if ((ret = nl_socket_add_membership(nl->sk, group))) {
log_message(LOG_INFO, "Netlink: Cannot add socket membership 0x%x : (%d)", group, ret);
return -1;
}
group = va_arg(gp,int);
}
va_end(gp);
if (flags & SOCK_NONBLOCK) {
if ((ret = nl_socket_set_nonblocking(nl->sk))) {
log_message(LOG_INFO, "Netlink: Cannot set netlink socket non-blocking : (%d)", ret);
return -1;
}
}
if ((ret = nl_socket_set_buffer_size(nl->sk, IF_DEFAULT_BUFSIZE, 0))) {
log_message(LOG_INFO, "Netlink: Cannot set netlink buffer size : (%d)", ret);
return -1;
}
nl->nl_pid = nl_socket_get_local_port(nl->sk);
nl->fd = nl_socket_get_fd(nl->sk);
/* Set CLOEXEC */
fcntl(nl->fd, F_SETFD, fcntl(nl->fd, F_GETFD) | FD_CLOEXEC);
#else
socklen_t addr_len;
struct sockaddr_nl snl;
#if !HAVE_DECL_SOCK_NONBLOCK
int sock_flags = flags;
flags &= ~SOCK_NONBLOCK;
#endif
nl->fd = socket(AF_NETLINK, SOCK_RAW | SOCK_CLOEXEC | flags, NETLINK_ROUTE);
if (nl->fd < 0) {
log_message(LOG_INFO, "Netlink: Cannot open netlink socket : (%s)",
strerror(errno));
return -1;
}
#if !HAVE_DECL_SOCK_NONBLOCK
if ((sock_flags & SOCK_NONBLOCK) &&
set_sock_flags(nl->fd, F_SETFL, O_NONBLOCK))
return -1;
#endif
memset(&snl, 0, sizeof (snl));
snl.nl_family = AF_NETLINK;
ret = bind(nl->fd, (struct sockaddr *) &snl, sizeof (snl));
if (ret < 0) {
log_message(LOG_INFO, "Netlink: Cannot bind netlink socket : (%s)",
strerror(errno));
close(nl->fd);
return -1;
}
/* Join the requested groups */
va_start(gp, group);
while (group != 0) {
if (group < 0) {
va_end(gp);
return -1;
}
ret = setsockopt(nl->fd, SOL_NETLINK, NETLINK_ADD_MEMBERSHIP, &group, sizeof(group));
if (ret < 0) {
log_message(LOG_INFO, "Netlink: Cannot add membership on netlink socket : (%s)",
strerror(errno));
va_end(gp);
return -1;
}
group = va_arg(gp,int);
}
va_end(gp);
addr_len = sizeof (snl);
ret = getsockname(nl->fd, (struct sockaddr *) &snl, &addr_len);
if (ret < 0 || addr_len != sizeof (snl)) {
log_message(LOG_INFO, "Netlink: Cannot getsockname : (%s)",
strerror(errno));
close(nl->fd);
return -1;
}
if (snl.nl_family != AF_NETLINK) {
log_message(LOG_INFO, "Netlink: Wrong address family %d",
snl.nl_family);
close(nl->fd);
return -1;
}
/* Save the port id for checking message source later */
nl->nl_pid = snl.nl_pid;
/* Set default rcvbuf size */
if_setsockopt_rcvbuf(&nl->fd, IF_DEFAULT_BUFSIZE);
#endif
nl->seq = (uint32_t)time(NULL);
if (nl->fd < 0)
return -1;
return ret;
}
