static void print_stack(struct key_t *key, __u64 count)
{
__u64 ip[PERF_MAX_STACK_DEPTH] = {};
static bool warned;
int i;
printf("%3lld %s;", count, key->comm);
if (bpf_map_lookup_elem(map_fd[1], &key->kernstack, ip) != 0) {
printf("---;");
} else {
for (i = PERF_MAX_STACK_DEPTH - 1; i >= 0; i--)
print_ksym(ip[i]);
}
printf("-;");
if (bpf_map_lookup_elem(map_fd[1], &key->userstack, ip) != 0) {
printf("---;");
} else {
for (i = PERF_MAX_STACK_DEPTH - 1; i >= 0; i--)
print_addr(ip[i]);
}
printf("\n");
if (key->kernstack == -EEXIST && !warned) {
printf("stackmap collisions seen. Consider increasing size\n");
warned = true;
} else if ((int)key->kernstack < 0 && (int)key->userstack < 0) {
printf("err stackid %d %d\n", key->kernstack, key->userstack);
}
}
int xdp_sock_prog(struct xdp_md *ctx)
{
int *qidconf, key = 0, idx;
unsigned int *rr;
qidconf = bpf_map_lookup_elem(&qidconf_map, &key);
if (!qidconf)
return XDP_ABORTED;
if (*qidconf != ctx->rx_queue_index)
return XDP_PASS;
#if RR_LB /* NB! RR_LB is configured in xdpsock.h */
rr = bpf_map_lookup_elem(&rr_map, &key);
if (!rr)
return XDP_ABORTED;
*rr = (*rr + 1) & (MAX_SOCKS - 1);
idx = *rr;
#else
idx = 0;
#endif
return bpf_redirect_map(&xsks_map, idx, 0);
}
int bpf_prog2(struct pt_regs *ctx)
{
u64 *ts, cur_ts, delta;
int key, cpu;
long *val;
cpu = bpf_get_smp_processor_id();
ts = bpf_map_lookup_elem(&my_map, &cpu);
if (!ts)
return 0;
cur_ts = bpf_ktime_get_ns();
delta = log2l(cur_ts - *ts);
if (delta > MAX_ENTRIES - 1)
delta = MAX_ENTRIES - 1;
key = cpu * MAX_ENTRIES + delta;
val = bpf_map_lookup_elem(&my_lat, &key);
if (val)
__sync_fetch_and_add((long *)val, 1);
return 0;
}
static void print_stack(struct key_t *key, __u64 count)
{
__u64 ip[PERF_MAX_STACK_DEPTH] = {};
static bool warned;
int i;
printf("%s;", key->target);
if (bpf_map_lookup_elem(map_fd[3], &key->tret, ip) != 0) {
printf("---;");
} else {
for (i = PERF_MAX_STACK_DEPTH - 1; i >= 0; i--)
print_ksym(ip[i]);
}
printf("-;");
if (bpf_map_lookup_elem(map_fd[3], &key->wret, ip) != 0) {
printf("---;");
} else {
for (i = 0; i < PERF_MAX_STACK_DEPTH; i++)
print_ksym(ip[i]);
}
printf(";%s %lld\n", key->waker, count);
if ((key->tret == -EEXIST || key->wret == -EEXIST) && !warned) {
printf("stackmap collisions seen. Consider increasing size\n");
warned = true;
} else if (((int)(key->tret) < 0 || (int)(key->wret) < 0)) {
printf("err stackid %d %d\n", key->tret, key->wret);
}
}
static int bpf_do_map(const char *file, uint32_t flags, uint32_t key,
uint32_t value)
{
int fd, ret;
if (flags & BPF_F_PIN) {
fd = bpf_map_create();
printf("bpf: map fd:%d (%s)\n", fd, strerror(errno));
assert(fd > 0);
ret = bpf_obj_pin(fd, file);
printf("bpf: pin ret:(%d,%s)\n", ret, strerror(errno));
assert(ret == 0);
} else {
fd = bpf_obj_get(file);
printf("bpf: get fd:%d (%s)\n", fd, strerror(errno));
assert(fd > 0);
}
if ((flags & BPF_F_KEY_VAL) == BPF_F_KEY_VAL) {
ret = bpf_map_update_elem(fd, &key, &value, 0);
printf("bpf: fd:%d u->(%u:%u) ret:(%d,%s)\n", fd, key, value,
ret, strerror(errno));
assert(ret == 0);
} else if (flags & BPF_F_KEY) {
ret = bpf_map_lookup_elem(fd, &key, &value);
printf("bpf: fd:%d l->(%u):%u ret:(%d,%s)\n", fd, key, value,
ret, strerror(errno));
assert(ret == 0);
}
return 0;
}
static bool map_collect_percpu(int fd, __u32 key, struct record *rec)
{
/* For percpu maps, userspace gets a value per possible CPU */
unsigned int nr_cpus = bpf_num_possible_cpus();
struct datarec values[nr_cpus];
__u64 sum_processed = 0;
__u64 sum_dropped = 0;
__u64 sum_issue = 0;
int i;
if ((bpf_map_lookup_elem(fd, &key, values)) != 0) {
fprintf(stderr,
"ERR: bpf_map_lookup_elem failed key:0x%X\n", key);
return false;
}
/* Get time as close as possible to reading map contents */
rec->timestamp = gettime();
/* Record and sum values from each CPU */
for (i = 0; i < nr_cpus; i++) {
rec->cpu[i].processed = values[i].processed;
sum_processed += values[i].processed;
rec->cpu[i].dropped = values[i].dropped;
sum_dropped += values[i].dropped;
rec->cpu[i].issue = values[i].issue;
sum_issue += values[i].issue;
}
rec->total.processed = sum_processed;
rec->total.dropped = sum_dropped;
rec->total.issue = sum_issue;
return true;
}
static void do_test_lru_sanity5(unsigned long long last_key, int map_fd)
{
unsigned long long key, value[nr_cpus];
/* Ensure the last key inserted by previous CPU can be found */
assert(!bpf_map_lookup_elem(map_fd, &last_key, value));
value[0] = 1234;
key = last_key + 1;
assert(!bpf_map_update_elem(map_fd, &key, value, BPF_NOEXIST));
assert(!bpf_map_lookup_elem(map_fd, &key, value));
/* Cannot find the last key because it was removed by LRU */
assert(bpf_map_lookup_elem(map_fd, &last_key, value));
}
/* simple per-protocol drop counter
*/
static void poll_stats(unsigned int kill_after_s)
{
const unsigned int nr_protos = 256;
unsigned int nr_cpus = bpf_num_possible_cpus();
time_t started_at = time(NULL);
__u64 values[nr_cpus], prev[nr_protos][nr_cpus];
__u32 proto;
int i;
memset(prev, 0, sizeof(prev));
while (!kill_after_s || time(NULL) - started_at <= kill_after_s) {
sleep(STATS_INTERVAL_S);
for (proto = 0; proto < nr_protos; proto++) {
__u64 sum = 0;
assert(bpf_map_lookup_elem(map_fd[0], &proto, values) == 0);
for (i = 0; i < nr_cpus; i++)
sum += (values[i] - prev[proto][i]);
if (sum)
printf("proto %u: sum:%10llu pkts, rate:%10llu pkts/s\n",
proto, sum, sum / STATS_INTERVAL_S);
memcpy(prev[proto], values, sizeof(values));
}
}
}
static __always_inline int do_reg_lookup(void *inner_map, u32 port)
{
int *result;
result = bpf_map_lookup_elem(inner_map, &port);
return result ? *result : -ENOENT;
}
/* simple per-protocol drop counter
*/
static void poll_stats(int interval)
{
unsigned int nr_cpus = bpf_num_possible_cpus();
const unsigned int nr_keys = 256;
__u64 values[nr_cpus], prev[nr_keys][nr_cpus];
__u32 key;
int i;
memset(prev, 0, sizeof(prev));
while (1) {
sleep(interval);
for (key = 0; key < nr_keys; key++) {
__u64 sum = 0;
assert(bpf_map_lookup_elem(map_fd[0], &key, values) == 0);
for (i = 0; i < nr_cpus; i++)
sum += (values[i] - prev[key][i]);
if (sum)
printf("proto %u: %10llu pkt/s\n",
key, sum / interval);
memcpy(prev[key], values, sizeof(values));
}
}
}
static void print_hist(int fd)
{
unsigned int nr_cpus = bpf_num_possible_cpus();
__u64 total_events = 0;
long values[nr_cpus];
__u64 max_cnt = 0;
__u64 cnt[SLOTS];
__u64 value;
__u32 key;
int i;
for (key = 0; key < SLOTS; key++) {
bpf_map_lookup_elem(fd, &key, values);
value = 0;
for (i = 0; i < nr_cpus; i++)
value += values[i];
cnt[key] = value;
total_events += value;
if (value > max_cnt)
max_cnt = value;
}
clear_stats(fd);
for (key = full_range ? 0 : 29; key < SLOTS; key++) {
int c = num_colors * cnt[key] / (max_cnt + 1);
if (text_only)
printf("%s", sym[c]);
else
printf("%s %s", color[c], nocolor);
}
printf(" # %lld\n", total_events);
}
static void print_stacks(void)
{
struct key_t key = {}, next_key;
__u64 value;
__u32 stackid = 0, next_id;
int fd = map_fd[0], stack_map = map_fd[1];
sys_read_seen = sys_write_seen = false;
while (bpf_map_get_next_key(fd, &key, &next_key) == 0) {
bpf_map_lookup_elem(fd, &next_key, &value);
print_stack(&next_key, value);
bpf_map_delete_elem(fd, &next_key);
key = next_key;
}
if (!sys_read_seen || !sys_write_seen) {
printf("BUG kernel stack doesn't contain sys_read() and sys_write()\n");
int_exit(0);
}
/* clear stack map */
while (bpf_map_get_next_key(stack_map, &stackid, &next_id) == 0) {
bpf_map_delete_elem(stack_map, &next_id);
stackid = next_id;
}
}
/* Test deletion */
static void test_lru_sanity4(int map_type, int map_flags, unsigned int tgt_free)
{
int lru_map_fd, expected_map_fd;
unsigned long long key, value[nr_cpus];
unsigned long long end_key;
int next_cpu = 0;
printf("%s (map_type:%d map_flags:0x%X): ", __func__, map_type,
map_flags);
assert(sched_next_online(0, &next_cpu) != -1);
if (map_flags & BPF_F_NO_COMMON_LRU)
lru_map_fd = create_map(map_type, map_flags,
3 * tgt_free * nr_cpus);
else
lru_map_fd = create_map(map_type, map_flags, 3 * tgt_free);
assert(lru_map_fd != -1);
expected_map_fd = create_map(BPF_MAP_TYPE_HASH, 0,
3 * tgt_free);
assert(expected_map_fd != -1);
value[0] = 1234;
for (key = 1; key <= 2 * tgt_free; key++)
assert(!bpf_map_update_elem(lru_map_fd, &key, value,
BPF_NOEXIST));
key = 1;
assert(bpf_map_update_elem(lru_map_fd, &key, value, BPF_NOEXIST));
for (key = 1; key <= tgt_free; key++) {
assert(!bpf_map_lookup_elem(lru_map_fd, &key, value));
assert(!bpf_map_update_elem(expected_map_fd, &key, value,
BPF_NOEXIST));
}
for (; key <= 2 * tgt_free; key++) {
assert(!bpf_map_delete_elem(lru_map_fd, &key));
assert(bpf_map_delete_elem(lru_map_fd, &key));
}
end_key = key + 2 * tgt_free;
for (; key < end_key; key++) {
assert(!bpf_map_update_elem(lru_map_fd, &key, value,
BPF_NOEXIST));
assert(!bpf_map_update_elem(expected_map_fd, &key, value,
BPF_NOEXIST));
}
assert(map_equal(lru_map_fd, expected_map_fd));
close(expected_map_fd);
close(lru_map_fd);
printf("Pass\n");
}
static __always_inline int do_inline_hash_lookup(void *inner_map, u32 port)
{
int *result;
if (inner_map != &port_h)
return -EINVAL;
result = bpf_map_lookup_elem(&port_h, &port);
return result ? *result : -ENOENT;
}
int verify_sockopt_result(int sock_map_fd)
{
__u32 key = 0;
int res;
int rv;
/* check setsockopt for SAVE_SYN */
rv = bpf_map_lookup_elem(sock_map_fd, &key, &res);
EXPECT_EQ(0, rv, "d");
EXPECT_EQ(0, res, "d");
key = 1;
/* check getsockopt for SAVED_SYN */
rv = bpf_map_lookup_elem(sock_map_fd, &key, &res);
EXPECT_EQ(0, rv, "d");
EXPECT_EQ(1, res, "d");
return 0;
err:
return -1;
}
int bpf_prog1(struct pt_regs *ctx)
{
int cpu = bpf_get_smp_processor_id();
u64 *ts = bpf_map_lookup_elem(&my_map, &cpu);
if (ts)
*ts = bpf_ktime_get_ns();
return 0;
}
static void test_map_in_map(void)
{
struct sockaddr_in6 in6 = { .sin6_family = AF_INET6 };
uint32_t result_key = 0, port_key;
int result, inline_result;
int magic_result = 0xfaceb00c;
int ret;
int i;
port_key = rand() & 0x00FF;
populate_map(port_key, magic_result);
in6.sin6_addr.s6_addr16[0] = 0xdead;
in6.sin6_addr.s6_addr16[1] = 0xbeef;
in6.sin6_port = port_key;
for (i = 0; i < NR_TESTS; i++) {
printf("%s: ", test_names[i]);
in6.sin6_addr.s6_addr16[7] = i;
ret = connect(-1, (struct sockaddr *)&in6, sizeof(in6));
assert(ret == -1 && errno == EBADF);
ret = bpf_map_lookup_elem(REG_RESULT_H, &result_key, &result);
assert(!ret);
ret = bpf_map_lookup_elem(INLINE_RESULT_H, &result_key,
&inline_result);
assert(!ret);
if (result != magic_result || inline_result != magic_result) {
printf("Error. result:%d inline_result:%d\n",
result, inline_result);
exit(1);
}
bpf_map_delete_elem(REG_RESULT_H, &result_key);
bpf_map_delete_elem(INLINE_RESULT_H, &result_key);
printf("Pass\n");
}
}
static void print_stacks(int fd)
{
struct key_t key = {}, next_key;
__u64 value;
while (bpf_map_get_next_key(fd, &key, &next_key) == 0) {
bpf_map_lookup_elem(fd, &next_key, &value);
print_stack(&next_key, value);
key = next_key;
}
}
int main(int ac, char **argv)
{
char filename[256];
FILE *f;
int i, sock;
snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]);
if (load_bpf_file(filename)) {
printf("%s", bpf_log_buf);
return 1;
}
sock = open_raw_sock("lo");
assert(setsockopt(sock, SOL_SOCKET, SO_ATTACH_BPF, prog_fd,
sizeof(prog_fd[0])) == 0);
f = popen("ping -c5 localhost", "r");
(void) f;
for (i = 0; i < 5; i++) {
long long tcp_cnt, udp_cnt, icmp_cnt;
int key;
key = IPPROTO_TCP;
assert(bpf_map_lookup_elem(map_fd[0], &key, &tcp_cnt) == 0);
key = IPPROTO_UDP;
assert(bpf_map_lookup_elem(map_fd[0], &key, &udp_cnt) == 0);
key = IPPROTO_ICMP;
assert(bpf_map_lookup_elem(map_fd[0], &key, &icmp_cnt) == 0);
printf("TCP %lld UDP %lld ICMP %lld bytes\n",
tcp_cnt, udp_cnt, icmp_cnt);
sleep(1);
}
return 0;
}
static inline int do_redirect(struct __sk_buff *skb, int nh_off, int dir)
{
__u16 dport, off;
__u8 ip_proto, ip_vl;
int *ifindex;
__be32 ip_dest, ip_src;
char fmt[] = "skb %p len %d ip_dest %x\n";
ip_proto = load_byte(skb, nh_off +
offsetof(struct iphdr, protocol));
ip_src = load_word(skb, nh_off +
offsetof(struct iphdr, saddr));
ip_dest = load_word(skb, nh_off +
offsetof(struct iphdr, daddr));
if (ip_proto != IPPROTO_TCP)
return 0;
ip_vl = load_byte(skb, nh_off);
if (likely(ip_vl == 0x45))
nh_off += sizeof(struct iphdr);
else
nh_off += (ip_vl & 0xF) << 2;
bpf_trace_printk(fmt, sizeof(fmt), skb, skb->len, ip_dest);
dport = load_half(skb, nh_off + offsetof(struct tcphdr, dest));
if (dport != 80)
return 0;
if (dir == 0) {
ifindex = bpf_map_lookup_elem(&container_map, &ip_dest);
if (ifindex) {
set_tcp_dest_port(skb, 8080);
bpf_clone_redirect(skb, *ifindex, 1);
}
} else {
ifindex = bpf_map_lookup_elem(&container_map, &ip_src);
if (ifindex)
set_tcp_dest_port(skb, 80);
}
return -1;
}
static void check_map_id(int inner_map_fd, int map_in_map_fd, uint32_t key)
{
struct bpf_map_info info = {};
uint32_t info_len = sizeof(info);
int ret, id;
ret = bpf_obj_get_info_by_fd(inner_map_fd, &info, &info_len);
assert(!ret);
ret = bpf_map_lookup_elem(map_in_map_fd, &key, &id);
assert(!ret);
assert(id == info.id);
}
static int map_subset(int map0, int map1)
{
unsigned long long next_key = 0;
unsigned long long value0[nr_cpus], value1[nr_cpus];
int ret;
while (!bpf_map_get_next_key(map1, &next_key, &next_key)) {
assert(!bpf_map_lookup_elem(map1, &next_key, value1));
ret = bpf_map_lookup_elem(map0, &next_key, value0);
if (ret) {
printf("key:%llu not found from map. %s(%d)\n",
next_key, strerror(errno), errno);
return 0;
}
if (value0[0] != value1[0]) {
printf("key:%llu value0:%llu != value1:%llu\n",
next_key, value0[0], value1[0]);
return 0;
}
}
return 1;
}
int bpf_prog1(struct __sk_buff *skb)
{
int index = load_byte(skb, ETH_HLEN + offsetof(struct iphdr, protocol));
long *value;
if (skb->pkt_type != PACKET_OUTGOING)
return 0;
value = bpf_map_lookup_elem(&my_map, &index);
if (value)
__sync_fetch_and_add(value, skb->len);
return 0;
}
int main(int argc, char **argv)
{
unsigned int nr_cpus = bpf_num_possible_cpus();
const char *map_filename = "/sys/fs/bpf/tc/globals/lwt_len_hist_map";
uint64_t values[nr_cpus], sum, max_value = 0, data[MAX_INDEX] = {};
uint64_t key = 0, next_key, max_key = 0;
char starstr[MAX_STARS];
int i, map_fd;
map_fd = bpf_obj_get(map_filename);
if (map_fd < 0) {
fprintf(stderr, "bpf_obj_get(%s): %s(%d)\n",
map_filename, strerror(errno), errno);
return -1;
}
while (bpf_map_get_next_key(map_fd, &key, &next_key) == 0) {
if (next_key >= MAX_INDEX) {
fprintf(stderr, "Key %lu out of bounds\n", next_key);
continue;
}
bpf_map_lookup_elem(map_fd, &next_key, values);
sum = 0;
for (i = 0; i < nr_cpus; i++)
sum += values[i];
data[next_key] = sum;
if (sum && next_key > max_key)
max_key = next_key;
if (sum > max_value)
max_value = sum;
key = next_key;
}
for (i = 1; i <= max_key + 1; i++) {
stars(starstr, data[i - 1], max_value, MAX_STARS);
printf("%8ld -> %-8ld : %-8ld |%-*s|\n",
(1l << i) >> 1, (1l << i) - 1, data[i - 1],
MAX_STARS, starstr);
}
close(map_fd);
return 0;
}
int bpf_prog2(struct pt_regs *ctx)
{
long rq = PT_REGS_PARM1(ctx);
u64 *value, l, base;
u32 index;
value = bpf_map_lookup_elem(&my_map, &rq);
if (!value)
return 0;
u64 cur_time = bpf_ktime_get_ns();
u64 delta = cur_time - *value;
bpf_map_delete_elem(&my_map, &rq);
/* the lines below are computing index = log10(delta)*10
* using integer arithmetic
* index = 29 ~ 1 usec
* index = 59 ~ 1 msec
* index = 89 ~ 1 sec
* index = 99 ~ 10sec or more
* log10(x)*10 = log2(x)*10/log2(10) = log2(x)*3
*/
l = log2l(delta);
base = 1ll << l;
index = (l * 64 + (delta - base) * 64 / base) * 3 / 64;
if (index >= SLOTS)
index = SLOTS - 1;
value = bpf_map_lookup_elem(&lat_map, &index);
if (value)
*value += 1;
return 0;
}
int bpf_prog2(struct pt_regs *ctx)
{
long loc = 0;
long init_val = 1;
long *value;
/* read ip of kfree_skb caller.
* non-portable version of __builtin_return_address(0)
*/
BPF_KPROBE_READ_RET_IP(loc, ctx);
value = bpf_map_lookup_elem(&my_map, &loc);
if (value)
*value += 1;
else
bpf_map_update_elem(&my_map, &loc, &init_val, BPF_ANY);
return 0;
}
static void verify_map(int map_id)
{
__u32 key = 0;
__u32 val;
if (bpf_map_lookup_elem(map_id, &key, &val) != 0) {
fprintf(stderr, "map_lookup failed: %s\n", strerror(errno));
return;
}
if (val == 0) {
fprintf(stderr, "failed: map #%d returns value 0\n", map_id);
return;
}
val = 0;
if (bpf_map_update_elem(map_id, &key, &val, BPF_ANY) != 0) {
fprintf(stderr, "map_update failed: %s\n", strerror(errno));
return;
}
}
static void print_old_objects(int fd)
{
long long val = time_get_ns();
__u64 key, next_key;
struct pair v;
key = write(1, "\e[1;1H\e[2J", 12); /* clear screen */
key = -1;
while (bpf_map_get_next_key(map_fd[0], &key, &next_key) == 0) {
bpf_map_lookup_elem(map_fd[0], &next_key, &v);
key = next_key;
if (val - v.val < 1000000000ll)
/* object was allocated more then 1 sec ago */
continue;
printf("obj 0x%llx is %2lldsec old was allocated at ip %llx\n",
next_key, (val - v.val) / 1000000000ll, v.ip);
}
}
static __u64 get_key32_value64_percpu(int fd, __u32 key)
{
/* For percpu maps, userspace gets a value per possible CPU */
unsigned int nr_cpus = bpf_num_possible_cpus();
__u64 values[nr_cpus];
__u64 sum = 0;
int i;
if ((bpf_map_lookup_elem(fd, &key, values)) != 0) {
fprintf(stderr,
"ERR: bpf_map_lookup_elem failed key:0x%X\n", key);
return 0;
}
/* Sum values from each CPU */
for (i = 0; i < nr_cpus; i++) {
sum += values[i];
}
return sum;
}
static int run_test(int server_fd, int results_fd)
{
int client = -1, srv_client = -1;
int ret = 0;
__u32 key = 0;
__u64 value = 0;
if (bpf_map_update_elem(results_fd, &key, &value, 0) < 0) {
log_err("Can't clear results");
goto err;
}
client = connect_to_server(server_fd);
if (client == -1)
goto err;
srv_client = accept(server_fd, NULL, 0);
if (srv_client == -1) {
log_err("Can't accept connection");
goto err;
}
if (bpf_map_lookup_elem(results_fd, &key, &value) < 0) {
log_err("Can't lookup result");
goto err;
}
if (value != 1) {
log_err("Didn't match syncookie: %llu", value);
goto err;
}
goto out;
err:
ret = 1;
out:
close(client);
close(srv_client);
return ret;
}
