void curve25519_alloc_or_maybe_die(struct curve25519_struct *curve)
{
curve->enc_buf_size = curve->dec_buf_size = TUNBUFF_SIZ;
curve->enc_buf = xmalloc_aligned(curve->enc_buf_size, 16);
curve->dec_buf = xmalloc_aligned(curve->dec_buf_size, 16);
spinlock_init(&curve->enc_lock);
spinlock_init(&curve->dec_lock);
}
int mbox_init(struct mbox *mb, size_t size)
{
int ret;
mb->read_idx = 0;
mb->write_idx = 0;
mb->size = size;
ret = sem_init(&mb->sem_read, 0, 0);
if (ret)
goto out_err;
ret = sem_init(&mb->sem_write, 0, size);
if (ret)
goto out_err;
ret = pthread_mutex_init(&mb->mutex_read, NULL);
if (ret)
goto out_err;
ret = pthread_mutex_init(&mb->mutex_write, NULL);
if (ret)
goto out_err;
mb->messages = xmalloc_aligned(mb->size * sizeof(*mb->messages),
sizeof(void *) * 8);
return 0;
out_err:
return -EIO;
}
int curve25519_alloc_or_maybe_die(struct curve25519_struct *c)
{
if (!c)
return -EINVAL;
c->enc_buf_size = TUNBUFF_SIZ;
c->dec_buf_size = TUNBUFF_SIZ;
c->enc_buf = xmalloc_aligned(c->enc_buf_size, 16);
c->dec_buf = xmalloc_aligned(c->dec_buf_size, 16);
spinlock_init(&c->enc_lock);
spinlock_init(&c->dec_lock);
mt_init_by_seed_rand_array();
return 0;
}
int init_pcap_sg(void)
{
unsigned long i;
c = 0;
memset(iov, 0, sizeof(iov));
for (i = 0; i < IOVSIZ; ++i) {
iov[i].iov_base = xmalloc_aligned(ALLSIZ, 64);
iov[i].iov_len = ALLSIZ;
}
spinlock_init(&lock);
return pcap_ops_group_register(&pcap_sg_ops, PCAP_OPS_SG);
}
static void *worker(void *self)
{
int fd, old_state;
ssize_t ret;
size_t blen = TUNBUFF_SIZ; //FIXME
const struct worker_struct *ws = self;
struct pollfd fds;
char *buff;
fds.fd = ws->efd[0];
fds.events = POLLIN;
ret = curve25519_alloc_or_maybe_die(ws->c);
if (ret < 0)
syslog_panic("Cannot init curve25519!\n");
buff = xmalloc_aligned(blen, 64);
syslog(LOG_INFO, "curvetun thread on CPU%u up!\n", ws->cpu);
pthread_cleanup_push(xfree_func, ws->c);
pthread_cleanup_push(curve25519_free, ws->c);
pthread_cleanup_push(xfree_func, buff);
while (likely(!sigint)) {
poll(&fds, 1, -1);
if ((fds.revents & POLLIN) != POLLIN)
continue;
pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old_state);
while ((ret = read_exact(ws->efd[0], &fd, sizeof(fd), 1)) > 0) {
if (ret != sizeof(fd)) {
sched_yield();
continue;
}
ret = ws->handler(fd, ws, buff, blen);
if (ret)
write_exact(ws->parent.refd, &fd, sizeof(fd), 1);
}
pthread_setcancelstate(old_state, NULL);
}
syslog(LOG_INFO, "curvetun thread on CPU%u down!\n", ws->cpu);
pthread_cleanup_pop(1);
pthread_cleanup_pop(1);
pthread_cleanup_pop(1);
pthread_exit((void *) ((long) ws->cpu));
}
void alloc_ring_frames_generic(struct ring *ring, int num, size_t size)
{
int i;
size_t len = num * sizeof(*ring->frames);
ring->frames = xmalloc_aligned(len, CO_CACHE_LINE_SIZE);
fmemset(ring->frames, 0, len);
for (i = 0; i < num; ++i) {
ring->frames[i].iov_len = size;
ring->frames[i].iov_base = ring->mm_space + (i * size);
}
}
CTEST(file, directio)
{
int r;
int fd;
const char *dbname = "file-test.brt";
char *datas = (char*)xmalloc_aligned(512, 1024);
memset(datas, 0, 512);
fd = ness_os_open_direct(dbname, O_CREAT | O_RDWR, 0777);
ASSERT_TRUE(fd > 0);
r = ness_os_pwrite(fd, datas, 512, 512UL);
ASSERT_EQUAL(0, r);
xfree(datas);
ness_os_close(fd);
}
static void thread_spawn_or_panic(unsigned int cpus, int efd, int refd,
int tunfd, int ipv4, int udp)
{
int i, ret;
cpu_set_t cpuset;
unsigned int threads;
threads = cpus * THREADS_PER_CPU;
for (i = 0; i < threads; ++i) {
CPU_ZERO(&cpuset);
threadpool[i].cpu = i % cpus;
CPU_SET(threadpool[i].cpu, &cpuset);
ret = pipe2(threadpool[i].efd, O_NONBLOCK);
if (ret < 0)
syslog_panic("Cannot create event socket!\n");
threadpool[i].c = xmalloc_aligned(sizeof(*threadpool[i].c), 64);
threadpool[i].parent.efd = efd;
threadpool[i].parent.refd = refd;
threadpool[i].parent.tunfd = tunfd;
threadpool[i].parent.ipv4 = ipv4;
threadpool[i].parent.udp = udp;
threadpool[i].handler = udp ? handler_udp : handler_tcp;
ret = pthread_create(&threadpool[i].trid, NULL,
worker, &threadpool[i]);
if (ret < 0)
syslog_panic("Thread creation failed!\n");
ret = pthread_setaffinity_np(threadpool[i].trid,
sizeof(cpuset), &cpuset);
if (ret < 0)
syslog_panic("Thread CPU migration failed!\n");
pthread_detach(threadpool[i].trid);
}
sleep(1);
}
static void enter_mode_read_pcap(struct mode *mode)
{
int ret, fd;
struct pcap_pkthdr phdr;
struct sock_fprog bpf_ops;
struct tx_stats stats;
struct frame_map fm;
uint8_t *out;
size_t out_len;
if (!pcap_ops[mode->pcap])
panic("pcap group not supported!\n");
fd = open_or_die(mode->device_in, O_RDONLY | O_LARGEFILE | O_NOATIME);
ret = pcap_ops[mode->pcap]->pull_file_header(fd);
if (ret)
panic("error reading pcap header!\n");
if (pcap_ops[mode->pcap]->prepare_reading_pcap) {
ret = pcap_ops[mode->pcap]->prepare_reading_pcap(fd);
if (ret)
panic("error prepare reading pcap!\n");
}
memset(&fm, 0, sizeof(fm));
memset(&bpf_ops, 0, sizeof(bpf_ops));
memset(&stats, 0, sizeof(stats));
bpf_parse_rules(mode->filter, &bpf_ops);
dissector_init_all(mode->print_mode);
out_len = device_mtu("lo");
out = xmalloc_aligned(out_len, 64);
printf("BPF:\n");
bpf_dump_all(&bpf_ops);
printf("MD: RD %s\n\n", pcap_ops[mode->pcap]->name);
while (likely(sigint == 0)) {
do {
ret = pcap_ops[mode->pcap]->read_pcap_pkt(fd, &phdr,
out, out_len);
if (unlikely(ret <= 0))
goto out;
} while (mode->filter && !bpf_run_filter(&bpf_ops, out, phdr.len));
pcap_pkthdr_to_tpacket_hdr(&phdr, &fm.tp_h);
stats.tx_bytes += fm.tp_h.tp_len;;
stats.tx_packets++;
show_frame_hdr(&fm, mode->print_mode, RING_MODE_EGRESS);
dissector_entry_point(out, fm.tp_h.tp_snaplen,
mode->link_type);
if (frame_cnt_max != 0 &&
stats.tx_packets >= frame_cnt_max) {
sigint = 1;
break;
}
}
out:
fflush(stdout);
printf("\n");
printf("\r%12lu frames outgoing\n", stats.tx_packets);
printf("\r%12lu bytes outgoing\n", stats.tx_bytes);
xfree(out);
dissector_cleanup_all();
if (pcap_ops[mode->pcap]->prepare_close_pcap)
pcap_ops[mode->pcap]->prepare_close_pcap(fd, PCAP_MODE_READ);
close(fd);
}
static void read_pcap(struct ctx *ctx)
{
__label__ out;
uint8_t *out;
int ret, fd, fdo = 0;
unsigned long trunced = 0;
size_t out_len;
pcap_pkthdr_t phdr;
struct sock_fprog bpf_ops;
struct frame_map fm;
struct timeval start, end, diff;
struct sockaddr_ll sll;
bug_on(!__pcap_io);
if (!strncmp("-", ctx->device_in, strlen("-"))) {
fd = dup_or_die(fileno(stdin));
close(fileno(stdin));
if (ctx->pcap == PCAP_OPS_MM)
ctx->pcap = PCAP_OPS_SG;
} else {
fd = open_or_die(ctx->device_in, O_RDONLY | O_LARGEFILE | O_NOATIME);
}
if (__pcap_io->init_once_pcap)
__pcap_io->init_once_pcap();
ret = __pcap_io->pull_fhdr_pcap(fd, &ctx->magic, &ctx->link_type);
if (ret)
panic("Error reading pcap header!\n");
if (__pcap_io->prepare_access_pcap) {
ret = __pcap_io->prepare_access_pcap(fd, PCAP_MODE_RD, ctx->jumbo);
if (ret)
panic("Error prepare reading pcap!\n");
}
fmemset(&fm, 0, sizeof(fm));
fmemset(&bpf_ops, 0, sizeof(bpf_ops));
bpf_parse_rules(ctx->filter, &bpf_ops, ctx->link_type);
if (ctx->dump_bpf)
bpf_dump_all(&bpf_ops);
dissector_init_all(ctx->print_mode);
out_len = round_up(1024 * 1024, PAGE_SIZE);
out = xmalloc_aligned(out_len, CO_CACHE_LINE_SIZE);
if (ctx->device_out) {
if (!strncmp("-", ctx->device_out, strlen("-"))) {
fdo = dup_or_die(fileno(stdout));
close(fileno(stdout));
} else {
fdo = open_or_die_m(ctx->device_out, O_RDWR | O_CREAT |
O_TRUNC | O_LARGEFILE, DEFFILEMODE);
}
}
drop_privileges(ctx->enforce, ctx->uid, ctx->gid);
printf("Running! Hang up with ^C!\n\n");
fflush(stdout);
bug_on(gettimeofday(&start, NULL));
while (likely(sigint == 0)) {
do {
ret = __pcap_io->read_pcap(fd, &phdr, ctx->magic,
out, out_len);
if (unlikely(ret < 0))
goto out;
if (unlikely(pcap_get_length(&phdr, ctx->magic) == 0)) {
trunced++;
continue;
}
if (unlikely(pcap_get_length(&phdr, ctx->magic) > out_len)) {
pcap_set_length(&phdr, ctx->magic, out_len);
trunced++;
}
} while (ctx->filter &&
!bpf_run_filter(&bpf_ops, out,
pcap_get_length(&phdr, ctx->magic)));
pcap_pkthdr_to_tpacket_hdr(&phdr, ctx->magic, &fm.tp_h, &sll);
ctx->tx_bytes += fm.tp_h.tp_len;
ctx->tx_packets++;
show_frame_hdr(&fm, ctx->print_mode);
dissector_entry_point(out, fm.tp_h.tp_snaplen,
ctx->link_type, ctx->print_mode);
if (ctx->device_out)
translate_pcap_to_txf(fdo, out, fm.tp_h.tp_snaplen);
if (frame_count_max != 0) {
if (ctx->tx_packets >= frame_count_max) {
sigint = 1;
break;
}
}
}
out:
bug_on(gettimeofday(&end, NULL));
timersub(&end, &start, &diff);
bpf_release(&bpf_ops);
dissector_cleanup_all();
if (__pcap_io->prepare_close_pcap)
__pcap_io->prepare_close_pcap(fd, PCAP_MODE_RD);
xfree(out);
fflush(stdout);
printf("\n");
printf("\r%12lu packets outgoing\n", ctx->tx_packets);
printf("\r%12lu packets truncated in file\n", trunced);
printf("\r%12lu bytes outgoing\n", ctx->tx_bytes);
printf("\r%12lu sec, %lu usec in total\n", diff.tv_sec, diff.tv_usec);
if (!strncmp("-", ctx->device_in, strlen("-")))
dup2(fd, fileno(stdin));
close(fd);
if (ctx->device_out) {
if (!strncmp("-", ctx->device_out, strlen("-")))
dup2(fdo, fileno(stdout));
close(fdo);
}
}
static void read_pcap(struct ctx *ctx)
{
uint8_t *out;
int ret, fd, fdo = 0;
unsigned long trunced = 0;
size_t out_len;
pcap_pkthdr_t phdr;
struct sock_fprog bpf_ops;
struct frame_map fm;
struct timeval start, end, diff;
bool is_out_pcap = ctx->device_out && strstr(ctx->device_out, ".pcap");
const struct pcap_file_ops *pcap_out_ops = pcap_ops[PCAP_OPS_RW];
bug_on(!__pcap_io);
if (!strncmp("-", ctx->device_in, strlen("-"))) {
fd = dup_or_die(fileno(stdin));
close(fileno(stdin));
if (ctx->pcap == PCAP_OPS_MM)
ctx->pcap = PCAP_OPS_SG;
} else {
/* O_NOATIME requires privileges, in case we don't have
* them, retry without them at a minor cost of updating
* atime in case the fs has been mounted as such.
*/
fd = open(ctx->device_in, O_RDONLY | O_LARGEFILE | O_NOATIME);
if (fd < 0 && errno == EPERM)
fd = open_or_die(ctx->device_in, O_RDONLY | O_LARGEFILE);
if (fd < 0)
panic("Cannot open file %s! %s.\n", ctx->device_in,
strerror(errno));
}
if (__pcap_io->init_once_pcap)
__pcap_io->init_once_pcap(false);
ret = __pcap_io->pull_fhdr_pcap(fd, &ctx->magic, &ctx->link_type);
if (ret)
panic("Error reading pcap header!\n");
if (__pcap_io->prepare_access_pcap) {
ret = __pcap_io->prepare_access_pcap(fd, PCAP_MODE_RD, ctx->jumbo);
if (ret)
panic("Error prepare reading pcap!\n");
}
fmemset(&fm, 0, sizeof(fm));
bpf_parse_rules(ctx->filter, &bpf_ops, ctx->link_type);
if (ctx->dump_bpf)
bpf_dump_all(&bpf_ops);
dissector_init_all(ctx->print_mode);
out_len = round_up(1024 * 1024, RUNTIME_PAGE_SIZE);
out = xmalloc_aligned(out_len, CO_CACHE_LINE_SIZE);
if (ctx->device_out) {
if (!strncmp("-", ctx->device_out, strlen("-"))) {
fdo = dup_or_die(fileno(stdout));
close(fileno(stdout));
} else {
fdo = open_or_die_m(ctx->device_out, O_RDWR | O_CREAT |
O_TRUNC | O_LARGEFILE, DEFFILEMODE);
}
}
if (is_out_pcap) {
ret = pcap_out_ops->push_fhdr_pcap(fdo, ctx->magic,
ctx->link_type);
if (ret)
panic("Error writing pcap header!\n");
}
drop_privileges(ctx->enforce, ctx->uid, ctx->gid);
printf("Running! Hang up with ^C!\n\n");
fflush(stdout);
bug_on(gettimeofday(&start, NULL));
while (likely(sigint == 0)) {
do {
ret = __pcap_io->read_pcap(fd, &phdr, ctx->magic,
out, out_len);
if (unlikely(ret < 0))
goto out;
if (unlikely(pcap_get_length(&phdr, ctx->magic) == 0)) {
trunced++;
continue;
}
if (unlikely(pcap_get_length(&phdr, ctx->magic) > out_len)) {
pcap_set_length(&phdr, ctx->magic, out_len);
trunced++;
}
} while (ctx->filter &&
!bpf_run_filter(&bpf_ops, out,
pcap_get_length(&phdr, ctx->magic)));
pcap_pkthdr_to_tpacket_hdr(&phdr, ctx->magic, &fm.tp_h, &fm.s_ll);
ctx->tx_bytes += fm.tp_h.tp_len;
ctx->tx_packets++;
show_frame_hdr(out, fm.tp_h.tp_snaplen, ctx->link_type, &fm,
ctx->print_mode, ctx->tx_packets);
dissector_entry_point(out, fm.tp_h.tp_snaplen,
ctx->link_type, ctx->print_mode,
&fm.s_ll);
if (is_out_pcap) {
size_t pcap_len = pcap_get_length(&phdr, ctx->magic);
int wlen = pcap_out_ops->write_pcap(fdo, &phdr,
ctx->magic, out,
pcap_len);
if (unlikely(wlen != (int)pcap_get_total_length(&phdr, ctx->magic)))
panic("Error writing to pcap!\n");
} else if (ctx->device_out) {
translate_pcap_to_txf(fdo, out, fm.tp_h.tp_snaplen);
}
if (frame_count_max != 0) {
if (ctx->tx_packets >= frame_count_max) {
sigint = 1;
break;
}
}
}
out:
bug_on(gettimeofday(&end, NULL));
timersub(&end, &start, &diff);
bpf_release(&bpf_ops);
dissector_cleanup_all();
if (__pcap_io->prepare_close_pcap)
__pcap_io->prepare_close_pcap(fd, PCAP_MODE_RD);
xfree(out);
fflush(stdout);
printf("\n");
printf("\r%12lu packets outgoing\n", ctx->tx_packets);
printf("\r%12lu packets truncated in file\n", trunced);
printf("\r%12lu bytes outgoing\n", ctx->tx_bytes);
printf("\r%12lu sec, %lu usec in total\n", diff.tv_sec, diff.tv_usec);
if (!strncmp("-", ctx->device_in, strlen("-")))
dup2(fd, fileno(stdin));
close(fd);
if (ctx->device_out) {
if (!strncmp("-", ctx->device_out, strlen("-")))
dup2(fdo, fileno(stdout));
close(fdo);
}
}
static void enter_mode_read_pcap(struct mode *mode)
{
int ret, fd, fdo = 0;
struct pcap_pkthdr phdr;
struct sock_fprog bpf_ops;
struct tx_stats stats;
struct frame_map fm;
uint8_t *out;
size_t out_len;
unsigned long trunced = 0;
struct timeval start, end, diff;
if (!pcap_ops[mode->pcap])
panic("pcap group not supported!\n");
fd = open_or_die(mode->device_in, O_RDONLY | O_LARGEFILE | O_NOATIME);
ret = pcap_ops[mode->pcap]->pull_file_header(fd, &mode->link_type);
if (ret)
panic("error reading pcap header!\n");
if (pcap_ops[mode->pcap]->prepare_reading_pcap) {
ret = pcap_ops[mode->pcap]->prepare_reading_pcap(fd);
if (ret)
panic("error prepare reading pcap!\n");
}
fmemset(&fm, 0, sizeof(fm));
fmemset(&bpf_ops, 0, sizeof(bpf_ops));
fmemset(&stats, 0, sizeof(stats));
bpf_parse_rules(mode->filter, &bpf_ops);
dissector_init_all(mode->print_mode);
out_len = 64 * 1024;
out = xmalloc_aligned(out_len, CO_CACHE_LINE_SIZE);
printf("BPF:\n");
bpf_dump_all(&bpf_ops);
printf("MD: RD %s ", pcap_ops[mode->pcap]->name);
#ifdef _LARGEFILE64_SOURCE
printf("lf64 ");
#endif
ioprio_print();
printf("\n");
if (mode->device_out) {
fdo = open_or_die_m(mode->device_out, O_RDWR | O_CREAT |
O_TRUNC | O_LARGEFILE, S_IRUSR | S_IWUSR);
}
gettimeofday(&start, NULL);
while (likely(sigint == 0)) {
do {
memset(&phdr, 0, sizeof(phdr));
ret = pcap_ops[mode->pcap]->read_pcap_pkt(fd, &phdr,
out, out_len);
if (unlikely(ret < 0))
goto out;
if (unlikely(phdr.len == 0)) {
trunced++;
continue;
}
if (unlikely(phdr.len > out_len)) {
phdr.len = out_len;
trunced++;
}
} while (mode->filter &&
!bpf_run_filter(&bpf_ops, out, phdr.len));
pcap_pkthdr_to_tpacket_hdr(&phdr, &fm.tp_h);
stats.tx_bytes += fm.tp_h.tp_len;
stats.tx_packets++;
show_frame_hdr(&fm, mode->print_mode, RING_MODE_EGRESS);
dissector_entry_point(out, fm.tp_h.tp_snaplen,
mode->link_type, mode->print_mode);
if (mode->device_out) {
int i = 0;
char bout[80];
slprintf(bout, sizeof(bout), "{\n ");
write_or_die(fdo, bout, strlen(bout));
while (i < fm.tp_h.tp_snaplen) {
slprintf(bout, sizeof(bout), "0x%02x, ", out[i]);
write_or_die(fdo, bout, strlen(bout));
i++;
if (i % 10 == 0) {
slprintf(bout, sizeof(bout), "\n", out[i]);
write_or_die(fdo, bout, strlen(bout));
if (i < fm.tp_h.tp_snaplen) {
slprintf(bout, sizeof(bout), " ", out[i]);
write_or_die(fdo, bout, strlen(bout));
}
}
}
if (i % 10 != 0) {
slprintf(bout, sizeof(bout), "\n");
write_or_die(fdo, bout, strlen(bout));
}
slprintf(bout, sizeof(bout), "}\n\n");
write_or_die(fdo, bout, strlen(bout));
}
if (frame_cnt_max != 0 &&
stats.tx_packets >= frame_cnt_max) {
sigint = 1;
break;
}
}
out:
gettimeofday(&end, NULL);
diff = tv_subtract(end, start);
fflush(stdout);
printf("\n");
printf("\r%12lu frames outgoing\n", stats.tx_packets);
printf("\r%12lu frames truncated (larger than mtu)\n", trunced);
printf("\r%12lu bytes outgoing\n", stats.tx_bytes);
printf("\r%12lu sec, %lu usec in total\n", diff.tv_sec, diff.tv_usec);
xfree(out);
bpf_release(&bpf_ops);
dissector_cleanup_all();
if (pcap_ops[mode->pcap]->prepare_close_pcap)
pcap_ops[mode->pcap]->prepare_close_pcap(fd, PCAP_MODE_READ);
close(fd);
if (mode->device_out)
close(fdo);
}
__hidden void *xzmalloc_aligned(size_t size, size_t alignment)
{
void *ptr = xmalloc_aligned(size, alignment);
memset(ptr, 0, size);
return ptr;
}