void reserve_msr(int msr_id, uint64_t evt, int cpu_filter) {
int i;
int per_node = is_per_node(evt);
for(i = 0; i < ncpus; i++) {
if(cpu_filter == -1 || cpu_filter == i
|| (per_node && (numa_node_of_cpu(cpu_filter) == numa_node_of_cpu(i))))
available_msr_usage[msr_id][i] = 1;
}
}
int is_reserved(int msr_id, uint64_t evt, int cpu_filter) {
int i;
int per_node = is_per_node(evt);
for(i = 0; i < ncpus; i++) {
if(available_msr_usage[msr_id][i] // msr has been configured on cpu i
&& ((cpu_filter == -1) // and we want to use it on all cpu
|| (cpu_filter == i) // or on this cpu
|| (per_node && (numa_node_of_cpu(cpu_filter) == numa_node_of_cpu(i))))) // or on the same node (and it is a problem)
return 1;
}
return 0;
}
/**
* @brief construct global 'cohort' lock.
*
* This lock performs handovers in three levels: First within
* the same NUMA node, then within the same ArgoDSM node, and
* finally over ArgoDSM nodes.
*/
cohort_lock() :
has_global_lock(false),
numanodes(1), // sane default
numahandover(0),
nodelockowner(NO_OWNER),
tas_flag(argo::conew_<bool>(false)),
global_lock(new argo::globallock::global_tas_lock(tas_flag)),
node_lock(new argo::locallock::ticket_lock())
{
int num_cpus = sysconf(_SC_NPROCESSORS_CONF); // sane default
numa_mapping.resize(num_cpus, 0);
#ifdef ARGO_USE_LIBNUMA
/* use libnuma only if it is actually available */
if(numa_available() != -1) {
numanodes = numa_num_configured_nodes();
/* Initialize the NUMA map */
for (int i = 0; i < num_cpus; ++i) {
numa_mapping[i] = numa_node_of_cpu(i);
}
}
#endif
/* initialize hierarchy components */
handovers = new int[numanodes]();
local_lock = new argo::locallock::mcs_lock[numanodes];
}
int pthread_create(pthread_t *thread, const pthread_attr_t *attr, void *(*start_routine) (void *), void *arg) {
int core;
int ret;
cpu_set_t mask;
CPU_ZERO(&mask);
ret = old_pthread_create(thread, attr, start_routine, arg);
if(!get_shm()->active)
return ret;
core = get_next_core();
if(!get_shm()->per_node) {
CPU_SET(core, &mask);
} else {
int i, node = numa_node_of_cpu(core);
struct bitmask * bmp = numa_allocate_cpumask();
numa_node_to_cpus(node, bmp);
for(i = 0; i < numa_num_configured_cpus(); i++) {
if(numa_bitmask_isbitset(bmp, i))
CPU_SET(i, &mask);
}
numa_free_cpumask(bmp);
}
old_pthread_setaffinity_np(*thread, sizeof(mask), &mask);
VERBOSE("-> Set affinity to %d\n", core);
return ret;
}
static int set_closest_numanode(int num_unique,
const struct bandwidth_nodes_t *bandwidth_nodes,
int target_bandwidth, int num_cpunode, int *closest_numanode)
{
/***************************************************************************
* num_unique (IN): *
* Length of bandwidth_nodes vector. *
* bandwidth_nodes (IN): *
* Output vector from create_bandwitdth_nodes(). *
* target_bandwidth (IN): *
* The bandwidth to select for comparison. *
* num_cpunode (IN): *
* Number of cpu's and length of closest_numanode. *
* closest_numanode (OUT): *
* Vector that maps cpu index to closest numa node of the specified *
* bandwidth. *
* RETURNS zero on success, error code on failure *
***************************************************************************/
int err = 0;
int min_distance, distance, i, j, old_errno, min_unique;
struct bandwidth_nodes_t match;
match.bandwidth = -1;
for (i = 0; i < num_cpunode; ++i) {
closest_numanode[i] = -1;
}
for (i = 0; i < num_unique; ++i) {
if (bandwidth_nodes[i].bandwidth == target_bandwidth) {
match = bandwidth_nodes[i];
break;
}
}
if (match.bandwidth == -1) {
err = MEMKIND_ERROR_UNAVAILABLE;
}
else {
for (i = 0; i < num_cpunode; ++i) {
min_distance = INT_MAX;
min_unique = 1;
for (j = 0; j < match.num_numanodes; ++j) {
old_errno = errno;
distance = numa_distance(numa_node_of_cpu(i),
match.numanodes[j]);
errno = old_errno;
if (distance < min_distance) {
min_distance = distance;
closest_numanode[i] = match.numanodes[j];
min_unique = 1;
}
else if (distance == min_distance) {
min_unique = 0;
}
}
if (!min_unique) {
err = MEMKIND_ERROR_RUNTIME;
}
}
}
return err;
}
static dynarray_t *do_create(unsigned long elem_size,
unsigned long alloc_grain,
unsigned long elems_nr,
int numa)
{
struct dynarray *da;
int node = 0;
if (numa) {
int cpu = sched_getcpu();
/* Numa-aware allocation */
if (cpu < 0) {
perror("dynarray_create: sched_getcpu");
exit(1);
}
node = numa_node_of_cpu(cpu);
if (node < 0) {
perror("dynarray_create: numa_node_of_cpu");
exit(1);
}
da = numa_alloc_onnode(sizeof(*da), node);
} else {
da = malloc(sizeof(*da));
}
if ( !da ) {
fprintf(stderr, "dynarray_create: malloc\n");
exit(1);
}
da->numa = numa;
da->next_idx = 0;
da->elem_size = elem_size;
if (elems_nr <= alloc_grain) {
da->elems_nr = alloc_grain;
} else {
unsigned long rem = elems_nr % alloc_grain;
da->elems_nr = elems_nr;
if (rem)
da->elems_nr += alloc_grain - rem;
}
da->alloc_grain = alloc_grain;
if (numa) {
da->elems = numa_alloc_onnode(elem_size*da->elems_nr, node);
} else {
da->elems = malloc(elem_size*da->elems_nr);
}
if ( !da->elems ){
fprintf(stderr, "dynarray_create: malloc\n");
exit(1);
}
return da;
}
std::vector<int> InputParserUtil::GetNUMANodesForCPUs() {
std::vector<int> numa_nodes_of_cpus;
#ifdef QUICKSTEP_HAVE_LIBNUMA
const int num_cpus = numa_num_configured_cpus();
numa_nodes_of_cpus.reserve(num_cpus);
for (int curr_cpu = 0; curr_cpu < num_cpus; ++curr_cpu) {
numa_nodes_of_cpus.push_back(numa_node_of_cpu(curr_cpu));
}
#endif
return numa_nodes_of_cpus;
}
int bind2node(int core_id) {
char node_str[8];
if (core_id < 0 || numa_available() == -1)
return -1;
snprintf(node_str, sizeof(node_str), "%u", numa_node_of_cpu(core_id));
numa_bind(numa_parse_nodestring(node_str));
return 0;
}
NumaInfo::info_t::info_t()
{
#pragma omp parallel
{
int tid = omp_get_thread_num();
int nid = numa_node_of_cpu(tid);
#pragma omp critical
{
numa_id[tid] = nid;
ord[tid] = info[nid].size();
info[nid][tid] = info[nid].size();
//printf("NumaInfo::Init thread id %d at numa %d ord = %d\n", tid, nid, info[nid][tid]);
}
}
}
static void set_affinity(pid_t tid, int cpu_id) {
if(!get_shm()->active)
return;
if(!get_shm()->per_node) {
cpu_set_t mask;
CPU_ZERO(&mask);
CPU_SET(cpu_id, &mask);
VERBOSE("--> Setting tid %d on core %d\n", tid, cpu_id);
int r = old_sched_setaffinity(tid, sizeof(mask), &mask);
if (r < 0) {
fprintf(stderr, "couldn't set affinity on %d\n", cpu_id);
exit(1);
}
} else {
int r = numa_run_on_node(numa_node_of_cpu(cpu_id));
if(r < 0) {
fprintf(stderr, "couldn't set affinity on node of cpu %d\n", cpu_id);
exit(1);
}
}
}
CsxSymMatrix<IndexType, ValueType> * CsxManager<IndexType, ValueType>::
MakeCsxSym()
{
CsxSymMatrix<IndexType, ValueType> *csx;
vector<ValueType> *diagonal = spm_sym_->GetDiagonal();
IndexType diagonal_size = spm_sym_->GetDiagonalSize();
#if SPX_USE_NUMA
NumaAllocator &numa_alloc = NumaAllocator::GetInstance();
#endif
spm_ = spm_sym_->GetLowerMatrix();
#if SPX_USE_NUMA
int cpu = sched_getcpu();
if (cpu < 0) {
LOG_ERROR << "sched_getcpu() failed " << strerror(errno);
exit(1);
}
int node = numa_node_of_cpu(cpu);
if (node < 0) {
LOG_ERROR << "numa_node_of_cpu() failed " << strerror(errno);
exit(1);
}
csx = new (numa_alloc, node) CsxSymMatrix<IndexType, ValueType>;
csx->dvalues = new (numa_alloc, node) ValueType[diagonal_size];
#else
csx = new CsxSymMatrix<IndexType, ValueType>;
csx->dvalues = new ValueType[diagonal_size];
#endif
for (IndexType i = 0; i < diagonal_size; i++)
csx->dvalues[i] = diagonal->operator[](i);
csx->lower_matrix = MakeCsx(true);
return csx;
}
int main(int argc, char* argv[]) {
char *device1 = NULL, *device2 = NULL, *bind_mask = NULL, c;
int cluster_id = -1;
u_int numCPU = sysconf( _SC_NPROCESSORS_ONLN );
dir[0].bind_core = dir[1].bind_core = -1;
startTime.tv_sec = 0;
while((c = getopt(argc,argv,"abc:g:hi:o:fv")) != '?') {
if((c == 255) || (c == -1)) break;
switch(c) {
case 'h':
printHelp();
break;
case 'a':
wait_for_packet = 0;
break;
case 'f':
flush_packet = 1;
break;
case 'v':
verbose = 1;
break;
case 'b':
bidirectional = 1;
break;
case 'c':
cluster_id = atoi(optarg);
break;
case 'i':
device1 = strdup(optarg);
break;
case 'o':
device2 = strdup(optarg);
break;
case 'g':
bind_mask = strdup(optarg);
break;
}
}
if (device1 == NULL) printHelp();
if (device2 == NULL) printHelp();
if (cluster_id < 0) printHelp();
if(bind_mask != NULL) {
char *id;
if ((id = strtok(bind_mask, ":")) != NULL)
dir[0].bind_core = atoi(id) % numCPU;
if ((id = strtok(NULL, ":")) != NULL)
dir[1].bind_core = atoi(id) % numCPU;
}
zc = pfring_zc_create_cluster(
cluster_id,
1536,
0,
(2 * MAX_CARD_SLOTS) + 1 + bidirectional,
numa_node_of_cpu(dir[0].bind_core),
NULL /* auto hugetlb mountpoint */
);
if(zc == NULL) {
fprintf(stderr, "pfring_zc_create_cluster error [%s] Please check your hugetlb configuration\n",
strerror(errno));
return -1;
}
if (init_direction(0, device1, device2) < 0)
return -1;
if (bidirectional)
if (init_direction(1, device2, device1) < 0)
return -1;
signal(SIGINT, sigproc);
signal(SIGTERM, sigproc);
signal(SIGINT, sigproc);
if (!verbose) { /* periodic stats */
signal(SIGALRM, my_sigalarm);
alarm(ALARM_SLEEP);
}
pthread_create(&dir[0].thread, NULL, packet_consumer_thread, (void *) &dir[0]);
if (bidirectional) pthread_create(&dir[1].thread, NULL, packet_consumer_thread, (void *) &dir[1]);
pthread_join(dir[0].thread, NULL);
if (bidirectional) pthread_join(dir[1].thread, NULL);
sleep(1);
pfring_zc_destroy_cluster(zc);
return 0;
}
ThreadPool* ThreadPool::allocThreadPools(x265_param* p, int& numPools)
{
enum { MAX_NODE_NUM = 127 };
int cpusPerNode[MAX_NODE_NUM + 1];
memset(cpusPerNode, 0, sizeof(cpusPerNode));
int numNumaNodes = X265_MIN(getNumaNodeCount(), MAX_NODE_NUM);
int cpuCount = getCpuCount();
bool bNumaSupport = false;
#if defined(_WIN32_WINNT) && _WIN32_WINNT >= _WIN32_WINNT_WIN7
bNumaSupport = true;
#elif HAVE_LIBNUMA
bNumaSupport = numa_available() >= 0;
#endif
for (int i = 0; i < cpuCount; i++)
{
#if defined(_WIN32_WINNT) && _WIN32_WINNT >= _WIN32_WINNT_WIN7
UCHAR node;
if (GetNumaProcessorNode((UCHAR)i, &node))
cpusPerNode[X265_MIN(node, (UCHAR)MAX_NODE_NUM)]++;
else
#elif HAVE_LIBNUMA
if (bNumaSupport >= 0)
cpusPerNode[X265_MIN(numa_node_of_cpu(i), MAX_NODE_NUM)]++;
else
#endif
cpusPerNode[0]++;
}
if (bNumaSupport && p->logLevel >= X265_LOG_DEBUG)
for (int i = 0; i < numNumaNodes; i++)
x265_log(p, X265_LOG_DEBUG, "detected NUMA node %d with %d logical cores\n", i, cpusPerNode[i]);
/* limit nodes based on param->numaPools */
if (p->numaPools && *p->numaPools)
{
const char *nodeStr = p->numaPools;
for (int i = 0; i < numNumaNodes; i++)
{
if (!*nodeStr)
{
cpusPerNode[i] = 0;
continue;
}
else if (*nodeStr == '-')
cpusPerNode[i] = 0;
else if (*nodeStr == '*')
break;
else if (*nodeStr == '+')
;
else
{
int count = atoi(nodeStr);
cpusPerNode[i] = X265_MIN(count, cpusPerNode[i]);
}
/* consume current node string, comma, and white-space */
while (*nodeStr && *nodeStr != ',')
++nodeStr;
if (*nodeStr == ',' || *nodeStr == ' ')
++nodeStr;
}
}
// In the case that numa is disabled and we have more CPUs than 64,
// spawn the last pool only if the # threads in that pool is > 1/2 max (heuristic)
if ((numNumaNodes == 1) && (cpusPerNode[0] % MAX_POOL_THREADS < (MAX_POOL_THREADS / 2)))
{
cpusPerNode[0] -= (cpusPerNode[0] % MAX_POOL_THREADS);
x265_log(p, X265_LOG_DEBUG, "Creating only %d worker threads to prevent asymmetry in pools; may not use all HW contexts\n", cpusPerNode[0]);
}
numPools = 0;
for (int i = 0; i < numNumaNodes; i++)
{
if (bNumaSupport)
x265_log(p, X265_LOG_DEBUG, "NUMA node %d may use %d logical cores\n", i, cpusPerNode[i]);
if (cpusPerNode[i])
numPools += (cpusPerNode[i] + MAX_POOL_THREADS - 1) / MAX_POOL_THREADS;
}
if (!numPools)
return NULL;
if (numPools > p->frameNumThreads)
{
x265_log(p, X265_LOG_DEBUG, "Reducing number of thread pools for frame thread count\n");
numPools = X265_MAX(p->frameNumThreads / 2, 1);
}
ThreadPool *pools = new ThreadPool[numPools];
if (pools)
{
int maxProviders = (p->frameNumThreads + numPools - 1) / numPools + 1; /* +1 is Lookahead, always assigned to threadpool 0 */
int node = 0;
for (int i = 0; i < numPools; i++)
{
while (!cpusPerNode[node])
node++;
int cores = X265_MIN(MAX_POOL_THREADS, cpusPerNode[node]);
if (!pools[i].create(cores, maxProviders, node))
{
X265_FREE(pools);
numPools = 0;
return NULL;
}
if (numNumaNodes > 1)
x265_log(p, X265_LOG_INFO, "Thread pool %d using %d threads on NUMA node %d\n", i, cores, node);
else
x265_log(p, X265_LOG_INFO, "Thread pool created using %d threads\n", cores);
cpusPerNode[node] -= cores;
}
}
else
numPools = 0;
return pools;
}
int main(int argc, char* argv[]) {
char *device = NULL, c;
long i;
int cluster_id = -1;
char *bind_mask = NULL;
pthread_t *threads;
char *id;
u_int numCPU = sysconf( _SC_NPROCESSORS_ONLN );
startTime.tv_sec = 0;
while((c = getopt(argc,argv,"ac:g:hi:f")) != '?') {
if((c == 255) || (c == -1)) break;
switch(c) {
case 'h':
printHelp();
break;
case 'a':
wait_for_packet = 0;
break;
case 'c':
cluster_id = atoi(optarg);
break;
case 'i':
device = strdup(optarg);
break;
case 'g':
bind_mask = strdup(optarg);
break;
case 'f':
flush_packet = 1;
break;
}
}
if (device == NULL) printHelp();
if (cluster_id < 0) printHelp();
if (bind_mask == NULL) printHelp();
id = strtok(bind_mask, ":");
while(id != NULL) {
bind_core = realloc(bind_core, sizeof(int) * (num_threads+1));
bind_core[num_threads] = atoi(id) % numCPU;
num_threads++;
id = strtok(NULL, ":");
}
if (num_threads < 1) printHelp();
threads = calloc(num_threads, sizeof(pthread_t));
buffers = calloc(num_threads, sizeof(pfring_zc_pkt_buff *));
zq = calloc(num_threads - 1, sizeof(pfring_zc_queue *));
zc = pfring_zc_create_cluster(
cluster_id,
1536,
#ifdef METADATA_TEST
8,
#else
0,
#endif
MAX_CARD_SLOTS + (num_threads - 1) * QUEUE_LEN + num_threads,
numa_node_of_cpu(bind_core[0]),
NULL /* auto hugetlb mountpoint */
);
if(zc == NULL) {
fprintf(stderr, "pfring_zc_create_cluster error [%s] Please check your hugetlb configuration\n",
strerror(errno));
return -1;
}
for (i = 0; i < num_threads; i++) {
buffers[i] = pfring_zc_get_packet_handle(zc);
if (buffers[i] == NULL) {
fprintf(stderr, "pfring_zc_get_packet_handle error\n");
return -1;
}
}
zq[0] = pfring_zc_open_device(zc, device, rx_only, 0);
if(zq[0] == NULL) {
fprintf(stderr, "pfring_zc_open_device error [%s] Please check that %s is up and not already used\n",
strerror(errno), device);
return -1;
}
for (i = 1; i < num_threads; i++) {
zq[i] = pfring_zc_create_queue(zc, QUEUE_LEN);
if(zq[i] == NULL) {
fprintf(stderr, "pfring_zc_create_queue error [%s]\n", strerror(errno));
return -1;
}
}
signal(SIGINT, sigproc);
signal(SIGTERM, sigproc);
signal(SIGINT, sigproc);
signal(SIGALRM, my_sigalarm);
alarm(ALARM_SLEEP);
printf("Starting pipeline with %d stages..\n", num_threads);
for (i = 0; i < num_threads; i++)
pthread_create(&threads[i], NULL, pipeline_stage_thread, (void*) i);
for (i = 0; i < num_threads; i++)
pthread_join(threads[i], NULL);
sleep(1);
pfring_zc_destroy_cluster(zc);
return 0;
}
int main(int argc, char* argv[])
{
char *dist_hca = NULL, *policy = NULL, *pch;
cpu_set_t cpuset;
int i, rc, my_rank,
numcpus, size;
int numa = -1, next, numa_node;
int num_numa_cores;
numcpus = get_cores_number();
if (numcpus < 0) {
fprintf(stderr, "\nrank = %d: Bad CPUs number. Skip.\n", my_rank);
fflush(stderr);
return 1;
}
rc = MPI_Init(&argc, &argv);
if (MPI_SUCCESS != rc) {
printf ("\nrank - %d: Error starting MPI program. Skip.\n", my_rank);
MPI_Abort(MPI_COMM_WORLD, rc);
}
MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
num_numa_cores = get_numa_cores_number();
if (size > num_numa_cores) {
fprintf(stderr, "\nrank - %d: number of processes exceeds number of cores at a single numa node. Test won't get correct results in this case: num_procs = %d, num_cores = %d. Skip.\n", my_rank, size, num_numa_cores);
fflush(stderr);
MPI_Finalize();
return 1;
}
policy = getenv("OMPI_MCA_rmaps_base_mapping_policy");
dist_hca = getenv("OMPI_MCA_rmaps_dist_device");
if (NULL != dist_hca) {
dist_hca = strdup(dist_hca);
if (NULL != (pch = strchr(dist_hca, ':'))) {
*pch = '\0';
}
} else if (NULL != policy) {
dist_hca = strstr(policy, "dist:");
dist_hca += strlen("dist:");
dist_hca = strdup(dist_hca);
if (NULL != (pch = strchr(dist_hca, ','))) {
*pch = '\0';
}
}
if (NULL == policy || NULL == dist_hca) {
fprintf(stderr, "\nrank - %d: the \"dist\" mapping policy was not specified. Skip.\n", my_rank);
fflush(stderr);
MPI_Finalize();
return 1;
}
numa_node = get_closest_numa(dist_hca);
if (-1 == numa_node) {
fprintf(stderr, "\nrank - %d: info about locality to %s isn't provided by the BIOS. Skip.\n", my_rank, dist_hca);
fflush(stderr);
MPI_Finalize();
free(dist_hca);
return 1;
}
free(dist_hca);
CPU_ZERO(&cpuset);
if (sched_getaffinity(0, sizeof(cpuset), &cpuset) < 0) {
fprintf(stderr, "\nrank - %d: sched_getaffinity failed, errno says %s. Skip.\n", my_rank, strerror(errno));
fflush(stderr);
MPI_Finalize();
return 1;
}
for (i = 0; i < numcpus; ++i) {
if (CPU_ISSET(i, &cpuset)) {
next = numa_node_of_cpu(i);
if (-1 != numa && next != numa) {
fprintf(stderr, "\nError rank - %d: scheduled on more than one numa node.\n", my_rank);
fflush(stderr);
MPI_Finalize();
return 1;
}
numa = next;
}
}
if (numa_node != numa) {
fprintf(stderr, "\nError rank - %d: scheduled on wrong NUMA node - %d, should be %d\n", my_rank, numa, numa_node);
fflush(stderr);
MPI_Finalize();
return 1;
}
fprintf(stderr, "\nSuccess rank - %d: only one NUMA is scheduled.\n", my_rank);
fflush(stderr);
MPI_Finalize();
return 0;
}
void *SPM_CRS_MT_NAME(_numa_init_mmf)(char *mmf_file, uint64_t *rows_nr,
uint64_t *cols_nr, uint64_t *nz_nr,
void *metadata)
{
spm_mt_t *spm_mt = SPM_CRS_MT_NAME(_init_mmf)(mmf_file, rows_nr, cols_nr,
nz_nr, metadata);
int nr_threads = spm_mt->nr_threads;
size_t *values_parts = malloc(nr_threads*sizeof(*values_parts));
size_t *rowptr_parts = malloc(nr_threads*sizeof(*rowptr_parts));
size_t *colind_parts = malloc(nr_threads*sizeof(*colind_parts));
int *nodes = malloc(nr_threads*sizeof(*nodes));
// Reallocate data structures in a numa-aware fashion.
int i;
SPM_CRS_TYPE *crs = NULL;
for (i = 0; i < nr_threads; i++) {
spm_mt_thread_t *spm_thread = spm_mt->spm_threads + i;
SPM_CRS_MT_TYPE *crs_mt = (SPM_CRS_MT_TYPE *) spm_thread->spm;
crs = crs_mt->crs;
SPM_CRS_IDX_TYPE row_start = crs_mt->row_start;
SPM_CRS_IDX_TYPE row_end = crs_mt->row_end;
SPM_CRS_IDX_TYPE vstart = crs->row_ptr[crs_mt->row_start];
SPM_CRS_IDX_TYPE vend = crs->row_ptr[crs_mt->row_end];
rowptr_parts[i] = (row_end-row_start)*sizeof(*crs->row_ptr);
colind_parts[i] = (vend-vstart)*sizeof(*crs->col_ind);
values_parts[i] = (vend-vstart)*sizeof(*crs->values);
nodes[i] = numa_node_of_cpu(spm_thread->cpu);
spm_thread->node = nodes[i];
spm_thread->row_start = row_start;
spm_thread->nr_rows = row_end - row_start;
}
rowptr_parts[nr_threads-1] += sizeof(*crs->row_ptr);
// Sanity check.
assert(crs);
SPM_CRS_IDX_TYPE *new_rowptr =
alloc_interleaved((crs->nrows+1)*sizeof(*crs->row_ptr), rowptr_parts,
nr_threads, nodes);
SPM_CRS_IDX_TYPE *new_colind =
alloc_interleaved(crs->nz*sizeof(*crs->col_ind), colind_parts,
nr_threads, nodes);
ELEM_TYPE *new_values =
alloc_interleaved(crs->nz*sizeof(*crs->values), values_parts,
nr_threads, nodes);
// Copy the old data to the new one.
memcpy(new_rowptr, crs->row_ptr, (crs->nrows+1)*sizeof(*crs->row_ptr));
memcpy(new_colind, crs->col_ind, crs->nz*sizeof(*crs->col_ind));
memcpy(new_values, crs->values, crs->nz*sizeof(*crs->values));
// Free old data and replace with the new one.
free(crs->row_ptr);
free(crs->col_ind);
free(crs->values);
crs->row_ptr = new_rowptr;
crs->col_ind = new_colind;
crs->values = new_values;
// Check for allocation errors.
int alloc_err;
alloc_err = check_interleaved((void *) crs->row_ptr, rowptr_parts,
nr_threads, nodes);
print_alloc_status("CSR rowptr", alloc_err);
alloc_err = check_interleaved((void *) crs->col_ind, colind_parts,
nr_threads, nodes);
print_alloc_status("CSR colind", alloc_err);
alloc_err = check_interleaved((void *) crs->values, values_parts,
nr_threads, nodes);
print_alloc_status("CSR values", alloc_err);
// Free the auxiliaries.
free(rowptr_parts);
free(colind_parts);
free(values_parts);
free(nodes);
return spm_mt;
}
int main(int argc, char* argv[]) {
char *device = NULL, *dev, c;
long i;
int cluster_id = -1;
char *bind_mask = NULL;
pthread_t *threads;
char *id;
u_int numCPU = sysconf( _SC_NPROCESSORS_ONLN );
int hash_mode = 0;
startTime.tv_sec = 0;
while((c = getopt(argc,argv,"ac:g:hi:r:m:")) != '?') {
if((c == 255) || (c == -1)) break;
switch(c) {
case 'h':
printHelp();
break;
case 'a':
wait_for_packet = 0;
break;
case 'c':
cluster_id = atoi(optarg);
break;
case 'm':
hash_mode = atoi(optarg);
break;
case 'i':
device = strdup(optarg);
break;
case 'g':
bind_mask = strdup(optarg);
break;
case 'r':
bind_worker_core = atoi(optarg);
break;
}
}
if (device == NULL) printHelp();
if (cluster_id < 0) printHelp();
if (bind_mask == NULL) printHelp();
id = strtok(bind_mask, ":");
while(id != NULL) {
bind_core = realloc(bind_core, sizeof(int) * (num_threads+1));
bind_core[num_threads] = atoi(id) % numCPU;
num_threads++;
id = strtok(NULL, ":");
}
if (num_threads < 1) printHelp();
dev = strtok(device, ",");
while(dev != NULL) {
devices = realloc(devices, sizeof(char *) * (num_devices+1));
devices[num_devices] = strdup(dev);
num_devices++;
dev = strtok(NULL, ",");
}
zc = pfring_zc_create_cluster(
cluster_id,
max_packet_len(devices[0]),
0,
(num_devices * MAX_CARD_SLOTS) + (num_threads * QUEUE_LEN) + num_threads + PREFETCH_BUFFERS,
numa_node_of_cpu(bind_worker_core),
NULL /* auto hugetlb mountpoint */
);
if(zc == NULL) {
fprintf(stderr, "pfring_zc_create_cluster error [%s] Please check your hugetlb configuration\n",
strerror(errno));
return -1;
}
threads = calloc(num_threads, sizeof(pthread_t));
buffers = calloc(num_threads, sizeof(pfring_zc_pkt_buff *));
inzq = calloc(num_devices, sizeof(pfring_zc_queue *));
outzq = calloc(num_threads, sizeof(pfring_zc_queue *));
consumers_stats = calloc(num_threads, sizeof(struct stats));
for (i = 0; i < num_threads; i++) {
buffers[i] = pfring_zc_get_packet_handle(zc);
if (buffers[i] == NULL) {
fprintf(stderr, "pfring_zc_get_packet_handle error\n");
return -1;
}
}
for (i = 0; i < num_devices; i++) {
inzq[i] = pfring_zc_open_device(zc, devices[i], rx_only, 0);
if(inzq[i] == NULL) {
fprintf(stderr, "pfring_zc_open_device error [%s] Please check that %s is up and not already used\n",
strerror(errno), devices[i]);
return -1;
}
//printf("Created device with ID=%u, INDEX=%u\n", pfring_zc_get_queue_id(inzq[i]), QUEUEID_TO_IFINDEX(pfring_zc_get_queue_id(inzq[i])));
}
for (i = 0; i < num_threads; i++) {
outzq[i] = pfring_zc_create_queue(zc, QUEUE_LEN);
if(outzq[i] == NULL) {
fprintf(stderr, "pfring_zc_create_queue error [%s]\n", strerror(errno));
return -1;
}
//printf("Created queue with ID=%u\n", pfring_zc_get_queue_id(outzq[i]));
}
wsp = pfring_zc_create_buffer_pool(zc, PREFETCH_BUFFERS);
if (wsp == NULL) {
fprintf(stderr, "pfring_zc_create_buffer_pool error\n");
return -1;
}
signal(SIGINT, sigproc);
signal(SIGTERM, sigproc);
signal(SIGINT, sigproc);
printf("Starting balancer with %d consumer threads..\n", num_threads);
if (hash_mode < 2) { /* balancer */
pfring_zc_distribution_func func;
if(strcmp(device, "sysdig") == 0)
func = (hash_mode == 0) ? rr_distribution_func : sysdig_distribution_func;
else
func = (hash_mode == 0) ? rr_distribution_func : NULL /* built-in IP-based */;
zw = pfring_zc_run_balancer(
inzq,
outzq,
num_devices,
num_threads,
wsp,
round_robin_bursts_policy,
NULL /* idle callback */,
func,
(void *) ((long) num_threads),
!wait_for_packet,
bind_worker_core
);
} else {
outzmq = pfring_zc_create_multi_queue(outzq, num_threads);
if(outzmq == NULL) {
fprintf(stderr, "pfring_zc_create_multi_queue error [%s]\n", strerror(errno));
return -1;
}
zw = pfring_zc_run_fanout(
inzq,
outzmq,
num_devices,
wsp,
round_robin_bursts_policy,
NULL /* idle callback */,
NULL /* built-in send-to-all */,
(void *) ((long) num_threads),
!wait_for_packet,
bind_worker_core
);
}
if(zw == NULL) {
fprintf(stderr, "pfring_zc_run_balancer error [%s]\n", strerror(errno));
return -1;
}
for (i = 0; i < num_threads; i++)
pthread_create(&threads[i], NULL, consumer_thread, (void*) i);
while (!do_shutdown) {
sleep(ALARM_SLEEP);
print_stats();
}
for (i = 0; i < num_threads; i++)
pthread_join(threads[i], NULL);
pfring_zc_kill_worker(zw);
pfring_zc_destroy_cluster(zc);
return 0;
}
int main(int argc, char* argv[]) {
char *device = NULL, c;
int i;
startTime.tv_sec = 0;
while((c = getopt(argc,argv,"ab:c:g:hi:n:p:l:zN:")) != '?') {
if((c == 255) || (c == -1)) break;
switch(c) {
case 'h':
printHelp();
break;
case 'a':
active = 1;
break;
case 'b':
num_ips = atoi(optarg);
break;
case 'c':
cluster_id = atoi(optarg);
break;
case 'i':
device = strdup(optarg);
break;
case 'l':
packet_len = atoi(optarg);
break;
case 'n':
num_to_send = atoi(optarg);
break;
case 'p':
pps = atoi(optarg);
break;
case 'g':
bind_core = atoi(optarg);
break;
#ifdef BURST_API
case 'z':
use_pkt_burst_api = 1;
break;
#endif
case 'N':
n2disk_producer = 1;
n2disk_threads = atoi(optarg);
break;
}
}
if (cluster_id < 0) printHelp();
if (n2disk_producer) {
if (device != NULL) printHelp();
if (n2disk_threads < 1) printHelp();
metadata_len = N2DISK_METADATA;
num_consumer_buffers += (n2disk_threads * (N2DISK_CONSUMER_QUEUE_LEN + 1)) + N2DISK_PREFETCH_BUFFERS;
}
if (device) {
num_queue_buffers = MAX_CARD_SLOTS;
} else {
num_queue_buffers = QUEUE_LEN;
}
stdin_packet_len = read_packet_hex(stdin_packet, sizeof(stdin_packet));
if (stdin_packet_len > 0)
packet_len = stdin_packet_len;
zc = pfring_zc_create_cluster(
cluster_id,
max_packet_len(device),
metadata_len,
num_queue_buffers + NBUFF + num_consumer_buffers,
numa_node_of_cpu(bind_core),
NULL /* auto hugetlb mountpoint */
);
if(zc == NULL) {
fprintf(stderr, "pfring_zc_create_cluster error [%s] Please check your hugetlb configuration\n",
strerror(errno));
return -1;
}
for (i = 0; i < NBUFF; i++) {
buffers[i] = pfring_zc_get_packet_handle(zc);
if (buffers[i] == NULL) {
fprintf(stderr, "pfring_zc_get_packet_handle error\n");
return -1;
}
}
if (device) {
zq = pfring_zc_open_device(zc, device, tx_only, 0);
if(zq == NULL) {
fprintf(stderr, "pfring_zc_open_device error [%s] Please check that %s is up and not already used\n",
strerror(errno), device);
return -1;
}
fprintf(stderr, "Sending packets to %s\n", device);
} else {
zq = pfring_zc_create_queue(zc, num_queue_buffers);
if(zq == NULL) {
fprintf(stderr, "pfring_zc_create_queue error [%s]\n", strerror(errno));
return -1;
}
fprintf(stderr, "Sending packets to cluster %u queue %u\n", cluster_id, 0);
if (n2disk_producer) {
char queues_list[256];
queues_list[0] = '\0';
for (i = 0; i < n2disk_threads; i++) {
if(pfring_zc_create_queue(zc, N2DISK_CONSUMER_QUEUE_LEN) == NULL) {
fprintf(stderr, "pfring_zc_create_queue error [%s]\n", strerror(errno));
return -1;
}
sprintf(&queues_list[strlen(queues_list)], "%d,", i+1);
}
queues_list[strlen(queues_list)-1] = '\0';
if (pfring_zc_create_buffer_pool(zc, N2DISK_PREFETCH_BUFFERS + n2disk_threads) == NULL) {
fprintf(stderr, "pfring_zc_create_buffer_pool error\n");
return -1;
}
fprintf(stderr, "Run n2disk with: --cluster-ipc-attach --cluster-id %d --cluster-ipc-queues %s --cluster-ipc-pool 0\n", cluster_id, queues_list);
}
}
signal(SIGINT, sigproc);
signal(SIGTERM, sigproc);
signal(SIGINT, sigproc);
signal(SIGALRM, my_sigalarm);
alarm(ALARM_SLEEP);
send_traffic();
print_stats();
pfring_zc_destroy_cluster(zc);
return 0;
}
void p_setup()
{
#if SWEET_THREADING || SWEET_REXI_THREAD_PARALLEL_SUM
if (omp_in_parallel())
{
std::cerr << "ERROR: NUMAMemManager may not be initialized within parallel region!" << std::endl;
std::cerr << " Call NUMAMemManager::setup() at program start" << std::endl;
exit(1);
}
#endif
if (setup_done)
return;
const char* env_verbosity = getenv("NUMA_BLOCK_ALLOC_VERBOSITY");
if (env_verbosity == nullptr)
verbosity = 0;
else
verbosity = atoi(env_verbosity);
#if NUMA_BLOCK_ALLOCATOR_TYPE == 0
if (verbosity > 0)
std::cout << "NUMA block alloc: Using default system's allocator" << std::endl;
num_alloc_domains = 1;
getThreadLocalDomainIdRef() = 0;
#elif NUMA_BLOCK_ALLOCATOR_TYPE == 1
if (verbosity > 0)
std::cout << "NUMA block alloc: Using NUMA node granularity" << std::endl;
/*
* NUMA granularity
*/
num_alloc_domains = numa_num_configured_nodes();
if (verbosity > 0)
std::cout << "num_alloc_domains: " << num_alloc_domains << std::endl;
// set NUMA id in case that master thread has a different id than the first thread
int cpuid = sched_getcpu();
getThreadLocalDomainIdRef() = numa_node_of_cpu(cpuid);
#if SWEET_THREADING || SWEET_REXI_THREAD_PARALLEL_SUM
#pragma omp parallel
{
int cpuid = sched_getcpu();
getThreadLocalDomainIdRef() = numa_node_of_cpu(cpuid);
}
#else
getThreadLocalDomainIdRef() = 0;
#endif
#elif NUMA_BLOCK_ALLOCATOR_TYPE == 2
if (verbosity > 0)
std::cout << "NUMA block alloc: Using allocator based on thread granularity" << std::endl;
/*
* Thread granularity, use this also per default
*/
#if SWEET_THREADING || SWEET_REXI_THREAD_PARALLEL_SUM
num_alloc_domains = omp_get_max_threads();
#else
num_alloc_domains = 1;
#endif
if (verbosity > 0)
std::cout << "num_alloc_domains: " << num_alloc_domains << std::endl;
// set NUMA id in case that master thread has a different id than the first thread
#if SWEET_THREADING || SWEET_REXI_THREAD_PARALLEL_SUM
getThreadLocalDomainIdRef() = omp_get_thread_num();
#pragma omp parallel
getThreadLocalDomainIdRef() = omp_get_thread_num();
#else
getThreadLocalDomainIdRef() = 0;
#endif
#elif NUMA_BLOCK_ALLOCATOR_TYPE == 3
if (verbosity > 0)
std::cout << "NUMA block alloc: Using non-numa single memory block chain" << std::endl;
num_alloc_domains = 1;
getThreadLocalDomainIdRef() = 0;
#else
# error "Invalid NUMA_BLOCK_ALLOCATOR_TYPE"
#endif
#if SWEET_THREADING || SWEET_REXI_THREAD_PARALLEL_SUM
if (verbosity > 0)
{
#pragma omp parallel
{
#pragma omp critical
{
std::cout << " thread id " << omp_get_thread_num() << " is assigned to memory allocator domain " << getThreadLocalDomainIdRef() << std::endl;
}
}
}
#endif
domain_block_groups.resize(num_alloc_domains);
#if 0
// TODO: care about first-touch policy
for (auto& n : domain_block_groups)
{
std::size_t S = num_alloc_domains*10;
// preallocate S different size of blocks which should be sufficient
n.block_groups.reserve(S);
}
#endif
setup_done = true;
}
int main(int argc, char* argv[]) {
char c;
char *in_pair = NULL, *out_pair = NULL;
char *vm_sockets = NULL, *vm_sock;
long i;
int cluster_id = -1;
int rc;
start_time.tv_sec = 0;
while((c = getopt(argc,argv,"ac:fhi:o:n:Q:r:t:")) != '?') {
if((c == 255) || (c == -1)) break;
switch(c) {
case 'h':
printHelp();
break;
case 'a':
wait_for_packet = 0;
break;
case 'c':
cluster_id = atoi(optarg);
break;
case 'f':
flush_packet = 1;
break;
case 'n':
num_ipc_queues = atoi(optarg);
break;
case 'i':
in_pair = strdup(optarg);
break;
case 'o':
out_pair = strdup(optarg);
break;
case 'r':
forwarder[RX_FWDR].bind_core = atoi(optarg);
break;
case 't':
forwarder[TX_FWDR].bind_core = atoi(optarg);
break;
case 'Q':
enable_vm_support = 1;
vm_sockets = strdup(optarg);
break;
}
}
if (cluster_id < 0) printHelp();
if (num_ipc_queues < 1) printHelp();
if (in_pair != NULL) {
char *q_id = strchr(in_pair, ';');
if (q_id == NULL) printHelp();
q_id[0] = '\0'; q_id++;
in_device = strdup(in_pair);
in_queue_id = atoi(q_id);
if (in_queue_id < 0 || in_queue_id >= num_ipc_queues) printHelp();
}
if (out_pair != NULL) {
char *q_id = strchr(out_pair, ';');
if (q_id == NULL) printHelp();
q_id[0] = '\0'; q_id++;
out_device = strdup(out_pair);
out_queue_id = atoi(q_id);
if (out_queue_id < 0 || out_queue_id >= num_ipc_queues) printHelp();
}
ipczqs = calloc(num_ipc_queues, sizeof(pfring_zc_queue *));
pools = calloc(num_ipc_queues, sizeof(pfring_zc_buffer_pool *));
zc = pfring_zc_create_cluster(
cluster_id,
in_device != NULL ? max_packet_len(in_device) : 1536,
0,
(((in_device != NULL) + (out_device != NULL)) * (MAX_CARD_SLOTS + 1)) + (num_ipc_queues * (QUEUE_LEN + 1)),
numa_node_of_cpu(forwarder[RX_FWDR].bind_core),
NULL /* auto hugetlb mountpoint */
);
if(zc == NULL) {
fprintf(stderr, "pfring_zc_create_cluster error [%s] Please check your hugetlb configuration\n",
strerror(errno));
return -1;
}
for (i = 0; i < num_ipc_queues; i++) {
ipczqs[i] = pfring_zc_create_queue(zc, QUEUE_LEN);
if(ipczqs[i] == NULL) {
fprintf(stderr, "pfring_zc_create_queue error [%s]\n", strerror(errno));
return -1;
}
}
for (i = 0; i < num_ipc_queues; i++) {
pools[i] = pfring_zc_create_buffer_pool(zc, 1);
if (pools[i] == NULL) {
fprintf(stderr, "pfring_zc_create_buffer_pool error\n");
return -1;
}
}
if (in_device != NULL) {
forwarder[RX_FWDR].inzq = pfring_zc_open_device(zc, in_device, rx_only, 0);
forwarder[RX_FWDR].outzq = ipczqs[in_queue_id];
if(forwarder[RX_FWDR].inzq == NULL) {
fprintf(stderr, "pfring_zc_open_device error [%s] Please check that %s is up and not already used\n",
strerror(errno), in_device);
return -1;
}
forwarder[RX_FWDR].buffer = pfring_zc_get_packet_handle(zc);
if (forwarder[RX_FWDR].buffer == NULL) {
fprintf(stderr, "pfring_zc_get_packet_handle error\n");
return -1;
}
printf("Forwarding from %s to Q%u\n", in_device, in_queue_id);
}
if (out_device != NULL) {
forwarder[TX_FWDR].inzq = ipczqs[out_queue_id];
forwarder[TX_FWDR].outzq = pfring_zc_open_device(zc, out_device, tx_only, 0);
if(forwarder[TX_FWDR].outzq == NULL) {
fprintf(stderr, "pfring_zc_open_device error [%s] Please check that %s is up and not already used\n",
strerror(errno), out_device);
return -1;
}
forwarder[TX_FWDR].buffer = pfring_zc_get_packet_handle(zc);
if (forwarder[TX_FWDR].buffer == NULL) {
fprintf(stderr, "pfring_zc_get_packet_handle error\n");
return -1;
}
printf("Forwarding from Q%u to %s\n", out_queue_id, out_device);
}
if (enable_vm_support) {
vm_sock = strtok(vm_sockets, ",");
while(vm_sock != NULL) {
rc = pfring_zc_vm_register(zc, vm_sock);
if (rc < 0) {
fprintf(stderr, "pfring_zc_vm_register error\n");
return -1;
}
vm_sock = strtok(NULL, ",");
}
rc = pfring_zc_vm_backend_enable(zc);
if (rc < 0) {
fprintf(stderr, "pfring_zc_vm_backend_enable error\n");
return -1;
}
}
signal(SIGINT, sigproc);
signal(SIGTERM, sigproc);
signal(SIGINT, sigproc);
printf("Starting master with %d queues..\n", num_ipc_queues);
if (out_device != NULL) pthread_create(&forwarder[TX_FWDR].thread, NULL, forwarder_thread, &forwarder[TX_FWDR]);
if (in_device != NULL) pthread_create(&forwarder[RX_FWDR].thread, NULL, forwarder_thread, &forwarder[RX_FWDR]);
while (!do_shutdown) {
sleep(ALARM_SLEEP);
print_stats();
}
if (out_device != NULL) pthread_join(forwarder[TX_FWDR].thread, NULL);
if (in_device != NULL) pthread_join(forwarder[RX_FWDR].thread, NULL);
pfring_zc_destroy_cluster(zc);
return 0;
}
static int pfring_zc_daq_initialize(const DAQ_Config_t *config,
void **ctxt_ptr, char *errbuf, size_t len) {
Pfring_Context_t *context;
DAQ_Dict* entry;
u_int numCPU = get_nprocs();
int i, max_buffer_len = 0, card_buffers;
int num_buffers;
int ipc_cluster_id;
context = calloc(1, sizeof(Pfring_Context_t));
if (context == NULL) {
snprintf(errbuf, len, "%s: Couldn't allocate memory for context!", __FUNCTION__);
return DAQ_ERROR_NOMEM;
}
context->mode = config->mode;
context->snaplen = config->snaplen;
context->promisc_flag =(config->flags & DAQ_CFG_PROMISC);
context->timeout = (config->timeout > 0) ? (int) config->timeout : -1;
context->devices[DAQ_PF_RING_PASSIVE_DEV_IDX] = strdup(config->name);
context->num_devices = 1;
context->ids_bridge = 0;
context->cluster_id = -1;
context->max_buffer_len = 0;
context->bindcpu = 0;
context->ipc_attach = 0;
if (!context->devices[DAQ_PF_RING_PASSIVE_DEV_IDX]) {
snprintf(errbuf, len, "%s: Couldn't allocate memory for the device string!", __FUNCTION__);
free(context);
return DAQ_ERROR_NOMEM;
}
for (entry = config->values; entry; entry = entry->next) {
if (!entry->value || !*entry->value) {
snprintf(errbuf, len, "%s: variable needs value(%s)\n", __FUNCTION__, entry->key);
return DAQ_ERROR;
} else if (!strcmp(entry->key, "bindcpu")) {
char *end = entry->value;
context->bindcpu = (int) strtol(entry->value, &end, 0);
if (*end
|| (context->bindcpu >= numCPU)) {
snprintf(errbuf, len, "%s: bad bindcpu(%s)\n", __FUNCTION__, entry->value);
return DAQ_ERROR;
} else {
cpu_set_t mask;
CPU_ZERO(&mask);
CPU_SET((int)context->bindcpu, &mask);
if (sched_setaffinity(0, sizeof(mask), &mask) < 0) {
snprintf(errbuf, len, "%s:failed to set bindcpu(%u) on pid %i\n", __FUNCTION__, context->bindcpu, getpid());
return DAQ_ERROR;
}
}
} else if (!strcmp(entry->key, "timeout")) {
char *end = entry->value;
context->timeout = (int) strtol(entry->value, &end, 0);
if (*end || (context->timeout < 0)) {
snprintf(errbuf, len, "%s: bad timeout(%s)\n", __FUNCTION__, entry->value);
return DAQ_ERROR;
}
} else if (!strcmp(entry->key, "idsbridge")) {
if (context->mode == DAQ_MODE_PASSIVE) {
char* end = entry->value;
context->ids_bridge = (int) strtol(entry->value, &end, 0);
if (*end || (context->ids_bridge < 0) || (context->ids_bridge > 2)) {
snprintf(errbuf, len, "%s: bad ids bridge mode(%s)\n", __FUNCTION__, entry->value);
return DAQ_ERROR;
}
} else {
snprintf(errbuf, len, "%s: idsbridge is for passive mode only\n", __FUNCTION__);
return DAQ_ERROR;
}
} else if (!strcmp(entry->key, "clusterid")) {
char *end = entry->value;
context->cluster_id = (int) strtol(entry->value, &end, 0);
if (*end || (context->cluster_id < 0)) {
snprintf(errbuf, len, "%s: bad clusterid(%s)\n", __FUNCTION__, entry->value);
return DAQ_ERROR;
}
} else {
snprintf(errbuf, len, "%s: unsupported variable(%s=%s)\n", __FUNCTION__, entry->key, entry->value);
return DAQ_ERROR;
}
}
if (context->mode == DAQ_MODE_READ_FILE) {
snprintf(errbuf, len, "%s: function not supported on PF_RING", __FUNCTION__);
free(context);
return DAQ_ERROR;
} else if (context->mode == DAQ_MODE_INLINE || (context->mode == DAQ_MODE_PASSIVE && context->ids_bridge)) {
/* zc:ethX+zc:ethY,zc:ethZ+zc:ethJ */
char *twins, *twins_pos = NULL;
context->num_devices = 0;
twins = strtok_r(context->devices[DAQ_PF_RING_PASSIVE_DEV_IDX], ",", &twins_pos);
while (twins != NULL) {
char *dev, *dev_pos = NULL, *tx_dev;
int last_twin = 0;
dev = strtok_r(twins, "+", &dev_pos);
while (dev != NULL) {
if (context->num_devices >= DAQ_PF_RING_MAX_NUM_DEVICES) {
snprintf(errbuf, len, "%s: Maximum num of devices reached (%d), you should increase "
"DAQ_PF_RING_MAX_NUM_DEVICES.\n", __FUNCTION__, DAQ_PF_RING_MAX_NUM_DEVICES);
free(context);
return DAQ_ERROR;
}
last_twin = context->num_devices;
context->devices[context->num_devices] = dev;
tx_dev = strchr(dev, '-');
if (tx_dev != NULL) { /* use the specified device for tx */
tx_dev[0] = '\0';
tx_dev++;
context->tx_devices[context->num_devices] = tx_dev;
} else {
context->tx_devices[context->num_devices] = dev;
}
context->num_devices++;
dev = strtok_r(NULL, "+", &dev_pos);
}
if (context->num_devices & 0x1) {
snprintf(errbuf, len, "%s: Wrong format: %s requires pairs of devices",
__FUNCTION__, context->mode == DAQ_MODE_INLINE ? "inline mode" : "ids bridge");
free(context);
return DAQ_ERROR;
}
if (last_twin > 0) /* new dev pair */
printf("%s <-> %s\n", context->devices[last_twin - 1], context->devices[last_twin]);
twins = strtok_r(NULL, ",", &twins_pos);
}
} else if (context->mode == DAQ_MODE_PASSIVE) {
/* zc:ethX,zc:ethY */
char *dev, *dev_pos = NULL;
context->num_devices = 0;
context->ipc_attach = 1; /* IPC queue attach supported in pure IDS only at the moment */
dev = strtok_r(context->devices[DAQ_PF_RING_PASSIVE_DEV_IDX], ",", &dev_pos);
while (dev != NULL) {
/* checking for IPC Queue */
if (!is_a_queue(dev, &ipc_cluster_id, &context->ipc_queues[context->num_devices])) {
context->ipc_attach = 0;
} else {
if (context->cluster_id == -1)
context->cluster_id = ipc_cluster_id;
else if (ipc_cluster_id != context->cluster_id)
context->ipc_attach = 0;
}
context->devices[context->num_devices++] = dev;
dev = strtok_r(NULL, ",", &dev_pos);
}
}
#ifdef SIG_RELOAD
/* catching the SIGRELOAD signal, replacing the default snort handler */
if ((default_sig_reload_handler = signal(SIGHUP, pfring_zc_daq_sig_reload)) == SIG_ERR)
default_sig_reload_handler = NULL;
#endif
if (!context->ipc_attach) {
num_buffers = 2 /* buffer, buffer_inject */;
#ifdef DAQ_PF_RING_BEST_EFFORT_BOOST
if (context->mode == DAQ_MODE_PASSIVE && context->ids_bridge == 2)
num_buffers += QUEUE_LEN;
#endif
for (i = 0; i < context->num_devices; i++) {
max_buffer_len = max_packet_len(context, context->devices[i], i, &card_buffers);
if (max_buffer_len > context->max_buffer_len) context->max_buffer_len = max_buffer_len;
if (strstr(context->devices[i], "zc:") != NULL) num_buffers += card_buffers;
if (context->tx_devices[i] != NULL) {
max_buffer_len = max_packet_len(context, context->tx_devices[i], i, &card_buffers);
if (max_buffer_len > context->max_buffer_len) context->max_buffer_len = max_buffer_len;
if (strstr(context->tx_devices[i], "zc:") != NULL) num_buffers += card_buffers;
}
}
context->cluster = pfring_zc_create_cluster(context->cluster_id, context->max_buffer_len, 0, num_buffers,
context->bindcpu == 0 ? -1 : numa_node_of_cpu(context->bindcpu), NULL);
if (context->cluster == NULL) {
DPE(context->errbuf, "%s: Cluster failed: %s(%d)", __FUNCTION__, strerror(errno), errno);
return DAQ_ERROR;
}
context->buffer = pfring_zc_get_packet_handle(context->cluster);
if (context->buffer == NULL) {
DPE(context->errbuf, "%s: Buffer allocation failed: %s(%d)", __FUNCTION__, strerror(errno), errno);
return DAQ_ERROR;
}
context->buffer_inject = pfring_zc_get_packet_handle(context->cluster);
if (context->buffer_inject == NULL) {
DPE(context->errbuf, "%s: Buffer allocation failed: %s(%d)", __FUNCTION__, strerror(errno), errno);
return DAQ_ERROR;
}
} else {
context->ipc_pool = pfring_zc_ipc_attach_buffer_pool(context->cluster_id, context->ipc_queues[0]);
if (context->ipc_pool == NULL) {
snprintf(errbuf, len, "%s: pfring_zc_ipc_attach_buffer_pool error %s(%d), please check that cluster %d is running\n",
__FUNCTION__, strerror(errno), errno, context->cluster_id);
return -1;
}
context->buffer = pfring_zc_get_packet_handle_from_pool(context->ipc_pool);
if (context->buffer == NULL) {
DPE(context->errbuf, "%s: Buffer allocation failed: %s(%d)", __FUNCTION__, strerror(errno), errno);
return DAQ_ERROR;
}
}
for (i = 0; i < context->num_devices; i++) {
if (pfring_zc_daq_open(context, i) == -1)
return DAQ_ERROR;
}
if (!context->ipc_attach) {
#ifdef DAQ_PF_RING_BEST_EFFORT_BOOST
if (context->mode == DAQ_MODE_PASSIVE && context->ids_bridge == 2) {
context->q = pfring_zc_create_queue(context->cluster, QUEUE_LEN);
if (context->q == NULL) {
snprintf(errbuf, len, "%s: Couldn't create queue: '%s'", __FUNCTION__, strerror(errno));
return DAQ_ERROR_NOMEM;
}
context->mq_queues[context->num_devices] = context->q;
context->mq = pfring_zc_create_multi_queue(context->mq_queues, context->num_devices + 1);
}
#endif
}
context->state = DAQ_STATE_INITIALIZED;
*ctxt_ptr = context;
return DAQ_SUCCESS;
}
int main(int argc, char* argv[]) {
char *device = NULL, c;
int i;
startTime.tv_sec = 0;
while((c = getopt(argc,argv,"ab:c:g:hi:n:p:l:z")) != '?') {
if((c == 255) || (c == -1)) break;
switch(c) {
case 'h':
printHelp();
break;
case 'a':
active = 1;
break;
case 'b':
num_ips = atoi(optarg);
break;
case 'c':
cluster_id = atoi(optarg);
break;
case 'i':
device = strdup(optarg);
break;
case 'l':
packet_len = atoi(optarg);
break;
case 'n':
num_to_send = atoi(optarg);
break;
case 'p':
pps = atoi(optarg);
break;
case 'g':
bind_core = atoi(optarg);
break;
#ifdef BURST_API
case 'z':
use_pkt_burst_api = 1;
break;
#endif
}
}
if (cluster_id < 0) printHelp();
if (device) {
num_queue_buffers = MAX_CARD_SLOTS;
} else {
num_queue_buffers = QUEUE_LEN;
}
stdin_packet_len = read_packet_hex(stdin_packet, sizeof(stdin_packet));
if (stdin_packet_len > 0)
packet_len = stdin_packet_len;
zc = pfring_zc_create_cluster(
cluster_id,
1536,
0,
num_queue_buffers + NBUFF,
numa_node_of_cpu(bind_core),
NULL /* auto hugetlb mountpoint */
);
if(zc == NULL) {
fprintf(stderr, "pfring_zc_create_cluster error [%s] Please check your hugetlb configuration\n",
strerror(errno));
return -1;
}
for (i = 0; i < NBUFF; i++) {
buffers[i] = pfring_zc_get_packet_handle(zc);
if (buffers[i] == NULL) {
fprintf(stderr, "pfring_zc_get_packet_handle error\n");
return -1;
}
}
if (device) {
zq = pfring_zc_open_device(zc, device, tx_only, 0);
if(zq == NULL) {
fprintf(stderr, "pfring_zc_open_device error [%s] Please check that %s is up and not already used\n",
strerror(errno), device);
return -1;
}
fprintf(stderr, "Sending packets to %s\n", device);
} else {
zq = pfring_zc_create_queue(zc, num_queue_buffers);
if(zq == NULL) {
fprintf(stderr, "pfring_zc_create_queue error [%s]\n", strerror(errno));
return -1;
}
zp = pfring_zc_create_buffer_pool(zc, POOL_SIZE);
if (zp == NULL) {
fprintf(stderr, "pfring_zc_create_buffer_pool error\n");
return -1;
}
fprintf(stderr, "Sending packets to cluster %u queue %u\n", cluster_id, 0);
}
signal(SIGINT, sigproc);
signal(SIGTERM, sigproc);
signal(SIGINT, sigproc);
signal(SIGALRM, my_sigalarm);
alarm(ALARM_SLEEP);
send_traffic();
print_stats();
pfring_zc_destroy_cluster(zc);
return 0;
}
int main(int argc, char* argv[]) {
char *device = NULL, c;
pthread_t thread;
pthread_t time_thread;
char *vm_sock = NULL;
int i, rc, ipc_q_attach = 0;
startTime.tv_sec = 0;
while((c = getopt(argc,argv,"ab:c:g:hi:n:p:l:zN:S:P:Q:")) != '?') {
if((c == 255) || (c == -1)) break;
switch(c) {
case 'h':
printHelp();
break;
case 'a':
active = 1;
break;
case 'b':
num_ips = atoi(optarg);
break;
case 'c':
cluster_id = atoi(optarg);
break;
case 'i':
device = strdup(optarg);
break;
case 'l':
packet_len = atoi(optarg);
break;
case 'n':
num_to_send = atoi(optarg);
break;
case 'p':
pps = atoi(optarg);
/* auto flush on wait flush_packet = 1; */
break;
case 'g':
bind_core = atoi(optarg);
break;
case 'Q':
enable_vm_support = 1;
vm_sock = strdup(optarg);
break;
#ifdef BURST_API
case 'z':
use_pkt_burst_api = 1;
break;
#endif
case 'N':
n2disk_producer = 1;
n2disk_threads = atoi(optarg);
break;
case 'S':
append_timestamp = 1;
bind_time_pulse_core = atoi(optarg);
break;
case 'P':
use_pulse_time = 1;
bind_time_pulse_core = atoi(optarg);
break;
}
}
if (n2disk_producer)
device = NULL;
/* checking if the interface is a queue allocated by an external cluster (ipc) */
if (device != NULL && is_a_queue(device, &cluster_id, &queue_id))
ipc_q_attach = 1;
if (cluster_id < 0) printHelp();
stdin_packet_len = read_packet_hex(stdin_packet, sizeof(stdin_packet));
if (stdin_packet_len > 0)
packet_len = stdin_packet_len;
if (n2disk_producer) {
if (device != NULL || ipc_q_attach) printHelp();
if (n2disk_threads < 1) printHelp();
metadata_len = N2DISK_METADATA;
num_consumer_buffers += (n2disk_threads * (N2DISK_CONSUMER_QUEUE_LEN + 1)) + N2DISK_PREFETCH_BUFFERS;
}
if (!ipc_q_attach) {
if (device != NULL)
num_queue_buffers = MAX_CARD_SLOTS;
else
num_queue_buffers = QUEUE_LEN;
zc = pfring_zc_create_cluster(
cluster_id,
max_packet_len(device),
metadata_len,
num_queue_buffers + NBUFF + num_consumer_buffers,
numa_node_of_cpu(bind_core),
NULL /* auto hugetlb mountpoint */
);
if(zc == NULL) {
fprintf(stderr, "pfring_zc_create_cluster error [%s] Please check your hugetlb configuration\n",
strerror(errno));
return -1;
}
for (i = 0; i < NBUFF; i++) {
buffers[i] = pfring_zc_get_packet_handle(zc);
if (buffers[i] == NULL) {
fprintf(stderr, "pfring_zc_get_packet_handle error\n");
return -1;
}
}
if (device) {
zq = pfring_zc_open_device(zc, device, tx_only, 0);
if(zq == NULL) {
fprintf(stderr, "pfring_zc_open_device error [%s] Please check that %s is up and not already used\n",
strerror(errno), device);
return -1;
}
fprintf(stderr, "Sending packets to %s\n", device);
} else {
zq = pfring_zc_create_queue(zc, num_queue_buffers);
if(zq == NULL) {
fprintf(stderr, "pfring_zc_create_queue error [%s]\n", strerror(errno));
return -1;
}
if (pfring_zc_create_buffer_pool(zc, n2disk_producer ? (N2DISK_PREFETCH_BUFFERS + n2disk_threads) : 1) == NULL) {
fprintf(stderr, "pfring_zc_create_buffer_pool error\n");
return -1;
}
fprintf(stderr, "Sending packets to cluster %u queue %u\n", cluster_id, 0);
if (n2disk_producer) {
char queues_list[256];
queues_list[0] = '\0';
for (i = 0; i < n2disk_threads; i++) {
if(pfring_zc_create_queue(zc, N2DISK_CONSUMER_QUEUE_LEN) == NULL) {
fprintf(stderr, "pfring_zc_create_queue error [%s]\n", strerror(errno));
return -1;
}
sprintf(&queues_list[strlen(queues_list)], "%d,", i+1);
}
queues_list[strlen(queues_list)-1] = '\0';
fprintf(stderr, "Run n2disk with: --cluster-ipc-attach --cluster-id %d --cluster-ipc-queues %s --cluster-ipc-pool 0\n", cluster_id, queues_list);
}
}
if (enable_vm_support) {
rc = pfring_zc_vm_register(zc, vm_sock);
if (rc < 0) {
fprintf(stderr, "pfring_zc_vm_register(%s) error\n", vm_sock);
return -1;
}
rc = pfring_zc_vm_backend_enable(zc);
if (rc < 0) {
fprintf(stderr, "pfring_zc_vm_backend_enable error\n");
return -1;
}
}
} else { /* IPC */
fprintf(stderr, "Attaching to cluster %d queue %d (IPC)\n", cluster_id, queue_id);
zq = pfring_zc_ipc_attach_queue(cluster_id, queue_id, tx_only);
if(zq == NULL) {
fprintf(stderr, "pfring_zc_ipc_attach_queue error [%s] Please check that cluster %d is running\n",
strerror(errno), cluster_id);
return -1;
}
zp = pfring_zc_ipc_attach_buffer_pool(cluster_id, queue_id);
if(zp == NULL) {
fprintf(stderr, "pfring_zc_ipc_attach_buffer_pool error [%s] Please check that cluster %d is running\n",
strerror(errno), cluster_id);
return -1;
}
for (i = 0; i < NBUFF; i++) {
buffers[i] = pfring_zc_get_packet_handle_from_pool(zp);
if (buffers[i] == NULL) {
fprintf(stderr, "pfring_zc_get_packet_handle_from_pool error\n");
return -1;
}
}
}
signal(SIGINT, sigproc);
signal(SIGTERM, sigproc);
signal(SIGINT, sigproc);
if (use_pulse_time) pulse_timestamp_ns = calloc(CACHE_LINE_LEN/sizeof(u_int64_t), sizeof(u_int64_t));
if (append_timestamp) pulse_timestamp_ns_n = calloc(CACHE_LINE_LEN/sizeof(u_int64_t), sizeof(u_int64_t));
if (append_timestamp || use_pulse_time) pthread_create(&time_thread, NULL, time_pulse_thread, NULL);
if (use_pulse_time) while (!*pulse_timestamp_ns && !do_shutdown); /* wait for ts */
if (append_timestamp) while (!*pulse_timestamp_ns_n && !do_shutdown); /* wait for ts */
pthread_create(&thread, NULL, send_traffic, NULL);
while (!do_shutdown) {
sleep(ALARM_SLEEP);
print_stats();
}
pthread_join(thread, NULL);
print_stats();
if (append_timestamp || use_pulse_time)
pthread_join(time_thread, NULL);
if (!ipc_q_attach) {
pfring_zc_destroy_cluster(zc);
} else {
for (i = 0; i < NBUFF; i++)
pfring_zc_release_packet_handle_to_pool(zp, buffers[i]);
pfring_zc_ipc_detach_queue(zq);
pfring_zc_ipc_detach_buffer_pool(zp);
}
return 0;
}
/*
* Class: xerial_jnuma_NumaNative
* Method: Node
* Signature: ()I
*/
JNIEXPORT jint JNICALL Java_xerial_jnuma_NumaNative_currentNode
(JNIEnv *env, jobject obj) {
return numa_node_of_cpu(sched_getcpu());
}
int main(int argc, char* argv[]) {
char c;
char *ingress_devices = NULL, *egress_devices = NULL, *dev;
long i;
int wait_for_packet = 1;
start_time.tv_sec = 0;
while((c = getopt(argc,argv, "ac:g:hi:o:")) != '?') {
if((c == 255) || (c == -1)) break;
switch(c) {
case 'h':
printHelp();
break;
case 'a':
wait_for_packet = 0;
break;
case 'c':
cluster_id = atoi(optarg);
break;
case 'i':
ingress_devices = strdup(optarg);
break;
case 'o':
egress_devices = strdup(optarg);
break;
case 'g':
bind_worker_core = atoi(optarg);
break;
}
}
if(cluster_id < 0) printHelp();
dev = strtok(ingress_devices, ",");
while(dev != NULL) {
in_devices = realloc(in_devices, sizeof(char *) * (num_in_devices+1));
in_devices[num_in_devices] = strdup(dev);
num_in_devices++;
dev = strtok(NULL, ",");
}
dev = strtok(egress_devices, ",");
while(dev != NULL) {
out_devices = realloc(out_devices, sizeof(char *) * (num_out_devices+1));
out_devices[num_out_devices] = strdup(dev);
num_out_devices++;
dev = strtok(NULL, ",");
}
if((num_in_devices == 0) || (num_out_devices == 0)) printHelp();
zc = pfring_zc_create_cluster(cluster_id,
max_packet_len(in_devices[0]),
metadata_len,
((num_in_devices + num_out_devices) * MAX_CARD_SLOTS) + PREFETCH_BUFFERS,
numa_node_of_cpu(bind_worker_core),
NULL /* auto hugetlb mountpoint */);
if(zc == NULL) {
fprintf(stderr, "pfring_zc_create_cluster error [%s] Please check your hugetlb configuration\n",
strerror(errno));
return -1;
}
inzqs = calloc(num_in_devices, sizeof(pfring_zc_queue *));
outzqs = calloc(num_out_devices, sizeof(pfring_zc_queue *));
for(i = 0; i < num_in_devices; i++) {
inzqs[i] = pfring_zc_open_device(zc, in_devices[i], rx_only, 0);
if(inzqs[i] == NULL) {
fprintf(stderr, "[RX] pfring_zc_open_device error [%s] Please check that %s is up and not already used\n",
strerror(errno), in_devices[i]);
return -1;
}
}
for(i = 0; i < num_out_devices; i++) {
outzqs[i] = pfring_zc_open_device(zc, out_devices[i], tx_only, 0);
if(outzqs[i] == NULL) {
fprintf(stderr, "[TX] pfring_zc_open_device error [%s] Please check that %s is up and not already used\n",
strerror(errno), out_devices[i]);
return -1;
}
}
signal(SIGINT, sigproc);
signal(SIGTERM, sigproc);
signal(SIGINT, sigproc);
outzmq = pfring_zc_create_multi_queue(outzqs, num_out_devices);
if(outzmq == NULL) {
fprintf(stderr, "pfring_zc_create_multi_queue error [%s]\n", strerror(errno));
return -1;
}
wsp = pfring_zc_create_buffer_pool(zc, PREFETCH_BUFFERS);
if(wsp == NULL) {
fprintf(stderr, "pfring_zc_create_buffer_pool error\n");
return -1;
}
zw = pfring_zc_run_fanout(inzqs,
outzmq,
num_in_devices,
wsp,
round_robin_bursts_policy,
NULL /* idle callback */,
NULL /* fanout */,
NULL,
!wait_for_packet,
bind_worker_core);
if(zw == NULL) {
fprintf(stderr, "pfring_zc_run_fanout error [%s]\n", strerror(errno));
return -1;
}
while(!do_shutdown) {
sleep(ALARM_SLEEP);
print_stats();
}
pfring_zc_destroy_cluster(zc);
return 0;
}
CsxMatrix<IndexType, ValueType> * CsxManager<IndexType, ValueType>::
MakeCsx(bool symmetric)
{
CsxMatrix<IndexType, ValueType> *csx;
#if SPX_USE_NUMA
NumaAllocator &numa_alloc = NumaAllocator::GetInstance();
int cpu = sched_getcpu();
if (cpu < 0) {
LOG_ERROR << "sched_getcpu() failed " << strerror(errno);
exit(1);
}
int node = numa_node_of_cpu(cpu);
if (node < 0) {
LOG_ERROR << "numa_node_of_cpu() failed " << strerror(errno);
exit(1);
}
csx = new (numa_alloc, node) CsxMatrix<IndexType, ValueType>;
values_ = new (numa_alloc, node) ValueType[spm_->GetNrNonzeros()];
rows_info_ = new (numa_alloc, node) row_info_t[spm_->GetNrRows()];
#else
csx = new CsxMatrix<IndexType, ValueType>;
values_ = new ValueType[spm_->GetNrNonzeros()];
rows_info_ = new row_info_t[spm_->GetNrRows()];
#endif // SPX_USE_NUMA
// Be greedy with the initial capacity (equal to CSR col_ind size)
// to avoid realloc()'s.
csx->nnz = spm_->GetNrNonzeros();
csx->nrows = spm_->GetNrRows();
csx->ncols = spm_->GetNrCols();
csx->row_start = spm_->GetRowStart();
values_idx_ = 0;
new_row_ = false; // Do not mark first row.
if (!symmetric) {
curr_row_ = 0;
for (size_t i = 0; i < spm_->GetRowptrSize() - 1; ++i, ++curr_row_) {
typename SparsePartition<IndexType, ValueType>::iterator rbegin =
spm_->begin(i);
typename SparsePartition<IndexType, ValueType>::iterator rend =
spm_->end(i);
// LOG_DEBUG << "MakeCsx(): row: " << i << "\n";
if (rbegin == rend) { // Check if row is empty.
// LOG_DEBUG << "MakeCsx(): row is empty\n";
if (new_row_ == false) {
rows_info_[i].rowptr = 0;
new_row_ = true; // In case the first row is empty.
} else {
empty_rows_++;
rows_info_[i].rowptr = rows_info_[i-1].rowptr;
}
rows_info_[i].valptr = 0;
rows_info_[i].span = 0;
continue;
}
if (i > 0)
rows_info_[i].rowptr = ctl_bld_.GetCtlSize();
else
rows_info_[i].rowptr = 0;
rows_info_[i].valptr = values_idx_;
DoRow(rbegin, rend, i);
rows_info_[i].span = span_;
new_row_ = true;
}
for (size_t i = spm_->GetRowptrSize() - 1;
i < (size_t) spm_->GetNrRows(); i++) {
rows_info_[i].valptr = 0;
rows_info_[i].rowptr = rows_info_[i-1].rowptr;
rows_info_[i].span = 0;
}
} else {
for (size_t i = 0; i < spm_->GetRowptrSize() - 1; i++) {
typename SparsePartition<IndexType, ValueType>::iterator rbegin =
spm_->begin(i);
typename SparsePartition<IndexType, ValueType>::iterator rend =
spm_->end(i);
// LOG_DEBUG << "MakeCsx(): row: " << i << "\n";
if (rbegin == rend){ // Check if row is empty.
// LOG_DEBUG << "MakeCsx(): row is empty\n";
if (new_row_ == false) {
rows_info_[i].rowptr = 0;
new_row_ = true; // In case the first row is empty.
} else {
rows_info_[i].rowptr = rows_info_[i-1].rowptr;
empty_rows_++;
}
rows_info_[i].valptr = 0;
rows_info_[i].span = 0;
continue;
}
if (i > 0)
rows_info_[i].rowptr = ctl_bld_.GetCtlSize();
else
rows_info_[i].rowptr = 0;
rows_info_[i].valptr = values_idx_;
DoSymRow(rbegin, rend);
rows_info_[i].span = span_;
new_row_ = true;
}
for (size_t i = spm_->GetRowptrSize() - 1;
i < (size_t) spm_->GetNrRows(); i++) {
rows_info_[i].valptr = 0;
rows_info_[i].rowptr = rows_info_[i-1].rowptr;
rows_info_[i].span = 0;
}
}
#if SPX_DEBUG
// LOG_DEBUG << "values_\n";
// for (size_t i = 0; i < spm_->GetNrNonzeros(); ++i)
// LOG_DEBUG << values_[i] << "\n";
#endif
csx->row_jumps = row_jmps_;
csx->ctl_size = ctl_bld_.GetCtlSize();
csx->ctl = ctl_bld_.Finalize();
assert(values_idx_ == spm_->GetNrNonzeros());
csx->values = values_;
values_ = NULL;
values_idx_ = 0;
csx->rows_info = rows_info_;
rows_info_ = NULL;
AddMappings(csx->id_map);
return csx;
}
static inline void
s_init_numa_thingies(void) {
numa_set_strict(1);
s_n_cpus = (int64_t)sysconf(_SC_NPROCESSORS_CONF);
if (s_n_cpus > 0) {
s_numa_nodes = (unsigned int *)malloc(sizeof(int) * (size_t)s_n_cpus);
if (s_numa_nodes != NULL) {
int i;
lagopus_result_t r;
(void)memset((void *)s_numa_nodes, 0, sizeof(int) * (size_t)s_n_cpus);
for (i = 0; i < (int)s_n_cpus; i++) {
s_numa_nodes[i] = (unsigned int)numa_node_of_cpu(i);
if (s_min_numa_node > s_numa_nodes[i]) {
s_min_numa_node = s_numa_nodes[i];
}
if (s_max_numa_node < s_numa_nodes[i]) {
s_max_numa_node = s_numa_nodes[i];
}
}
#ifndef DO_NUMA_EVNE_ONE_NODE
if (s_max_numa_node > s_min_numa_node) {
r = lagopus_hashmap_create(&s_tbl,
LAGOPUS_HASHMAP_TYPE_ONE_WORD, NULL);
if (r == LAGOPUS_RESULT_OK) {
s_is_numa = true;
s_alloc_proc = s_numa_alloc;
s_free_proc = s_numa_free;
lagopus_msg_debug(5, "The NUMA aware memory allocator is "
"initialized.\n");
} else {
lagopus_perror(r);
lagopus_exit_fatal("can't initialize the "
"NUMA memory allocation table.\n");
}
} else {
s_alloc_proc = s_uma_alloc;
s_free_proc = s_uma_free;
lagopus_msg_debug(5, "There is only one NUMA node on this machine. "
"No NUMA aware memory allocation is enabled.\n");
}
#else
r = lagopus_hashmap_create(&s_tbl,
LAGOPUS_HASHMAP_TYPE_ONE_WORD, NULL);
if (r == LAGOPUS_RESULT_OK) {
s_is_numa = true;
s_alloc_proc = s_numa_alloc;
s_free_proc = s_numa_free;
lagopus_msg_debug(5, "The NUMA aware memory allocator is "
"initialized.\n");
} else {
lagopus_perror(r);
lagopus_exit_fatal("can't initialize the "
"NUMA memory allocation table.\n");
}
#endif /* ! DO_NUMA_EVNE_ONE_NODE */
}
}
}
