static int create_work_queues(void)
{
struct work_queue *util_wq;
if (init_work_queue(get_nr_nodes))
return -1;
sys->net_wqueue = create_work_queue("net", WQ_UNLIMITED);
sys->gateway_wqueue = create_work_queue("gway", WQ_UNLIMITED);
sys->io_wqueue = create_work_queue("io", WQ_UNLIMITED);
sys->recovery_wqueue = create_work_queue("rw", WQ_UNLIMITED);
sys->deletion_wqueue = create_ordered_work_queue("deletion");
sys->block_wqueue = create_ordered_work_queue("block");
sys->md_wqueue = create_ordered_work_queue("md");
sys->areq_wqueue = create_work_queue("async_req", WQ_UNLIMITED);
if (sys->enable_object_cache) {
sys->oc_reclaim_wqueue =
create_ordered_work_queue("oc_reclaim");
sys->oc_push_wqueue = create_work_queue("oc_push", WQ_DYNAMIC);
if (!sys->oc_reclaim_wqueue || !sys->oc_push_wqueue)
return -1;
}
if (!sys->gateway_wqueue || !sys->io_wqueue || !sys->recovery_wqueue ||
!sys->deletion_wqueue || !sys->block_wqueue || !sys->md_wqueue ||
!sys->areq_wqueue)
return -1;
util_wq = create_ordered_work_queue("util");
if (!util_wq)
return -1;
register_util_wq(util_wq);
return 0;
}
static int accord_init(const char *option, uint8_t *myaddr)
{
if (!option) {
eprintf("specify one of the accord servers.\n");
eprintf("e.g. sheep /store -c accord:127.0.0.1\n");
return -1;
}
pthread_mutex_lock(&start_lock);
ahandle = acrd_init(option, 9090, acrd_join_fn, acrd_leave_fn, NULL);
if (!ahandle) {
eprintf("failed to connect to accrd server %s\n", option);
return -1;
}
if (get_addr(myaddr) < 0)
return -1;
efd = eventfd(0, EFD_NONBLOCK);
if (efd < 0) {
eprintf("failed to create an event fd: %m\n");
return -1;
}
acrd_wq = init_work_queue(1);
if (!acrd_wq)
eprintf("failed to create accord workqueue: %m\n");
return -1;
}
static int create_work_queues(void)
{
struct work_queue *util_wq;
if (init_work_queue(get_nr_nodes))
return -1;
if (wq_net_threads) {
sd_info("# of threads in net workqueue: %d", wq_net_threads);
sys->net_wqueue = create_fixed_work_queue("net", wq_net_threads);
} else {
sd_info("net workqueue is created as unlimited, it is not recommended!");
sys->net_wqueue = create_work_queue("net", WQ_UNLIMITED);
}
if (wq_gway_threads) {
sd_info("# of threads in gway workqueue: %d", wq_gway_threads);
sys->gateway_wqueue = create_fixed_work_queue("gway", wq_gway_threads);
} else {
sd_info("gway workqueue is created as unlimited, it is not recommended!");
sys->gateway_wqueue = create_work_queue("gway", WQ_UNLIMITED);
}
if (wq_io_threads) {
sd_info("# of threads in io workqueue: %d", wq_io_threads);
sys->io_wqueue = create_fixed_work_queue("io", wq_io_threads);
} else {
sd_info("io workqueue is created as unlimited, it is not recommended!");
sys->io_wqueue = create_work_queue("io", WQ_UNLIMITED);
}
if (wq_recovery_threads) {
sd_info("# of threads in rw workqueue: %d", wq_recovery_threads);
sys->recovery_wqueue = create_fixed_work_queue("rw", wq_recovery_threads);
} else {
sd_info("recovery workqueue is created as unlimited, it is not recommended!");
sys->recovery_wqueue = create_work_queue("rw", WQ_UNLIMITED);
}
sys->deletion_wqueue = create_ordered_work_queue("deletion");
sys->block_wqueue = create_ordered_work_queue("block");
sys->md_wqueue = create_ordered_work_queue("md");
if (wq_async_threads) {
sd_info("# of threads in async_req workqueue: %d", wq_async_threads);
sys->areq_wqueue = create_fixed_work_queue("async_req", wq_async_threads);
} else {
sd_info("async_req workqueue is created as unlimited, it is not recommended!");
sys->areq_wqueue = create_work_queue("async_req", WQ_UNLIMITED);
}
if (!sys->gateway_wqueue || !sys->io_wqueue || !sys->recovery_wqueue ||
!sys->deletion_wqueue || !sys->block_wqueue || !sys->md_wqueue ||
!sys->areq_wqueue)
return -1;
util_wq = create_ordered_work_queue("util");
if (!util_wq)
return -1;
register_util_wq(util_wq);
return 0;
}
/* ==================== init_server_work_queue() ==================== */
int init_server_work_queue(server_t *server)
{
/*pthread_key_create(&key_actived_sockbuf, NULL);*/
int i;
for ( i = 0; i < RECV_THREADS; i++ ) {
server->recv_queue[i] = init_work_queue(recv_queue_handle_request, RECV_INTERVAL);
if ( server->recv_queue[i] == NULL ){
return -1;
}
}
for ( i = 0; i < SEND_THREADS; i++ ) {
server->send_queue[i] = init_work_queue(send_queue_handle_response, SEND_INTERVAL);
if ( server->send_queue[i] == NULL )
return -1;
}
return 0;
}
static int accord_init(const char *option)
{
int ret;
if (!option) {
eprintf("specify one of the accord servers.\n");
eprintf("e.g. sheep /store -c accord:127.0.0.1\n");
return -1;
}
pthread_mutex_lock(&start_lock);
ahandle = acrd_init(option, 9090, acrd_join_fn, acrd_leave_fn, NULL);
if (!ahandle) {
eprintf("failed to connect to accrd server %s\n", option);
return -1;
}
efd = eventfd(0, EFD_NONBLOCK);
if (efd < 0) {
eprintf("failed to create an event fd: %m\n");
return -1;
}
acrd_wq = init_work_queue("accord", true);
if (!acrd_wq) {
eprintf("failed to create accord workqueue: %m\n");
return -1;
}
pthread_cond_wait(&start_cond, &start_lock);
pthread_mutex_unlock(&start_lock);
if (need_cleanup)
for_each_acrd_file(ahandle, BASE_FILE, __acrd_del, NULL);
else {
queue_start_pos = -1;
queue_end_pos = -1;
for_each_acrd_file(ahandle, QUEUE_FILE, find_queue_end,
&queue_end_pos);
}
acrd_add_watch(ahandle, QUEUE_FILE, ACRD_EVENT_PREFIX | ACRD_EVENT_ALL,
acrd_watch_fn, NULL);
ret = register_event(efd, acrd_handler, NULL);
if (ret) {
eprintf("failed to register accord event handler (%d)\n", ret);
return -1;
}
return 0;
}
/*! \fn void mpegEncapUnitInit(mpeg_process_private* priv)
**************************************************************************
* \brief Init Mpeg Encap Unit
* \return 0 or error code
*/
void mpegEncapUnitInit(mpeg_process_private* priv)
{
int chn_idx, pid_idx;
priv->mpeg_proc = create_proc_entry("mpeg", 0, NULL);
if (NULL==priv->mpeg_proc)
{
printk("MPEG driver failure, can not create MPEG proc node\n");
BUG();
return;
}
priv->mpeg_proc->read_proc = mpeg_read_proc;
priv->mpeg_proc->write_proc = mpeg_write_proc;
priv->mpeg_proc->data = (void*)priv;
priv->irq_counter = ~0;
for (chn_idx=0; chn_idx<MAX_MPEG_CHNS; chn_idx++)
{
for (pid_idx=0; pid_idx<PID_SPACE; pid_idx++)
{
SET_ACTIVE_PID_INDEX(priv->pid_filters[chn_idx][pid_idx].filter, INVALID_ACTIVE_PID_IDX);
}
for (pid_idx=0; pid_idx<MAX_MPEG_ACTIVE_PIDS; pid_idx++)
{
priv->encap_pid_params[chn_idx][pid_idx].pid = MPEG_INVALID_PID;
}
}
/* get a PAL handle */
priv->pal_hnd = PAL_cppi4Init(NULL, NULL);
init_cppi_rx(priv);
init_cppi_tx(priv);
init_work_queue(priv);
printk("MPEG driver initialization complete.\n");
}
static int local_init(struct cdrv_handlers *handlers, const char *option,
uint8_t *myaddr)
{
sigset_t mask;
static struct timer t = {
.callback = check_pids,
.data = &t,
};
lhdlrs = *handlers;
if (option)
shmfile = option;
/* set 127.0.0.1 */
memset(myaddr, 0, 16);
myaddr[12] = 127;
myaddr[15] = 1;
shm_queue_init();
sigemptyset(&mask);
sigaddset(&mask, SIGUSR1);
sigprocmask(SIG_BLOCK, &mask, NULL);
sigfd = signalfd(-1, &mask, SFD_NONBLOCK);
if (sigfd < 0) {
eprintf("failed to create a signal fd: %m\n");
return -1;
}
add_timer(&t, 1);
local_block_wq = init_work_queue(1);
return sigfd;
}
static int create_work_queues(void)
{
if (init_work_queue(get_nr_nodes))
return -1;
sys->gateway_wqueue = create_work_queue("gway", WQ_UNLIMITED);
sys->io_wqueue = create_work_queue("io", WQ_UNLIMITED);
sys->recovery_wqueue = create_ordered_work_queue("rw");
sys->deletion_wqueue = create_ordered_work_queue("deletion");
sys->block_wqueue = create_ordered_work_queue("block");
sys->md_wqueue = create_ordered_work_queue("md");
if (sys->enable_object_cache) {
sys->oc_reclaim_wqueue =
create_ordered_work_queue("oc_reclaim");
sys->oc_push_wqueue = create_work_queue("oc_push", WQ_DYNAMIC);
if (!sys->oc_reclaim_wqueue || !sys->oc_push_wqueue)
return -1;
}
if (!sys->gateway_wqueue || !sys->io_wqueue || !sys->recovery_wqueue ||
!sys->deletion_wqueue || !sys->block_wqueue || !sys->md_wqueue)
return -1;
return 0;
}
int main(int argc, char **argv)
{
int ch, longindex, ret;
unsigned long flags;
struct option *long_options;
const struct command *commands;
const char *short_options;
char *p;
const struct sd_option *sd_opts;
uint8_t sdhost[16];
int sdport;
log_dog_operation(argc, argv);
install_crash_handler(crash_handler);
init_commands(&commands);
if (argc < 2)
usage(commands, 0);
flags = setup_commands(commands, argv[1], argv[2]);
optind = 3;
sd_opts = build_sd_options(command_opts);
long_options = build_long_options(sd_opts);
short_options = build_short_options(sd_opts);
while ((ch = getopt_long(argc, argv, short_options, long_options,
&longindex)) >= 0) {
switch (ch) {
case 'a':
if (!str_to_addr(optarg, sdhost)) {
sd_err("Invalid ip address %s", optarg);
return EXIT_FAILURE;
}
memcpy(sd_nid.addr, sdhost, sizeof(sdhost));
break;
case 'p':
sdport = strtol(optarg, &p, 10);
if (optarg == p || sdport < 1 || sdport > UINT16_MAX) {
sd_err("Invalid port number '%s'", optarg);
exit(EXIT_USAGE);
}
sd_nid.port = sdport;
break;
case 'r':
raw_output = true;
break;
case 'v':
verbose = true;
break;
case 'h':
subcommand_usage(argv[1], argv[2], EXIT_SUCCESS);
break;
case '?':
usage(commands, EXIT_USAGE);
break;
default:
if (command_parser)
command_parser(ch, optarg);
else
usage(commands, EXIT_USAGE);
break;
}
}
if (!is_stdout_console() || raw_output)
highlight = false;
if (flags & CMD_NEED_NODELIST) {
ret = update_node_list(SD_MAX_NODES);
if (ret < 0) {
sd_err("Failed to get node list");
exit(EXIT_SYSFAIL);
}
}
if (flags & CMD_NEED_ARG && argc == optind)
subcommand_usage(argv[1], argv[2], EXIT_USAGE);
if (init_event(EPOLL_SIZE) < 0)
exit(EXIT_SYSFAIL);
if (init_work_queue(get_nr_nodes) != 0) {
sd_err("Failed to init work queue");
exit(EXIT_SYSFAIL);
}
if (sockfd_init()) {
sd_err("sockfd_init() failed");
exit(EXIT_SYSFAIL);
}
ret = command_fn(argc, argv);
if (ret == EXIT_USAGE)
subcommand_usage(argv[1], argv[2], EXIT_USAGE);
return ret;
}
int main(int argc, char **argv)
{
int ch, longindex, ret;
unsigned long flags;
struct option *long_options;
const struct command *commands;
const char *short_options;
char *p, *env;
const struct sd_option *sd_opts;
uint8_t sdhost[16];
int sdport;
struct timespec start, end;
start = get_time_tick();
log_dog_operation(argc, argv);
install_crash_handler(crash_handler);
init_commands(&commands);
if (argc < 2)
usage(commands, 0);
flags = setup_commands(commands, argv[1], argv[2]);
optind = 3;
sd_opts = build_sd_options(command_opts);
long_options = build_long_options(sd_opts);
short_options = build_short_options(sd_opts);
env = getenv("SHEEPDOG_DOG_ADDR");
if (env) {
if (!str_to_addr(env, sdhost)) {
sd_err("Invalid ip address %s", env);
return EXIT_FAILURE;
}
memcpy(sd_nid.addr, sdhost, sizeof(sdhost));
}
env = getenv("SHEEPDOG_DOG_PORT");
if (env) {
sdport = strtol(env, &p, 10);
if (env == p || sdport < 1 || sdport > UINT16_MAX
|| !is_numeric(env)) {
sd_err("Invalid port number '%s'", env);
exit(EXIT_USAGE);
}
sd_nid.port = sdport;
}
while ((ch = getopt_long(argc, argv, short_options, long_options,
&longindex)) >= 0) {
switch (ch) {
case 'a':
if (!str_to_addr(optarg, sdhost)) {
sd_err("Invalid ip address %s", optarg);
return EXIT_FAILURE;
}
memcpy(sd_nid.addr, sdhost, sizeof(sdhost));
break;
case 'p':
sdport = strtol(optarg, &p, 10);
if (optarg == p || sdport < 1 || sdport > UINT16_MAX
|| !is_numeric(optarg)) {
sd_err("Invalid port number '%s'", optarg);
exit(EXIT_USAGE);
}
sd_nid.port = sdport;
break;
case 'r':
raw_output = true;
break;
case 'v':
verbose = true;
break;
case 'h':
subcommand_usage(argv[1], argv[2], EXIT_SUCCESS);
break;
case 'T':
elapsed_time = true;
break;
case '?':
usage(commands, EXIT_USAGE);
break;
default:
if (command_parser)
command_parser(ch, optarg);
else
usage(commands, EXIT_USAGE);
break;
}
}
if (sd_inode_actor_init(dog_bnode_writer, dog_bnode_reader) < 0)
exit(EXIT_SYSFAIL);
if (!is_stdout_console() || raw_output)
highlight = false;
if (flags & CMD_NEED_NODELIST) {
ret = update_node_list(SD_MAX_NODES);
if (ret < 0) {
sd_err("Failed to get node list");
exit(EXIT_SYSFAIL);
}
}
if (flags & CMD_NEED_ARG && argc == optind)
subcommand_usage(argv[1], argv[2], EXIT_USAGE);
if (init_event(EPOLL_SIZE) < 0)
exit(EXIT_SYSFAIL);
if (wq_trace_init() < 0)
exit(EXIT_SYSFAIL);
if (init_work_queue(get_nr_nodes) != 0) {
sd_err("Failed to init work queue");
exit(EXIT_SYSFAIL);
}
if (sockfd_init()) {
sd_err("sockfd_init() failed");
exit(EXIT_SYSFAIL);
}
ret = command_fn(argc, argv);
if (ret == EXIT_USAGE)
subcommand_usage(argv[1], argv[2], EXIT_USAGE);
if (elapsed_time) {
end = get_time_tick();
printf("\nElapsed time: %.3lf seconds\n",
get_time_interval(&start, &end));
}
return ret;
}
vnode_t *vnode_new(const char *root_dir, uint32_t id, enum eVnodeStorageType storage_type, vnode_write_queue_handle_write_cb vnode_write_queue_handle_write)
{
vnode_t *vnode = (vnode_t*)zmalloc(sizeof(vnode_t));
memset(vnode, 0, sizeof(vnode_t));
vnode->id = id;
vnode->storage_type = storage_type;
/*vnode->max_dbsize = 1024L * 1024L * 1024L * 4L;*/
vnode->max_dbsize = 1024L * 1024L * 500L;
/* Create vnode root dir */
sprintf(vnode->root_dir, "%s/%04d", root_dir, id);
if ( mkdir_if_not_exist(vnode->root_dir) != 0 ){
error_log("Cann't create vnode(%d) dir:%s", id, vnode->root_dir);
zfree(vnode);
return NULL;
}
/* datazones */
int i;
for ( i = 0 ; i < MAX_DATAZONES ; i++ ){
vnode->datazones[i] = (datazone_t*)zmalloc(sizeof(datazone_t));
if ( datazone_init(vnode->datazones[i], vnode, i) != 0 ){
zfree(vnode);
return NULL;
}
}
/* Slices DB */
if ( vnode->storage_type >= STORAGE_KVDB ){
// Create Metadata DB.
const char *metadata_dbname = "metadata";
kvdb_t *kvdb_metadata = vnode_open_kvdb(vnode, metadata_dbname);
if ( kvdb_metadata == NULL ){
error_log("MetadataDB create failed. dbname:%s", metadata_dbname);
zfree(vnode);
return NULL;
}
vnode->kvdb_metadata = kvdb_metadata;
uint32_t active_slicedb_id = 0;
if ( kvdb_get_uint32(kvdb_metadata, "active_slicedb_id", &active_slicedb_id) != 0 ){
active_slicedb_id = 0;
}
notice_log("vnode active_slicedb_id:%d", active_slicedb_id);
// Create Slice DB.
for ( int db_id = 0 ; db_id <= active_slicedb_id ; db_id++ ){
slicedb_t *slicedb = vnode_open_slicedb(vnode, db_id);
if ( slicedb != NULL ){
} else {
/*char dbname[NAME_MAX];*/
/*sprintf(dbname, "slice-%03d", db_id);*/
/*kvdb_t *kvdb = vnode_open_kvdb(vnode, dbname);*/
/*if ( kvdb != NULL ){*/
/*vnode->slicedbs[db_id] = slicedb_new(db_id, kvdb, vnode->max_dbsize);*/
/*} else {*/
/*error_log("SliceDB create failed. dbname:%s", dbname);*/
for ( int n = 0 ; n < db_id ; n++ ){
slicedb_free(vnode->slicedbs[n]);
vnode->slicedbs[n] = NULL;
}
zfree(vnode);
return NULL;
}
}
vnode->active_slicedb = vnode->slicedbs[active_slicedb_id];
/*vnode->kvdb = vnode->active_slicedb->kvdb;*/
}
vnode->caching_objects = object_queue_new(object_compare_md5_func);
vnode->received_objects = listCreate();
vnode->received_object_size = 0;
vnode->standby_objects = listCreate();
vnode->standby_object_size = 0;
vnode->write_queue = init_work_queue(vnode_write_queue_handle_write, VNODE_WRITE_QUEUE_INTERVAL);
return vnode;
}
