/*
* rpmemd_fip_getinfo -- obtain fabric interface information
*/
static int
rpmemd_fip_getinfo(struct rpmemd_fip *fip, const char *service,
const char *node, enum rpmem_provider provider)
{
int ret;
struct fi_info *hints = rpmem_fip_get_hints(provider);
if (!hints) {
RPMEMD_LOG(ERR, "getting fabric interface hints");
ret = -1;
goto err_fi_get_hints;
}
ret = fi_getinfo(RPMEM_FIVERSION, node, service, 0,
hints, &fip->fi);
if (ret) {
RPMEMD_FI_ERR(ret, "getting fabric interface information");
goto err_fi_getinfo;
}
if (fip->fi->addr_format != FI_SOCKADDR_IN) {
RPMEMD_LOG(ERR, "unsupported address family -- %d",
fip->fi->addr_format);
goto err_addr_format;
}
fi_freeinfo(hints);
return 0;
err_addr_format:
fi_freeinfo(fip->fi);
err_fi_getinfo:
fi_freeinfo(hints);
err_fi_get_hints:
return ret;
}
/*
* rpmemd_req_close -- handle close request
*/
static int
rpmemd_req_close(struct rpmemd_obc *obc, void *arg)
{
RPMEMD_ASSERT(arg != NULL);
RPMEMD_LOG(NOTICE, "close request");
struct rpmemd *rpmemd = (struct rpmemd *)arg;
rpmemd->closing = 1;
int ret;
int status = 0;
if (!rpmemd->pool) {
RPMEMD_LOG(ERR, "pool not opened");
status = RPMEM_ERR_FATAL;
return rpmemd_obc_close_resp(rpmemd->obc, status);
}
ret = rpmemd_fip_stop(rpmemd);
if (ret) {
status = RPMEM_ERR_FATAL;
} else {
rpmemd_fip_close(rpmemd->fip);
rpmemd_fip_fini(rpmemd->fip);
}
int remove = rpmemd->created && status;
ret = rpmemd_close_pool(rpmemd, remove);
RPMEMD_LOG(NOTICE, "close request response (status = %u)", status);
ret = rpmemd_obc_close_resp(rpmemd->obc, status);
return ret;
}
/*
* rpmemd_req_set_attr -- handle set attributes request
*/
static int
rpmemd_req_set_attr(struct rpmemd_obc *obc, void *arg,
const struct rpmem_pool_attr *pool_attr)
{
RPMEMD_ASSERT(arg != NULL);
RPMEMD_LOG(NOTICE, "set attributes request");
struct rpmemd *rpmemd = (struct rpmemd *)arg;
RPMEMD_ASSERT(rpmemd->pool != NULL);
int ret;
int status = 0;
int err_send = 1;
ret = rpmemd_db_pool_set_attr(rpmemd->pool, pool_attr);
if (ret) {
ret = -1;
status = rpmemd_db_get_status(errno);
goto err_set_attr;
}
RPMEMD_LOG(NOTICE, "new pool attributes:");
rpmemd_print_pool_attr(pool_attr);
ret = rpmemd_obc_set_attr_resp(obc, status);
if (ret)
goto err_set_attr_resp;
return ret;
err_set_attr_resp:
err_send = 0;
err_set_attr:
if (err_send)
ret = rpmemd_obc_set_attr_resp(obc, status);
return ret;
}
/*
* rpmemd_print_pool_attr -- print pool attributes
*/
static void
rpmemd_print_pool_attr(const struct rpmem_pool_attr *attr)
{
if (attr == NULL) {
RPMEMD_LOG(INFO, RPMEMD_LOG_INDENT "NULL");
} else {
RPMEMD_LOG(INFO, RPMEMD_LOG_INDENT "signature: '%s'",
_str(attr->signature));
RPMEMD_LOG(INFO, RPMEMD_LOG_INDENT "major: %u", attr->major);
RPMEMD_LOG(INFO, RPMEMD_LOG_INDENT "compat_features: 0x%x",
attr->compat_features);
RPMEMD_LOG(INFO, RPMEMD_LOG_INDENT "incompat_features: 0x%x",
attr->incompat_features);
RPMEMD_LOG(INFO, RPMEMD_LOG_INDENT "ro_compat_features: 0x%x",
attr->ro_compat_features);
RPMEMD_LOG(INFO, RPMEMD_LOG_INDENT "poolset_uuid: %s",
uuid2str(attr->poolset_uuid));
RPMEMD_LOG(INFO, RPMEMD_LOG_INDENT "uuid: %s",
uuid2str(attr->uuid));
RPMEMD_LOG(INFO, RPMEMD_LOG_INDENT "next_uuid: %s",
uuid2str(attr->next_uuid));
RPMEMD_LOG(INFO, RPMEMD_LOG_INDENT "prev_uuid: %s",
uuid2str(attr->prev_uuid));
}
}
/*
* rpmemd_close_pool -- close pool and remove it if required
*/
static int
rpmemd_close_pool(struct rpmemd *rpmemd, int remove)
{
int ret = 0;
RPMEMD_LOG(NOTICE, "closing pool");
rpmemd_db_pool_close(rpmemd->db, rpmemd->pool);
RPMEMD_LOG(INFO, "pool closed");
if (remove) {
RPMEMD_LOG(NOTICE, "removing '%s'", rpmemd->pool_desc);
ret = rpmemd_db_pool_remove(rpmemd->db,
rpmemd->pool_desc, 0, 0);
if (ret) {
RPMEMD_LOG(ERR, "!removing pool '%s' failed",
rpmemd->pool_desc);
} else {
RPMEMD_LOG(INFO, "removed '%s'", rpmemd->pool_desc);
}
}
free(rpmemd->pool_desc);
return ret;
}
/*
* rpmemd_req_close -- handle close request
*/
static int
rpmemd_req_close(struct rpmemd_obc *obc, void *arg)
{
RPMEMD_ASSERT(arg != NULL);
struct rpmemd *rpmemd = (struct rpmemd *)arg;
rpmemd->closing = 1;
int ret;
int status = 0;
if (!rpmemd->pool) {
RPMEMD_LOG(ERR, "pool not opened");
status = RPMEM_ERR_FATAL;
return rpmemd_obc_close_resp(rpmemd->obc, status);
}
rpmemd_db_pool_close(rpmemd->db, rpmemd->pool);
ret = rpmemd_fip_process_stop(rpmemd->fip);
if (ret) {
RPMEMD_LOG(ERR, "!stopping fip process failed");
status = errno;
}
ret = rpmemd_obc_close_resp(rpmemd->obc, status);
if (!ret)
rpmemd_fip_wait_close(rpmemd->fip, -1);
rpmemd_fip_close(rpmemd->fip);
rpmemd_fip_fini(rpmemd->fip);
return ret;
}
/*
* rpmemd_get_ssh_addr -- returns an address which the ssh connection is
* established on
*
* This function utilizes the SSH_CONNECTION environment variable to retrieve
* the server IP address. See ssh(1) for details.
*/
static char *
rpmemd_get_ssh_addr(void)
{
char *ssh_conn = getenv("SSH_CONNECTION");
if (!ssh_conn) {
RPMEMD_LOG(ERR, "SSH_CONNECTION variable is not set");
return NULL;
}
char *sp = strchr(ssh_conn, ' ');
if (!sp) {
RPMEMD_LOG(ERR, "invalid format of SSH_CONNECTION variable");
return NULL;
}
char *addr = strchr(sp + 1, ' ');
if (!addr) {
RPMEMD_LOG(ERR, "invalid format of SSH_CONNECTION variable");
return NULL;
}
addr++;
sp = strchr(addr, ' ');
if (!sp) {
RPMEMD_LOG(ERR, "invalid format of SSH_CONNECTION variable");
return NULL;
}
*sp = '\0';
return addr;
}
/*
* rpmem_print_pm_policy -- print persistency method policy
*/
static void
rpmem_print_pm_policy(enum rpmem_persist_method persist_method,
int (*persist)(const void *addr, size_t len))
{
RPMEMD_LOG(NOTICE, RPMEMD_LOG_INDENT "persist method: %s",
rpmem_persist_method_to_str(persist_method));
RPMEMD_LOG(NOTICE, RPMEMD_LOG_INDENT "persist flush: %s",
rpmemd_persist_to_str(persist));
}
/*
* rpmemd_print_resp_attr -- print response attributes
*/
static void
rpmemd_print_resp_attr(const struct rpmem_resp_attr *attr)
{
RPMEMD_LOG(NOTICE, "\tport: %u", attr->port);
RPMEMD_LOG(NOTICE, "\trkey: 0x%lx", attr->rkey);
RPMEMD_LOG(NOTICE, "\traddr: 0x%lx", attr->raddr);
RPMEMD_LOG(NOTICE, "\tnlanes: %u", attr->nlanes);
RPMEMD_LOG(NOTICE, "\tpersist method: %s",
rpmem_persist_method_to_str(attr->persist_method));
}
/*
* test_log_messages -- test log messages on specified level
*/
static void
test_log_messages(enum rpmemd_log_level level)
{
rpmemd_log_level = level;
RPMEMD_LOG(ERR, "ERR message on %s level", l2s(level));
RPMEMD_LOG(WARN, "WARN message on %s level", l2s(level));
RPMEMD_LOG(NOTICE, "NOTICE message on %s level", l2s(level));
RPMEMD_LOG(INFO, "INFO message on %s level", l2s(level));
RPMEMD_DBG("DBG message on %s level", l2s(level));
}
/*
* rpmemd_common_fip_init -- initialize fabric provider
*/
static int
rpmemd_common_fip_init(struct rpmemd *rpmemd, const struct rpmem_req_attr *req,
struct rpmem_resp_attr *resp, int *status)
{
/* register the whole pool with header in RDMA */
void *addr = (void *)((uintptr_t)rpmemd->pool->pool_addr);
struct rpmemd_fip_attr fip_attr = {
.addr = addr,
.size = req->pool_size,
.nlanes = req->nlanes,
.nthreads = rpmemd->nthreads,
.provider = req->provider,
.persist_method = rpmemd->persist_method,
.deep_persist = rpmemd_deep_persist,
.ctx = rpmemd
};
const int is_pmem = rpmemd_db_pool_is_pmem(rpmemd->pool);
if (rpmemd_apply_pm_policy(&fip_attr.persist_method, &fip_attr.persist,
is_pmem)) {
*status = RPMEM_ERR_FATAL;
goto err_fip_init;
}
const char *node = rpmem_get_ssh_conn_addr();
enum rpmem_err err;
rpmemd->fip = rpmemd_fip_init(node, NULL, &fip_attr, resp, &err);
if (!rpmemd->fip) {
*status = (int)err;
goto err_fip_init;
}
return 0;
err_fip_init:
return -1;
}
/*
* rpmemd_print_req_attr -- print request attributes
*/
static void
rpmemd_print_req_attr(const struct rpmem_req_attr *req)
{
RPMEMD_LOG(NOTICE, RPMEMD_LOG_INDENT "pool descriptor: '%s'",
_str(req->pool_desc));
RPMEMD_LOG(NOTICE, RPMEMD_LOG_INDENT "pool size: %lu", req->pool_size);
RPMEMD_LOG(NOTICE, RPMEMD_LOG_INDENT "nlanes: %u", req->nlanes);
RPMEMD_LOG(NOTICE, RPMEMD_LOG_INDENT "provider: %s",
rpmem_provider_to_str(req->provider));
}
/*
* rpmemd_fip_stop_thread -- stop background thread for in-band connection
*/
static int
rpmemd_fip_stop_thread(struct rpmemd *rpmemd)
{
RPMEMD_ASSERT(rpmemd->fip_running);
void *tret;
errno = pthread_join(rpmemd->fip_thread, &tret);
if (errno)
RPMEMD_LOG(ERR, "!waiting for in-band thread");
int ret = (int)(uintptr_t)tret;
if (ret)
RPMEMD_LOG(ERR, "in-band thread failed -- '%d'", ret);
return ret;
}
/*
* print_help -- (internal) prints help message
*/
static void
print_help(const char *name)
{
print_usage(name);
print_version();
RPMEMD_LOG(ERR, help_str, DAEMON_NAME);
}
/*
* rpmemd_fip_cq_thread -- completion queue worker thread
*/
static void *
rpmemd_fip_cq_thread(void *arg)
{
struct rpmemd_fip *fip = arg;
struct fi_cq_err_entry err;
const char *str_err;
ssize_t sret;
int ret = 0;
while (!fip->closing) {
sret = fi_cq_sread(fip->cq, fip->cq_entries,
fip->cq_size, NULL,
RPMEM_FIP_CQ_WAIT_MS);
if (unlikely(fip->closing))
break;
if (unlikely(sret == -FI_EAGAIN))
continue;
if (unlikely(sret < 0)) {
ret = (int)sret;
goto err_cq_read;
}
for (ssize_t i = 0; i < sret; i++) {
struct fi_cq_msg_entry *entry = &fip->cq_entries[i];
RPMEMD_ASSERT(entry->op_context);
struct rpmemd_fip_lane *lanep = entry->op_context;
/* signal lane about SEND completion */
if (entry->flags & FI_SEND)
rpmem_fip_lane_signal(&lanep->lane, FI_SEND);
/* add lane to worker's ring buffer */
if (entry->flags & FI_RECV) {
ret = rpmemd_fip_worker_push(lanep->worker,
lanep);
}
if (ret)
goto err;
}
}
return 0;
err_cq_read:
sret = fi_cq_readerr(fip->cq, &err, 0);
if (sret < 0) {
RPMEMD_FI_ERR((int)sret, "error reading from completion queue: "
"cannot read error from completion queue");
goto err;
}
str_err = fi_cq_strerror(fip->cq, err.prov_errno, NULL, NULL, 0);
RPMEMD_LOG(ERR, "error reading from completion queue: %s", str_err);
err:
return (void *)(uintptr_t)ret;
}
/*
* rpmemd_fip_set_resp -- fill the response structure
*/
static int
rpmemd_fip_set_resp(struct rpmemd_fip *fip, struct rpmem_resp_attr *resp)
{
int ret;
struct sockaddr_in addr_in;
size_t addrlen = sizeof(addr_in);
ret = fi_getname(&fip->pep->fid, &addr_in, &addrlen);
if (ret) {
RPMEMD_FI_ERR(ret, "getting local endpoint address");
goto err_fi_getname;
}
if (!addr_in.sin_port) {
RPMEMD_LOG(ERR, "dynamic allocation of port failed");
goto err_port;
}
resp->port = htons(addr_in.sin_port);
resp->rkey = fi_mr_key(fip->mr);
resp->persist_method = fip->persist_method;
resp->raddr = (uint64_t)fip->addr;
resp->nlanes = fip->nlanes;
return 0;
err_port:
err_fi_getname:
return -1;
}
/*
* rpmemd_print_pool_attr -- print pool attributes
*/
static void
rpmemd_print_pool_attr(const struct rpmem_pool_attr *attr)
{
RPMEMD_LOG(INFO, "\tsignature: '%s'", _str(attr->signature));
RPMEMD_LOG(INFO, "\tmajor: %u", attr->major);
RPMEMD_LOG(INFO, "\tcompat_features: 0x%x", attr->compat_features);
RPMEMD_LOG(INFO, "\tincompat_features: 0x%x", attr->incompat_features);
RPMEMD_LOG(INFO, "\tro_compat_features: 0x%x",
attr->ro_compat_features);
RPMEMD_LOG(INFO, "\tpoolset_uuid: %s", uuid2str(attr->poolset_uuid));
RPMEMD_LOG(INFO, "\tuuid: %s", uuid2str(attr->uuid));
RPMEMD_LOG(INFO, "\tnext_uuid: %s", uuid2str(attr->next_uuid));
RPMEMD_LOG(INFO, "\tprev_uuid: %s", uuid2str(attr->prev_uuid));
}
/*
* rpmemd_fip_process_stop_gpspm -- stop processing GPSPM messages
*/
static int
rpmemd_fip_process_stop_gpspm(struct rpmemd_fip *fip)
{
int lret = 0;
/* this stops all worker threads */
fip->closing = 1;
/*
* Signal all lanes that SEND has been completed.
* Some workers may still be waiting for this completion.
*/
for (unsigned i = 0; i < fip->nlanes; i++)
rpmem_fip_lane_signal(&fip->lanes[i].lane, FI_SEND);
void *tret;
int ret;
errno = pthread_join(fip->cq_thread, &tret);
if (errno) {
RPMEMD_LOG(ERR, "!joining cq thread");
lret = -1;
} else {
ret = (int)(uintptr_t)tret;
if (ret) {
RPMEMD_LOG(ERR, "cq thread failed with "
"code -- %d", ret);
lret = ret;
}
}
free(fip->cq_entries);
for (size_t i = 0; i < fip->nthreads; i++) {
ret = rpmemd_fip_worker_fini(fip->workers[i]);
if (ret) {
RPMEMD_LOG(ERR, "worker failed with code -- %d", ret);
lret = ret;
}
}
free(fip->workers);
return lret;
}
/*
* rpmemd_check_pool -- verify pool parameters
*/
static int
rpmemd_check_pool(struct rpmemd *rpmemd, const struct rpmem_req_attr *req,
int *status)
{
if (rpmemd->pool->pool_size < RPMEM_MIN_POOL) {
RPMEMD_LOG(ERR, "invalid pool size -- must be >= %zu",
RPMEM_MIN_POOL);
*status = RPMEM_ERR_FATAL;
return -1;
}
if (rpmemd->pool->pool_size - POOL_HDR_SIZE < req->pool_size) {
RPMEMD_LOG(ERR, "requested size is too big");
*status = RPMEM_ERR_BADSIZE;
return -1;
}
return 0;
}
/*
* rpmemd_common_fip_init -- initialize fabric provider
*/
static int
rpmemd_common_fip_init(struct rpmemd *rpmemd, const struct rpmem_req_attr *req,
struct rpmem_resp_attr *resp, int *status)
{
void *addr = (void *)((uintptr_t)rpmemd->pool->pool_addr +
POOL_HDR_SIZE);
struct rpmemd_fip_attr fip_attr = {
.addr = addr,
.size = req->pool_size,
.nlanes = req->nlanes,
.nthreads = rpmemd->nthreads,
.provider = req->provider,
.persist_method = rpmemd->persist_method,
.persist = rpmemd->persist,
};
const char *node = rpmem_get_ssh_conn_addr();
enum rpmem_err err;
rpmemd->fip = rpmemd_fip_init(node, NULL, &fip_attr, resp, &err);
if (!rpmemd->fip) {
*status = (int)err;
goto err_fip_init;
}
return 0;
err_fip_init:
return -1;
}
/*
* rpmemd_print_req_attr -- print request attributes
*/
static void
rpmemd_print_req_attr(const struct rpmem_req_attr *req)
{
RPMEMD_LOG(NOTICE, "\tpool descriptor: '%s'", _str(req->pool_desc));
RPMEMD_LOG(NOTICE, "\tpool size: %lu", req->pool_size);
RPMEMD_LOG(NOTICE, "\tnlanes: %u", req->nlanes);
RPMEMD_LOG(NOTICE, "\tprovider: %s",
rpmem_provider_to_str(req->provider));
}
/*
* rpmemd_get_nthreads -- returns number of threads to use for fabric
* processing
*/
static size_t
rpmemd_get_nthreads(void)
{
long ncpus = sysconf(_SC_NPROCESSORS_ONLN);
if (ncpus < 0) {
RPMEMD_LOG(ERR, "getting number of CPUs");
return 0;
}
return (size_t)ncpus;
}
/*
* rpmemd_fip_check_pmsg -- verify persist message
*/
static inline int
rpmemd_fip_check_pmsg(struct rpmemd_fip *fip, struct rpmem_msg_persist *pmsg)
{
if (pmsg->lane >= fip->nlanes) {
RPMEMD_LOG(ERR, "invalid lane number -- %lu", pmsg->lane);
return -1;
}
uintptr_t raddr = pmsg->addr;
uintptr_t laddr = (uintptr_t)fip->addr;
if (raddr < laddr || raddr + pmsg->size > laddr + fip->size) {
RPMEMD_LOG(ERR, "invalid address or size requested "
"for persist operation (0x%lx, %lu)",
raddr, pmsg->size);
return -1;
}
return 0;
}
/*
* rpmemd_check_pool -- verify pool parameteres
*/
static int
rpmemd_check_pool(struct rpmemd *rpmemd, const struct rpmem_req_attr *req,
int *status)
{
if (rpmemd->pool->pool_size - POOL_HDR_SIZE < req->pool_size) {
RPMEMD_LOG(ERR, "requested size is too big");
*status = RPMEM_ERR_BADSIZE;
return -1;
}
return 0;
}
/*
* rpmemd_fip-stop -- stop in-band thread and stop processing thread
*/
static int
rpmemd_fip_stop(struct rpmemd *rpmemd)
{
int ret;
int fip_ret = rpmemd_fip_stop_thread(rpmemd);
if (fip_ret) {
RPMEMD_LOG(ERR, "!in-band thread failed");
}
if (!fip_ret) {
ret = rpmemd_fip_process_stop(rpmemd->fip);
if (ret) {
RPMEMD_LOG(ERR, "!stopping fip process failed");
}
}
rpmemd->fip_running = 0;
return fip_ret;
}
/*
* rpmemd_fip_thread -- background thread for establishing in-band connection
*/
static void *
rpmemd_fip_thread(void *arg)
{
struct rpmemd *rpmemd = (struct rpmemd *)arg;
int ret;
RPMEMD_LOG(INFO, "waiting for in-band connection");
ret = rpmemd_fip_accept(rpmemd->fip, RPMEM_ACCEPT_TIMEOUT);
if (ret)
goto err_accept;
RPMEMD_LOG(NOTICE, "in-band connection established");
ret = rpmemd_fip_process_start(rpmemd->fip);
if (ret)
goto err_process_start;
return NULL;
err_process_start:
rpmemd_fip_close(rpmemd->fip);
err_accept:
return (void *)(uintptr_t)ret;
}
/*
* rpmemd_fip_start_thread -- start background thread for establishing
* in-band connection
*/
static int
rpmemd_fip_start_thread(struct rpmemd *rpmemd)
{
errno = pthread_create(&rpmemd->fip_thread, NULL,
rpmemd_fip_thread, rpmemd);
if (errno) {
RPMEMD_LOG(ERR, "!creating in-band thread");
goto err_pthread_create;
}
rpmemd->fip_running = 1;
return 0;
err_pthread_create:
return -1;
}
/*
* rpmemd_apply_pm_policy -- choose the persistency method and the flush
* function according to the pool type and the persistency method read from the
* config
*/
int
rpmemd_apply_pm_policy(enum rpmem_persist_method *persist_method,
int (**persist)(const void *addr, size_t len),
void *(**memcpy_persist)(void *pmemdest, const void *src, size_t len),
const int is_pmem)
{
switch (*persist_method) {
case RPMEM_PM_APM:
if (is_pmem) {
*persist_method = RPMEM_PM_APM;
*persist = rpmemd_flush_fatal;
} else {
*persist_method = RPMEM_PM_GPSPM;
*persist = pmem_msync;
}
break;
case RPMEM_PM_GPSPM:
*persist_method = RPMEM_PM_GPSPM;
*persist = is_pmem ? rpmemd_pmem_persist : pmem_msync;
break;
default:
RPMEMD_FATAL("invalid persist method: %d", *persist_method);
return -1;
}
/* this is for RPMEM_PERSIST_INLINE */
if (is_pmem)
*memcpy_persist = pmem_memcpy_persist;
else
*memcpy_persist = rpmem_memcpy_msync;
RPMEMD_LOG(NOTICE, "persistency policy:");
rpmem_print_pm_policy(*persist_method, *persist);
return 0;
}
/*
* rpmemd_print_info -- print basic info and configuration
*/
static void
rpmemd_print_info(struct rpmemd *rpmemd)
{
RPMEMD_LOG(NOTICE, "ssh connection: %s",
_str(getenv("SSH_CONNECTION")));
RPMEMD_LOG(NOTICE, "user: %s", _str(getenv("USER")));
RPMEMD_LOG(NOTICE, "configuration");
RPMEMD_LOG(NOTICE, "\tpool set directory: '%s'",
_str(rpmemd->config.poolset_dir));
RPMEMD_LOG(NOTICE, "\tpersist method: %s",
rpmem_persist_method_to_str(rpmemd->persist_method));
RPMEMD_LOG(NOTICE, "\tnumber of threads: %lu", rpmemd->nthreads);
RPMEMD_DBG("\tpersist APM: %s",
bool2str(rpmemd->config.persist_apm));
RPMEMD_DBG("\tpersist GPSPM: %s",
bool2str(rpmemd->config.persist_general));
RPMEMD_DBG("\tuse syslog: %s", bool2str(rpmemd->config.use_syslog));
RPMEMD_DBG("\tlog file: %s", _str(rpmemd->config.log_file));
RPMEMD_DBG("\tlog level: %s",
rpmemd_log_level_to_str(rpmemd->config.log_level));
}
int
main(int argc, char *argv[])
{
util_init();
int send_status = 1;
int ret = 1;
struct rpmemd *rpmemd = calloc(1, sizeof(*rpmemd));
if (!rpmemd) {
RPMEMD_LOG(ERR, "!calloc");
goto err_rpmemd;
}
rpmemd->obc = rpmemd_obc_init(STDIN_FILENO, STDOUT_FILENO);
if (!rpmemd->obc) {
RPMEMD_LOG(ERR, "out-of-band connection intitialization");
goto err_obc;
}
if (rpmemd_log_init(DAEMON_NAME, NULL, 0)) {
RPMEMD_LOG(ERR, "logging subsystem initialization failed");
goto err_log_init;
}
if (rpmemd_config_read(&rpmemd->config, argc, argv) != 0) {
RPMEMD_LOG(ERR, "reading configuration failed");
goto err_config;
}
rpmemd_log_close();
rpmemd_log_level = rpmemd->config.log_level;
if (rpmemd_log_init(DAEMON_NAME, rpmemd->config.log_file,
rpmemd->config.use_syslog)) {
RPMEMD_LOG(ERR, "logging subsystem initialization"
" failed (%s, %d)", rpmemd->config.log_file,
rpmemd->config.use_syslog);
goto err_log_init_config;
}
RPMEMD_LOG(INFO, "%s version %s", DAEMON_NAME, SRCVERSION);
rpmemd->persist = pmem_persist;
rpmemd->persist_method = rpmemd_get_pm(&rpmemd->config);
rpmemd->nthreads = rpmemd_get_nthreads();
if (!rpmemd->nthreads) {
RPMEMD_LOG(ERR, "invalid number of threads -- '%lu'",
rpmemd->nthreads);
goto err_nthreads;
}
rpmemd->db = rpmemd_db_init(rpmemd->config.poolset_dir, 0666);
if (!rpmemd->db) {
RPMEMD_LOG(ERR, "!pool set db initialization");
goto err_db_init;
}
if (rpmemd->config.rm_poolset) {
RPMEMD_LOG(INFO, "removing '%s'",
rpmemd->config.rm_poolset);
if (rpmemd_db_pool_remove(rpmemd->db,
rpmemd->config.rm_poolset,
rpmemd->config.force)) {
RPMEMD_LOG(ERR, "removing '%s' failed",
rpmemd->config.rm_poolset);
} else {
RPMEMD_LOG(NOTICE, "removed '%s'",
rpmemd->config.rm_poolset);
ret = 0;
}
send_status = 0;
goto out_rm;
}
ret = rpmemd_obc_status(rpmemd->obc, 0);
if (ret) {
RPMEMD_LOG(ERR, "writing status failed");
goto err_status;
}
rpmemd_print_info(rpmemd);
while (!ret) {
ret = rpmemd_obc_process(rpmemd->obc, &rpmemd_req, rpmemd);
if (ret) {
RPMEMD_LOG(ERR, "out-of-band connection"
" process failed");
goto err;
}
if (rpmemd->closing)
break;
}
rpmemd_db_fini(rpmemd->db);
rpmemd_config_free(&rpmemd->config);
rpmemd_log_close();
rpmemd_obc_fini(rpmemd->obc);
free(rpmemd);
return 0;
err:
rpmemd_req_cleanup(rpmemd);
err_status:
out_rm:
rpmemd_db_fini(rpmemd->db);
err_db_init:
err_nthreads:
err_log_init_config:
rpmemd_config_free(&rpmemd->config);
err_config:
rpmemd_log_close();
err_log_init:
if (send_status) {
if (rpmemd_obc_status(rpmemd->obc, (uint32_t)errno))
RPMEMD_LOG(ERR, "writing status failed");
}
rpmemd_obc_fini(rpmemd->obc);
err_obc:
free(rpmemd);
err_rpmemd:
return ret;
}
/*
* rpmemd_fip_init -- initialize fabric provider
*/
struct rpmemd_fip *
rpmemd_fip_init(const char *node, const char *service,
struct rpmemd_fip_attr *attr, struct rpmem_resp_attr *resp,
enum rpmem_err *err)
{
int ret;
RPMEMD_ASSERT(resp);
RPMEMD_ASSERT(err);
RPMEMD_ASSERT(attr);
RPMEMD_ASSERT(attr->persist);
RPMEMD_ASSERT(attr->nthreads);
struct rpmemd_fip *fip = calloc(1, sizeof(*fip));
if (!fip) {
RPMEMD_LOG(ERR, "!allocating fabric handle");
*err = RPMEM_ERR_FATAL;
return NULL;
}
ret = rpmemd_fip_getinfo(fip, service, node, attr->provider);
if (ret) {
*err = RPMEM_ERR_BADPROVIDER;
goto err_getinfo;
}
rpmemd_fip_set_attr(fip, attr);
ret = rpmemd_fip_init_fabric_res(fip);
if (ret) {
*err = RPMEM_ERR_FATAL;
goto err_init_fabric_res;
}
ret = rpmemd_fip_init_memory(fip);
if (ret) {
*err = RPMEM_ERR_FATAL;
goto err_init_memory;
}
ret = fip->ops->init(fip);
if (ret) {
*err = RPMEM_ERR_FATAL;
goto err_init;
}
ret = fi_listen(fip->pep);
if (ret) {
*err = RPMEM_ERR_FATAL_CONN;
goto err_fi_listen;
}
ret = rpmemd_fip_set_resp(fip, resp);
if (ret) {
*err = RPMEM_ERR_FATAL;
goto err_set_resp;
}
return fip;
err_set_resp:
RPMEMD_FI_CLOSE(fip->pep, "closing passive endpoint");
err_fi_listen:
fip->ops->fini(fip);
err_init:
rpmemd_fip_fini_memory(fip);
err_init_memory:
rpmemd_fip_fini_fabric_res(fip);
err_init_fabric_res:
fi_freeinfo(fip->fi);
err_getinfo:
free(fip);
return NULL;
}
/*
* rpmemd_req_open -- handle open request
*/
static int
rpmemd_req_open(struct rpmemd_obc *obc, void *arg,
const struct rpmem_req_attr *req)
{
RPMEMD_ASSERT(arg != NULL);
RPMEMD_LOG(NOTICE, "open request:");
rpmemd_print_req_attr(req);
struct rpmemd *rpmemd = (struct rpmemd *)arg;
int ret;
int status = 0;
int err_send = 1;
struct rpmem_resp_attr resp;
memset(&resp, 0, sizeof(resp));
struct rpmem_pool_attr pool_attr;
memset(&pool_attr, 0, sizeof(pool_attr));
if (rpmemd->pool) {
RPMEMD_LOG(ERR, "pool already opened");
ret = -1;
status = RPMEM_ERR_FATAL;
goto err_pool_opened;
}
rpmemd->pool_desc = strdup(req->pool_desc);
if (!rpmemd->pool_desc) {
RPMEMD_LOG(ERR, "!allocating pool descriptor");
ret = -1;
status = RPMEM_ERR_FATAL;
goto err_strdup;
}
rpmemd->pool = rpmemd_db_pool_open(rpmemd->db,
req->pool_desc, 0, &pool_attr);
if (!rpmemd->pool) {
ret = -1;
status = rpmemd_db_get_status(errno);
goto err_pool_open;
}
RPMEMD_LOG(NOTICE, "pool attributes:");
rpmemd_print_pool_attr(&pool_attr);
ret = rpmemd_check_pool(rpmemd, req, &status);
if (ret)
goto err_pool_check;
ret = rpmemd_common_fip_init(rpmemd, req, &resp, &status);
if (ret)
goto err_fip_init;
RPMEMD_LOG(NOTICE, "open request response: (status = %u)", status);
if (!status)
rpmemd_print_resp_attr(&resp);
ret = rpmemd_obc_open_resp(rpmemd->obc, status, &resp, &pool_attr);
if (ret)
goto err_open_resp;
ret = rpmemd_fip_start_thread(rpmemd);
if (ret)
goto err_fip_start;
return 0;
err_fip_start:
err_open_resp:
err_send = 0;
rpmemd_fip_fini(rpmemd->fip);
err_fip_init:
err_pool_check:
rpmemd_db_pool_close(rpmemd->db, rpmemd->pool);
err_pool_open:
free(rpmemd->pool_desc);
err_strdup:
err_pool_opened:
if (err_send)
ret = rpmemd_obc_open_resp(rpmemd->obc, status,
&resp, &pool_attr);
rpmemd->closing = 1;
return ret;
}
