void *si_new(uint64_t nh, uint64_t ne, uint64_t nb) {
SI *si = am_malloc(sizeof(SI));
if (si == NULL)
{
errno = ENOMEM;
KV_TRC_FFDC(pAT, "FFDC1: nh %ld ne %ld nb %ld, rc = %d",
nh, ne, nb, errno);
return NULL;
}
si->nh = nh;
si->ne = ne;
si->nb = nb;
si->ent_next = 0;
si->gid_next = 0;
si->dat_next = 0;
si->tbl = am_malloc(nh * sizeof(uint64_t));
if ( si->tbl == NULL )
{
errno = ENOMEM;
KV_TRC_FFDC(pAT, "FFDC2: nh %ld ne %ld nb %ld, rc = %d",
nh, ne, nb, errno);
}
else
{
memset(si->tbl, 0xFF, nh * sizeof(uint64_t));
si->dat = am_malloc(nb);
if (si->dat == NULL)
{
errno = ENOMEM;
KV_TRC_FFDC(pAT, "FFDC3: nh %ld ne %ld nb %ld, rc = %d",
nh, ne, nb, errno);
am_free(si->tbl);
am_free(si);
si = NULL;
}
else
{
si->ent = am_malloc(ne * sizeof(SIE));
if (si->ent == NULL)
{
errno = ENOMEM;
KV_TRC_FFDC(pAT, "FFDC4: nh %ld ne %ld nb %ld, rc = %d",
nh, ne, nb, errno);
am_free(si->tbl);
am_free(si->dat);
am_free(si);
si = NULL;
}
}
}
memset(si->tbl, 0xFF, nh * sizeof(uint64_t));
return si;
}
/** \brief Create a Transmission-specific JSON packet in order to add a new download
* to Transmission.
*
* \param[in] data Pointer to the torrent data
* \param[in] tsize size of the torrent data
* \param[in] start Determines if the torrent shall start to download right away or be added in a paused state
* \param[in] folder Optional parameter to set the download folder for this torrent
* \param[out] setme_size size of the resulting JSON packet
* \return pointer to the JSON string
*
* The function Base64-encodes the given torrent content and encapsulates it in a JSON packet.
* The packet can then be sent to Transmission via HTTP POST.
*/
char* makeTorrentAddMetaInfoJSON(const void *data, uint32_t tsize, uint8_t start, const char* folder, uint32_t *setme_size) {
char *encoded = NULL;
char *buf = NULL;
char *folder_str = NULL;
int buf_size, json_size, folderstr_size = 0;
uint32_t enc_size;
const char *JSONstr =
"{\n"
"\"method\": \"torrent-add\",\n"
"\"arguments\": {\n"
"\"metainfo\": \"%s\",\n"
"%s"
"\"paused\": %d\n"
"}\n"
"}";
*setme_size = 0;
encoded = base64_encode((const char*)data, tsize, &enc_size);
if(encoded && enc_size > 0) {
if(folder && *folder) {
folderstr_size = strlen(folder) + 20;
folder_str = (char*)am_malloc(folderstr_size);
assert(folder_str && "am_malloc(folder_str) failed!");
snprintf(folder_str, folderstr_size, "\"download-dir\": \"%s\",\n", folder);
dbg_printf(P_INFO, "folder_str: %s", folder_str);
}
buf_size = enc_size + strlen(JSONstr) + folderstr_size + 10;
buf = (char*)am_malloc(buf_size);
memset(buf, 0, buf_size);
json_size = snprintf(buf, buf_size, JSONstr, encoded, folder_str ? folder_str : "", start ? 0 : 1);
if(json_size < 0 || json_size >= buf_size) {
dbg_printf(P_ERROR, "Error producing JSON string with Base64-encoded metadata: %s", strerror(errno));
am_free(encoded);
am_free(buf);
return NULL;
}
buf[json_size] = '\0';
dbg_printf(P_INFO2, "JSON: %s", buf);
if(setme_size) {
*setme_size = json_size;
}
am_free(folder_str);
am_free(encoded);
return buf;
}
return NULL;
}
PRIVATE size_t write_data_callback(void *ptr, size_t size, size_t nmemb, void *data) {
size_t line_len = size * nmemb;
WebData *mem = data;
/**
* if content-length detection in write_header_callback was not successful, mem->response->data will be NULL
* as a fallback, allocate a predefined size of memory and realloc if necessary
**/
if(!mem->response->data) {
mem->response->buffer_size = DATA_BUFFER_SIZE;
mem->response->data = (char*)am_malloc(mem->response->buffer_size);
dbg_printf(P_INFO2, "[write_data_callback] allocated %d bytes for mem->response->data", mem->response->buffer_size);
}
if(mem->response->buffer_pos + line_len + 1 > mem->response->buffer_size) {
do {
mem->response->buffer_size *= 2;
}while(mem->response->buffer_size < mem->response->buffer_pos + line_len + 1);
mem->response->data = (char *)am_realloc(mem->response->data, mem->response->buffer_size);
}
if(mem->response->data) {
memcpy(&(mem->response->data[mem->response->buffer_pos]), ptr, line_len);
mem->response->buffer_pos += line_len;
mem->response->data[mem->response->buffer_pos] = 0;
}
return line_len;
}
/**
*******************************************************************************
* \brief
******************************************************************************/
IV *iv_new(uint64_t n, uint64_t m) {
uint64_t bits = n * m;
uint64_t words = divup(bits, 64);
uint64_t bytes = sizeof(IV) + words * sizeof(uint64_t);
IV *iv = am_malloc(bytes);
if (iv == NULL)
{
errno = ENOMEM;
KV_TRC_FFDC(pAT, "FFDC: n %"PRIu64" m %"PRIu64", errno = %d", n, m, errno);
}
else
{
memset(iv,0x00, bytes);
iv->n = n;
iv->m = m;
iv->bits = bits;
iv->words = words;
iv->mask = 1;
iv->mask <<= m;
iv->mask -= 1;
iv->bar = 64 - m;
}
KV_TRC(pAT, "iv:%p n:%ld m:%ld", iv, n, m);
return iv;
}
auto_handle* session_init(void) {
char path[MAXPATHLEN];
char *home;
auto_handle *ses = am_malloc(sizeof(auto_handle));
/* numbers */
ses->max_bucket_items = AM_DEFAULT_MAXBUCKET;
ses->bucket_changed = 0;
ses->check_interval = AM_DEFAULT_INTERVAL;
/* strings */
ses->download_folder = get_temp_folder();
home = get_home_folder();
sprintf(path, "%s/%s", home, AM_DEFAULT_STATEFILE);
am_free(home);
ses->statefile = am_strdup(path);
ses->prowl_key = NULL;
ses->prowl_key_valid = 0;
ses->download_done_script = NULL;
ses->match_only = 0;
/* lists */
ses->filters = NULL;
ses->feeds = NULL;
ses->downloads = NULL;
return ses;
}
static char* createProwlMessage(const char* apikey, const char* event, const char* desc, int32_t *size) {
int32_t result, apikey_length, event_length, desc_length, total_size;
char *msg = NULL;
*size = 0;
if(!apikey) {
dbg_printf(P_ERROR, "[createProwlMessage] apikey == NULL");
*size = 0;
return NULL;
}
if((!event && !desc)) {
dbg_printf(P_ERROR, "[createProwlMessage] event == NULL && desc == NULL");
*size = 0;
return NULL;
}
apikey_length = strlen(apikey);
event_length = event ? strlen(event) : 0;
desc_length = desc ? strlen(desc) : 0;
total_size = apikey_length + event_length + desc_length + 80;
msg = (char*)am_malloc(total_size);
if(msg) {
result = snprintf(msg, total_size, "apikey=%s&priority=0&application=Automatic&event=%s&description=%s",
apikey, event, desc);
*size = result;
}
return msg;
}
kvp_t *
kvp_new (void){
kvp_t *k;
k = am_malloc (sizeof (kvp_t));
k->key = NULL;
k->value = NULL;
return k;
}
/** \brief Create a new RSS feed node
*
* \return Pointer to the new feed node
*/
PUBLIC rss_feed* feed_new(void) {
rss_feed* i = (rss_feed*)am_malloc(sizeof(struct rss_feed));
if(i != NULL) {
i->url = NULL;
i->cookies = NULL;
i->ttl = -1;
}
return i;
}
char* makeChangeUpSpeedJSON(torrent_id_t tID, uint32_t upspeed, uint8_t rpcVersion, uint32_t *setme_size) {
char *buf = NULL;
int buf_size, json_size = 0;
const char *JSONstr =
"{\n"
"\"method\": \"torrent-set\",\n"
"\"arguments\": {\n"
"\"ids\": %d,\n"
"\"%s\": %d,\n"
"\"%s\": true\n"
"}\n"
"}";
*setme_size = 0;
if(rpcVersion <= 0) {
dbg_printf(P_ERROR, "Invalid RPC version: %d", rpcVersion);
return NULL;
}
if(upspeed <= 0) {
dbg_printf(P_ERROR, "Invalid upspeed value: %d", upspeed);
return NULL;
}
if(tID <= 0) {
dbg_printf(P_ERROR, "Invalid torrent ID: %d", tID);
return NULL;
}
buf_size = strlen(JSONstr) + 60;
buf = am_malloc(buf_size);
if(!buf) {
dbg_printf(P_DBG, "Mem alloc for JSON string failed!");
return NULL;
}
memset(buf, 0, buf_size);
if(rpcVersion <= 4) {
json_size = snprintf(buf, buf_size, JSONstr, tID, "speed-limit-up", upspeed, "speed-limit-up-enabled");
} else if(rpcVersion >= 5 ) {
json_size = snprintf(buf, buf_size, JSONstr, tID, "uploadLimit", upspeed, "uploadLimited");
}
if(json_size < 0 || json_size >= buf_size) {
dbg_printf(P_ERROR, "Error producing JSON string with Base64-encoded metadata: %s", strerror(errno));
am_free(buf);
return NULL;
}
buf[json_size] = '\0';
if(setme_size) {
*setme_size = json_size;
}
return buf;
}
/**
* Create a list of expandable variables from envp and possibly argv
*/
am_list_t *
create_expandable_variable_list (MYSQL *mysql, const char *envp[], const char *argv[]){
time_t t;
char *escaped_username = NULL;
char *escaped_password = NULL;
const char *username, *password;
am_list_t *l = am_list_new ();
if (l == NULL)
return l;
t = time (NULL);
username = get_env ("username", envp);
password = get_env ("password", envp);
if (username != NULL){
escaped_username = (char *) am_malloc (sizeof(char) * (strlen (username) * 2 + 1));
mysql_real_escape_string (mysql, escaped_username, username, strlen (username));
}
if (password != NULL){
escaped_password = (char *) am_malloc (sizeof(char) * (strlen (password) * 2 + 1));
mysql_real_escape_string (mysql, escaped_password, password, strlen (password));
}
am_list_append (l, kvp_new_with_kv (EV_DUP ("time_now"), strdupf ("%d", t)));
am_list_append (l, kvp_new_with_kv (EV_DUP ("username"), NULL_OR_DUP (username)));
am_list_append (l, kvp_new_with_kv (EV_DUP ("password"), NULL_OR_DUP (password)));
am_list_append (l, kvp_new_with_kv (EV_DUP ("escaped_username"), NULL_OR_DUP (escaped_username)));
am_list_append (l, kvp_new_with_kv (EV_DUP ("escaped_password"), NULL_OR_DUP (escaped_password)));
am_list_append (l, kvp_new_with_kv (EV_DUP ("trusted_port"), NULL_OR_DUP (get_env ("trusted_port", envp))));
am_list_append (l, kvp_new_with_kv (EV_DUP ("trusted_ip"), NULL_OR_DUP (get_env ("trusted_ip", envp))));
am_list_append (l, kvp_new_with_kv (EV_DUP ("time_unix"), NULL_OR_DUP (get_env ("time_unix", envp))));
am_list_append (l, kvp_new_with_kv (EV_DUP ("ifconfig_pool_remote_ip"), NULL_OR_DUP (get_env ("ifconfig_pool_remote_ip", envp))));
am_list_append (l, kvp_new_with_kv (EV_DUP ("ifconfig_pool_local_ip"), NULL_OR_DUP (get_env ("ifconfig_pool_local_ip", envp))));
am_list_append (l, kvp_new_with_kv (EV_DUP ("ifconfig_local"), NULL_OR_DUP (get_env ("ifconfig_local", envp))));
am_list_append (l, kvp_new_with_kv (EV_DUP ("time_duration"), NULL_OR_DUP (get_env ("time_duration", envp))));
am_list_append (l, kvp_new_with_kv (EV_DUP ("bytes_sent"), NULL_OR_DUP (get_env ("bytes_sent", envp))));
am_list_append (l, kvp_new_with_kv (EV_DUP ("bytes_received"), NULL_OR_DUP (get_env ("bytes_received", envp))));
am_free (escaped_username);
am_free (escaped_password);
return l;
}
PRIVATE HTTPResponse* HTTPResponse_new(void) {
HTTPResponse* resp = (HTTPResponse*)am_malloc(sizeof(struct HTTPResponse));
if(resp) {
resp->size = 0;
resp->responseCode = 0;
resp->data = NULL;
resp->content_filename = NULL;
}
return resp;
}
queue_t *queue_new(uint32_t n) {
queue_t *q = am_malloc(sizeof(queue_t) + n * sizeof(int32_t));
q->n = n;
q->c = 0;
q->h = 0;
q->t = 0;
q->waiters = 0;
pthread_mutex_init(&q->m,NULL);
pthread_cond_init(&q->cond,NULL);
return q;
}
PRIVATE struct HTTPData* HTTPData_new(void) {
HTTPData* data = NULL;
data = am_malloc(sizeof(struct HTTPData));
if(!data) {
return NULL;
}
data->data = NULL;
data->buffer_size = 0;
data->buffer_pos = 0;
return data;
}
PRIVATE char* shorten(const char *str) {
int tmp_pos;
char c;
char *retStr;
char *tmp = (char*)am_malloc(MAX_PARAM_LEN+1);
uint32_t line_pos = 0, i;
uint32_t len = strlen(str);
if(!tmp) {
dbg_printf(P_ERROR, "[shorten] calloc(MAX_PARAM_LEN) failed!");
return NULL;
}
memset(tmp, 0, MAX_PARAM_LEN+1);
while (isspace(str[line_pos])) {
++line_pos;
}
tmp_pos = 0;
while(line_pos < len) {
/* case 1: quoted strings */
if(tmp_pos != 0) {
for(i = 0; i < strlen(AM_DELIMITER); ++i)
tmp[tmp_pos++] = AM_DELIMITER[i];
}
if (str[line_pos] == '"' || str[line_pos] == '\'') {
c = str[line_pos];
++line_pos; /* skip quote */
while(str[line_pos] != c && line_pos < len && str[line_pos] != '\n' && str[line_pos] != '\0') {
tmp[tmp_pos++] = str[line_pos++];
}
if(str[line_pos] == c) {
line_pos++; /* skip the closing quote */
}
} else {
while(line_pos < len && str[line_pos] != '\n' && str[line_pos] != '\0') {
tmp[tmp_pos++] = str[line_pos++];
}
}
while (isspace(str[line_pos])) {
++line_pos;
}
}
tmp[tmp_pos] = '\0';
assert(strlen(tmp) < MAX_PARAM_LEN);
retStr = am_strdup(tmp);
am_free(tmp);
return retStr;
}
int addToTail(void* elem, NODE **head) {
NODE *newnode = NULL;
if(head != NULL && elem != NULL) {
newnode = (NODE*)am_malloc(sizeof(struct NODE));
if(newnode != NULL) {
newnode->data = elem;
newnode->next = NULL;
addNodeLast(head, newnode);
return 0;
}
return -1;
}
return -1;
}
auto_handle* session_init(void) {
char path[MAXPATHLEN];
char *home;
am_session_t *ses = am_malloc(sizeof(am_session_t));
/* numbers */
ses->rpc_version = AM_DEFAULT_RPC_VERSION;
ses->max_bucket_items = AM_DEFAULT_MAXBUCKET;
ses->bucket_changed = 0;
ses->check_interval = AM_DEFAULT_INTERVAL;
ses->use_transmission = AM_DEFAULT_USETRANSMISSION;
ses->start_torrent = AM_DEFAULT_STARTTORRENTS;
ses->transmission_version = AM_TRANSMISSION_1_3;
ses->rpc_port = AM_DEFAULT_RPCPORT;
/* strings */
ses->transmission_path = get_tr_folder();
ses->torrent_folder = get_temp_folder();
ses->host = NULL;
ses->auth = NULL;
home = get_home_folder();
sprintf(path, "%s/%s", home, AM_DEFAULT_STATEFILE);
am_free(home);
ses->statefile = am_strdup(path);
ses->prowl_key = NULL;
ses->toasty_key = NULL;
ses->pushalot_key = NULL;
ses->pushover_key = NULL;
ses->prowl_key_valid = 0;
ses->match_only = 0;
ses->transmission_external = NULL;
/* lists */
ses->filters = NULL;
ses->feeds = NULL;
ses->downloads = NULL;
ses->upspeed = -1;
return ses;
}
PRIVATE suboption_t* parseSubOption(char* line) {
const char *subopt_delim = "=>";
uint32_t i = 0;
suboption_t* option_item = NULL;
char *option = NULL;
char *param = NULL;
assert(line && *line);
while(line[i] != '\0') {
if(line[i] == subopt_delim[0] && line[i+1] == subopt_delim[1]) {
if(i >1) {
option = am_strndup(line, i-1);
param = trim(line + i + strlen(subopt_delim));
} else {
dbg_printf(P_ERROR, "Error: Suboption '%s' is malformed!", line);
}
break;
}
i++;
}
if(option && param) {
option_item = (suboption_t*)am_malloc(sizeof(suboption_t));
option_item->option = option;
option_item->value = param;
}
if(!option_item) {
dbg_printf(P_ERROR, "Error parsing suboption from input string '%s')", line);
am_free(option);
am_free(param);
}
return option_item;
}
/** \brief Create a new WebData object
*
* \param[in] url URL for a WebData object
*
* The parameter \a url is optional. You may provide \c NULL if no URL is required or not known yet.
*/
PRIVATE struct WebData* WebData_new(const char *url) {
WebData *data = NULL;
data = am_malloc(sizeof(WebData));
if(!data)
return NULL;
data->url = NULL;
data->content_filename = NULL;
data->content_length = -1;
data->response = NULL;
if(url) {
data->url = am_strdup((char*)url);
}
data->response = HTTPData_new();
if(!data->response) {
WebData_free(data);
return NULL;
}
return data;
}
/**
* handle the actual query of _am_mysql_handle_pf_rules_single_query
* given the query to execute
*/
int
_am_mysql_handle_pf_rules_do_single_query (
MYSQL *mysql,
char *query,
am_list_t *expandable_vars,
struct pf_rules *pf_rules)
{
MYSQL_RES *result = NULL;
MYSQL_ROW row;
unsigned long *lengths;
int rc = 0;
my_ulonglong num_rows;
char *fmt_query = NULL;
if (query == NULL)
return -1;
fmt_query = expand_query (query, expandable_vars);
if (fmt_query == NULL){
LOGERROR ("Could not set expandable variables in _am_mysql_handle_default_pf_rules_query\n");
return -1;
}
/* Now for the query itself */
if (mysql_query(mysql, fmt_query))
{
/* query failed */
rc = -1;
LOGERROR ("_am_mysql_handle_default_pf_rules_query: Failed to execute query: Error (%d): %s\n", mysql_errno(mysql), mysql_error(mysql));
goto _handle_pf_rules_do_single_query;
}
if ((result = mysql_store_result(mysql)) == NULL)
{
/*
* Failed to get result
* 2 cases:
* - no result expected (after an insert statement...
* - error getting result
*/
if (mysql_field_count (mysql) == 0){
/* insert like statement, should not happen!; */
rc = -1;
goto _handle_pf_rules_do_single_query;
}else{
rc = -1;
LOGERROR ("_am_mysql_handle_default_pf_rules_query: Failed to store query results: Error (%d): %s\n", mysql_errno(mysql), mysql_error(mysql));
goto _handle_pf_rules_do_single_query;
}
}
num_rows = mysql_num_rows(result);
/* If num_rows > 1, we consider it is an error */
if (num_rows > 1 ){
rc = -1;
LOGERROR ("_am_mysql_handle_default_pf_rules_query: Many results were found while only 1 was expected!\n");
goto _handle_pf_rules_do_single_query;
}
if (num_rows == 0){
/* no result found */
rc = 0;
goto _handle_pf_rules_do_single_query;
}
/* If we dont have 4 fields, we should fail too */
if (mysql_num_fields(result) != 4 ){
rc = -1;
LOGERROR ("_am_mysql_handle_default_pf_rules_query: MySQL result do not contain exactly 4 fields!\n");
goto _handle_pf_rules_do_single_query;
}
row = mysql_fetch_row(result);
if (row == NULL){
/* An error occurred */
rc = -1;
LOGERROR ("_am_mysql_handle_default_pf_rules_query: Failed to return first row, Error (%d): %s\n", mysql_errno(mysql), mysql_error(mysql));
goto _handle_pf_rules_do_single_query;
}
lengths = mysql_fetch_lengths(result);
/* no we set the values */
rc = 1;
/* default rules */
pf_rules->default_pf_rules_clients = pf_default_drop_or_accept (row[0]);
pf_rules->default_pf_rules_subnets = pf_default_drop_or_accept (row[1]);
/* clients rules */
if ( lengths[2] != 0 ){
pf_rules->pf_rules_clients = am_malloc (lengths[2] + 1);
if (pf_rules->pf_rules_clients == NULL){
LOGERROR ("Could not allocate memory for pf_rules_clients\n");
}else{
memcpy (pf_rules->pf_rules_clients, row[2], lengths[2]);
pf_rules->pf_rules_clients[lengths[2]] = '\0';
}
}else{
pf_rules->pf_rules_clients = NULL;
}
/* subnets rules */
if ( lengths[3] != 0 ){
pf_rules->pf_rules_subnets = am_malloc (lengths[3] + 1);
if (pf_rules->pf_rules_subnets == NULL){
LOGERROR ("Could not allocate memory for pf_rules_subnets\n");
}else{
memcpy (pf_rules->pf_rules_subnets, row[3], lengths[3]);
pf_rules->pf_rules_subnets[lengths[3]] = '\0';
}
}else{
pf_rules->pf_rules_subnets = NULL;
}
_handle_pf_rules_do_single_query:
if (result != NULL)
mysql_free_result (result);
if (fmt_query != NULL)
am_free (fmt_query);
return rc;
}
/** \brief parse configuration file.
*
* \param[in,out] as Pointer to session handle
* \param[in] filename Path to the configuration file
* \return 0 if parsing was successful, -1 if an error occured.
*/
int parse_config_file(struct auto_handle *as, const char *filename) {
FILE *fp = NULL;
char *line = NULL;
char opt[MAX_OPT_LEN + 1];
char *param = NULL;
char erbuf[100];
char c; /* for the "" and '' check */
int line_num = 0;
int line_pos; /* line pos */
int opt_pos; /* opt pos */
int param_pos; /* param pos */
int parse_error = 0;
int opt_good = 0;
int param_good = 0;
struct stat fs;
option_type type;
if(stat(filename, &fs) == -1) {
return -1;
}
dbg_printf(P_INFO2, "Configuration file size: %d", fs.st_size);
if ((fp = fopen(filename, "rb")) == NULL) {
perror("fopen");
return -1;
}
if ((line = am_malloc(fs.st_size + 1)) == NULL) {
dbg_printf(P_ERROR, "Can't allocate memory for 'line': %s (%ldb)", strerror(errno), fs.st_size + 1);
return -1;
}
if(fread(line, fs.st_size, 1, fp) != 1) {
perror("fread");
fclose(fp);
am_free(line);
return -1;
}
if(fp) {
fclose(fp);
}
line_pos = 0;
param = (char*)am_malloc(MAX_PARAM_LEN + 1);
if(!param) {
dbg_printf(P_ERROR, "Can't allocate memory for 'param': %s (%ldb)", strerror(errno), MAX_PARAM_LEN + 1);
am_free(line);
return -1;
}
while(line_pos != fs.st_size) {
/* skip whitespaces */
while (isspace(line[line_pos])) {
if(line[line_pos] == '\n') {
dbg_printf(P_INFO2, "skipping newline (line %d)", line_num);
line_num++;
}
++line_pos;
}
if(line_pos >= fs.st_size) {
break;
}
/* comment */
if (line[line_pos] == '#') {
dbg_printf(P_INFO2, "skipping comment (line %d)", line_num);
while (line[line_pos] != '\n') {
++line_pos;
}
++line_num;
++line_pos; /* skip the newline as well */
continue;
}
/* read option */
for (opt_pos = 0; isprint(line[line_pos]) && line[line_pos] != ' ' &&
line[line_pos] != '#' && line[line_pos] != '='; /* NOTHING */) {
opt[opt_pos++] = line[line_pos++];
if (opt_pos >= MAX_OPT_LEN) {
dbg_printf(P_ERROR, "too long option at line %d", line_num);
parse_error = 1;
opt_good = 0;
}
}
if (opt_pos == 0 || parse_error == 1) {
dbg_printf(P_ERROR, "parse error at line %d (pos: %d)", line_num, line_pos);
parse_error = 1;
break;
} else {
opt[opt_pos] = '\0';
opt_good = 1;
}
/* skip whitespaces */
while (isspace(line[line_pos])) {
if(line[line_pos] == '\n') {
line_num++;
dbg_printf(P_INFO2, "skipping newline (line %d)", line_num);
}
++line_pos;
}
if(line_pos >= fs.st_size) {
break;
}
/* check for '=' */
if (line[line_pos++] != '=') {
snprintf(erbuf, sizeof(erbuf), "Option '%s' needs a parameter (line %d)", opt, line_num);
parse_error = 1;
break;
}
/* skip whitespaces */
while (isspace(line[line_pos])) {
if(line[line_pos] == '\n') {
line_num++;
dbg_printf(P_INFO2, "skipping newline (line %d)", line_num);
}
++line_pos;
}
if(line_pos >= fs.st_size) {
break;
}
/* read the parameter */
/* case 1: single string, no linebreaks allowed */
if (line[line_pos] == '"' || line[line_pos] == '\'') {
c = line[line_pos]; /* single or double quote */
++line_pos; /* skip quote */
parse_error = 0;
for (param_pos = 0; line[line_pos] != c; /* NOTHING */) {
if(line_pos < fs.st_size && param_pos < MAX_PARAM_LEN && line[line_pos] != '\n') {
param[param_pos++] = line[line_pos++];
} else {
snprintf(erbuf, sizeof(erbuf), "Option %s has a too long parameter (line %d)\n",opt, line_num);
parse_error = 1;
break;
}
}
if(parse_error == 0) {
line_pos++; /* skip the closing " or ' */
type = CONF_TYPE_STRING;
} else {
break;
}
/* case 2: multiple items, linebreaks allowed */
} else if (line[line_pos] == '{') {
dbg_printf(P_INFO2, "reading multiline param", line_num);
++line_pos;
parse_error = 0;
for (param_pos = 0; line[line_pos] != '}'; /* NOTHING */) {
if(line_pos < fs.st_size && param_pos < MAX_PARAM_LEN) {
param[param_pos++] = line[line_pos++];
if(line[line_pos] == '\n')
line_num++;
} else {
snprintf(erbuf, sizeof(erbuf), "Option %s has a too long parameter (line %d)\n", opt, line_num);
parse_error = 1;
break;
}
}
dbg_printf(P_INFO2, "multiline param: param_good=%d", param_good);
if(parse_error == 0) {
line_pos++; /* skip the closing '}' */
type = CONF_TYPE_STRINGLIST;
} else {
break;
}
/* Case 3: integers */
} else {
parse_error = 0;
for (param_pos = 0; isprint(line[line_pos]) && !isspace(line[line_pos])
&& line[line_pos] != '#'; /* NOTHING */) {
param[param_pos++] = line[line_pos++];
if (param_pos >= MAX_PARAM_LEN) {
snprintf(erbuf, sizeof(erbuf), "Option %s has a too long parameter (line %d)\n", opt, line_num);
parse_error = 1;
break;
}
}
if(parse_error == 0) {
type = CONF_TYPE_INT;
} else {
break;
}
}
param[param_pos] = '\0';
dbg_printf(P_INFO2, "[parse_config_file] option: %s", opt);
dbg_printf(P_INFO2, "[parse_config_file] param: %s (%d byte)", param, strlen(param));
dbg_printf(P_INFO2, "[parse_config_file] -----------------");
set_option(as, opt, param, type);
/* skip whitespaces */
while (isspace(line[line_pos])) {
if(line[line_pos] == '\n')
line_num++;
++line_pos;
}
if(line_pos >= fs.st_size) {
break;
}
}
am_free(line);
am_free(param);
return (parse_error == 1) ? -1 : 0;
}
hash_t *hash_new(uint64_t n) {
hash_t *hash = am_malloc(sizeof(hash_t) + n * sizeof(uint64_t));
bzero(hash, sizeof(hash_t) + n * sizeof(uint64_t));
hash->n = n;
return hash;
}
PRIVATE simple_list parseMultiOption(const char *str) {
int tmp_pos;
uint32_t line_pos = 0;
uint32_t len = strlen(str);
simple_list options = NULL;
char tmp[MAX_PARAM_LEN];
int last_dbl_quote_pos;
int8_t parse_error = 0;
int32_t current_line_pos = -1;
if(len == 0) {
dbg_printf(P_ERROR, "[parseMultiOption] empty input string!");
return NULL;
}
while(line_pos < len) {
memset(&tmp, 0, sizeof(tmp));
// Skip any initial whitespace
while (line_pos < len && isspace(str[line_pos])) {
++line_pos;
}
tmp_pos = 0;
parse_error = 0;
last_dbl_quote_pos = -1;
while(line_pos < len && str[line_pos] != '\0') {
if(str[line_pos] == '\"') {
last_dbl_quote_pos = tmp_pos;
} else if(str[line_pos] == '\n') {
// Text is broken over multiple lines
if(str[line_pos - 1] == '\\' || str[line_pos - 1] == '+') {
// skip newline
line_pos++;
// skip whitespace at the beginning of the next line
while (line_pos < len && isspace(str[line_pos])) {
++line_pos;
}
if(str[line_pos] == '\"' && last_dbl_quote_pos != -1) {
// Reset the string index to the position of the last double-quote, and properly null-terminate it
tmp_pos = last_dbl_quote_pos;
tmp[tmp_pos] = '\0';
// Skip the double-quote on the new line as well
line_pos++;
} else {
tmp[tmp_pos] = '\0';
dbg_printf(P_ERROR, "[parseMultiOption] Parsing error at line '%s'", &tmp[current_line_pos]);
parse_error = 1;
break;
}
} else {
// If the character before the newline is not a backslash ('\'), consider this suboption complete
break;
}
current_line_pos = tmp_pos;
}
tmp[tmp_pos++] = str[line_pos++];
}
if(parse_error) {
break;
}
/* A suboption is finished, end it with a null terminator */
tmp[tmp_pos] = '\0';
/* store the line in our list */
if(tmp_pos != 0) {
suboption_t* i = (suboption_t*)am_malloc(sizeof(suboption_t));
if(i != NULL) {
if(parseSubOption(tmp, &i->option, &i->value) == SUCCESS) {
addItem(i, &options);
} else {
dbg_printf(P_ERROR, "Invalid suboption string: '%s'", tmp);
}
}
}
}
return options;
}
/** \brief parse configuration file.
*
* \param[in,out] as Pointer to session handle
* \param[in] filename Path to the configuration file
* \return 0 if parsing was successful, -1 if an error occured.
*/
int parse_config_file(struct auto_handle *as, const char *filename) {
FILE *fp = NULL;
char *line = NULL;
char opt[MAX_OPT_LEN + 1];
char param[MAX_PARAM_LEN + 1];
char c; /* for the "" and '' check */
int line_num = 0;
int line_pos = 0;
int opt_pos;
int param_pos;
int parse_error = 0;
struct stat fs;
option_type type;
if ((fp = fopen(filename, "rb")) == NULL) {
perror("fopen");
return -1;
}
if(stat(filename, &fs) == -1) {
fclose(fp);
return -1;
}
if ((line = am_malloc(fs.st_size + 1)) == NULL) {
dbg_printf(P_ERROR, "Can't allocate memory for 'line': %s (%ldb)", strerror(errno), fs.st_size + 1);
fclose(fp);
return -1;
}
if(fread(line, fs.st_size, 1, fp) != 1) {
perror("fread");
fclose(fp);
am_free(line);
return -1;
}
/* NULL-terminate the result */
line[fs.st_size] = '\0';
if(fp) {
fclose(fp);
}
while(line_pos != fs.st_size) {
line_pos = SkipWhitespace(line, line_pos, &line_num);
if(line_pos < 0) {
parse_error = 1;
break;
}
if(line_pos >= fs.st_size) {
break;
}
/* comment */
if (line[line_pos] == '#') {
////dbg_printf(P_INFO2, "skipping comment (line %d)", line_num);
while (line[line_pos] != '\n') {
++line_pos;
}
++line_num;
++line_pos; /* skip the newline as well */
continue;
}
/* read option */
for (opt_pos = 0; isprint(line[line_pos]) && line[line_pos] != ' ' &&
line[line_pos] != '#' && line[line_pos] != '='; /* NOTHING */) {
opt[opt_pos++] = line[line_pos++];
if (opt_pos >= MAX_OPT_LEN) {
dbg_printf(P_ERROR, "too long option at line %d", line_num);
parse_error = 1;
}
}
if (opt_pos == 0 || parse_error == 1) {
dbg_printf(P_ERROR, "parse error at line %d (pos: %d)", line_num, line_pos);
parse_error = 1;
break;
} else {
opt[opt_pos] = '\0';
}
line_pos = SkipWhitespace(line, line_pos, &line_num);
if(line_pos < 0) {
parse_error = 1;
break;
}
if(line_pos >= fs.st_size) {
break;
}
/* check for '=' */
if (line[line_pos++] != '=') {
dbg_printf(P_ERROR, "Option '%s' needs a parameter (line %d)", opt, line_num);
parse_error = 1;
break;
}
line_pos = SkipWhitespace(line, line_pos, &line_num);
if(line_pos < 0) {
parse_error = 1;
break;
}
if(line_pos >= fs.st_size) {
break;
}
/* read the parameter */
/* case 1: single string, no linebreaks allowed */
if (line[line_pos] == '"' || line[line_pos] == '\'') {
c = line[line_pos]; /* single or double quote */
++line_pos; /* skip quote */
parse_error = 1;
for (param_pos = 0; (param_pos < MAX_PARAM_LEN) && (line_pos < fs.st_size) && (line[line_pos] != '\n'); /* NOTHING */) {
if( line[line_pos] == c) {
parse_error = 0;
break;
}
param[param_pos++] = line[line_pos++];
}
if(parse_error == 0) {
line_pos++; /* skip the closing single or double quote */
type = CONF_TYPE_STRING;
} else {
dbg_printf(P_ERROR, "Option '%s' has a too long parameter (line %d). Closing quote missing?", opt, line_num);
break;
}
} else if (line[line_pos] == '{') { /* case 2: multiple items, linebreaks allowed */
dbg_printf(P_DBG, "reading multiline param", line_num);
++line_pos;
parse_error = 1;
for (param_pos = 0; (line_pos < fs.st_size) && (param_pos < MAX_PARAM_LEN); /* NOTHING */) {
if(line[line_pos] == '}') {
parse_error = 0;
break;
}
param[param_pos++] = line[line_pos++];
if(line[line_pos] == '\n') {
line_num++;
}
}
if(parse_error == 0) {
line_pos++; /* skip the closing '}' */
type = CONF_TYPE_STRINGLIST;
} else {
dbg_printf(P_ERROR, "Option %s has a too long parameter (line %d). Closing bracket missing?", opt, line_num);
parse_error = 1;
break;
}
} else { /* Case 3: integers */
parse_error = 0;
for (param_pos = 0; isprint(line[line_pos]) && !isspace(line[line_pos]) && line[line_pos] != '#'; /* NOTHING */) {
param[param_pos++] = line[line_pos++];
if (param_pos >= MAX_PARAM_LEN) {
dbg_printf(P_ERROR, "Option %s has a too long parameter (line %d)", opt, line_num);
parse_error = 1;
break;
}
}
if(parse_error == 0) {
type = CONF_TYPE_INT;
} else {
break;
}
}
param[param_pos] = '\0';
dbg_printf(P_DBG, "[parse_config_file] option: %s", opt);
dbg_printf(P_DBG, "[parse_config_file] param: %s (%d byte)", param, strlen(param));
dbg_printf(P_DBG, "[parse_config_file] -----------------");
if(set_option(as, opt, param, type) == FAILURE) {
parse_error = 1;
break;
}
line_pos = SkipWhitespace(line, line_pos, &line_num);
if(line_pos < 0) {
parse_error = 1;
break;
}
if(line_pos >= fs.st_size) {
break;
}
}
am_free(line);
return (parse_error == 1) ? -1 : 0;
}
int ark_create_verbose(char *path, ARK **arkret,
uint64_t size, uint64_t bsize, uint64_t hcount,
int nthrds, int nqueue, int basyncs, uint64_t flags)
{
int rc = 0;
int p_rc = 0;
uint64_t bcount = 0;
uint64_t x = 0;
int i = 0;
int tnum = 0;
int rnum = 0;
scb_t *scbp = NULL;
KV_TRC_OPEN(pAT, "arkdb");
if (NULL == arkret)
{
KV_TRC_FFDC(pAT, "Incorrect value for ARK control block: rc=EINVAL");
rc = EINVAL;
goto ark_create_ark_err;
}
if ( (flags & (ARK_KV_PERSIST_LOAD|ARK_KV_PERSIST_STORE)) &&
(flags & ARK_KV_VIRTUAL_LUN) )
{
KV_TRC_FFDC(pAT, "Invalid persistence combination with ARK flags: %016lx",
flags);
rc = EINVAL;
goto ark_create_ark_err;
}
if (nthrds <= 0)
{
KV_TRC_FFDC(pAT, "invalid nthrds:%d", nthrds);
rc = EINVAL;
goto ark_create_ark_err;
}
_ARK *ark = am_malloc(sizeof(_ARK));
if (ark == NULL) {
rc = ENOMEM;
KV_TRC_FFDC(pAT, "Out of memory allocating ARK control structure for %ld",
sizeof(_ARK));
goto ark_create_ark_err;
}
KV_TRC(pAT, "%p path(%s) size %ld bsize %ld hcount %ld "
"nthrds %d nqueue %d basyncs %d flags:%08lx",
ark, path, size, bsize, hcount,
nthrds, nqueue, basyncs, flags);
ark->bsize = bsize;
ark->rthread = 0;
ark->persload = 0;
ark->nasyncs = ((nqueue <= 0) ? ARK_MAX_ASYNC_OPS : nqueue);
ark->basyncs = basyncs;
ark->ntasks = ARK_MAX_TASK_OPS;
ark->nthrds = ARK_VERBOSE_NTHRDS_DEF; // hardcode, perf requirement
// Create the KV storage, whether that will be memory based
// or flash
ark->ea = ea_new(path, ark->bsize, basyncs, &size, &bcount,
(flags & ARK_KV_VIRTUAL_LUN));
if (ark->ea == NULL)
{
if (!errno) {KV_TRC_FFDC(pAT, "UNSET_ERRNO"); errno=ENOMEM;}
rc = errno;
KV_TRC_FFDC(pAT, "KV storage initialization failed: rc/errno:%d", rc);
goto ark_create_ea_err;
}
// Now that the "connection" to the store has been established
// we need to check to see if data was persisted from a previous
// instantiation of the KV store.
p_rc = ark_check_persistence(ark, flags);
if (p_rc > 0)
{
// We ran into an error while trying to read from
// the store.
rc = p_rc;
KV_TRC_FFDC(pAT, "Persistence check failed: %d", rc);
goto ark_create_persist_err;
}
else if (p_rc == -1)
{
KV_TRC(pAT, "NO PERSIST LOAD FLAG");
// There was no persistence data, so we just build off
// of what was passed into the API.
ark->size = size;
ark->bcount = bcount;
ark->hcount = hcount;
ark->vlimit = ARK_VERBOSE_VLIMIT_DEF;
ark->blkbits = ARK_VERBOSE_BLKBITS_DEF;
ark->grow = ARK_VERBOSE_GROW_DEF;
ark->rthread = 0;
ark->flags = flags;
ark->astart = 0;
ark->blkused = 1;
ark->ark_exit = 0;
ark->nactive = 0;
ark->pers_stats.kv_cnt = 0;
ark->pers_stats.blk_cnt = 0;
ark->pers_stats.byte_cnt = 0;
ark->pcmd = PT_IDLE;
// Create the requests and tag control blocks and queues.
x = ark->hcount / ark->nthrds;
ark->npart = x + (ark->hcount % ark->nthrds ? 1 : 0);
// Create the hash table
ark->ht = hash_new(ark->hcount);
if (ark->ht == NULL)
{
if (!errno) {KV_TRC_FFDC(pAT, "UNSET_ERRNO"); errno=ENOMEM;}
rc = errno;
KV_TRC_FFDC(pAT, "Hash initialization failed: %d", rc);
goto ark_create_ht_err;
}
// Create the block list
ark->bl = bl_new(ark->bcount, ark->blkbits);
if (ark->bl == NULL)
{
if (!errno) {KV_TRC_FFDC(pAT, "UNSET_ERRNO"); errno=ENOMEM;}
rc = errno;
KV_TRC_FFDC(pAT, "Block list initialization failed: %d", rc);
goto ark_create_bl_err;
}
if (flags & ARK_KV_PERSIST_STORE)
{
ark_persistence_calc(ark);
if (bl_reserve(ark->bl, ark->pers_max_blocks))
{goto ark_create_bl_err;}
}
}
else
{
KV_TRC(pAT, "PERSIST: %p path(%s) size %ld bsize %ld hcount %ld "
"nthrds %d nqueue %ld basyncs %d bcount %ld blkbits %ld",
ark, path, ark->size, ark->bsize, ark->hcount,
ark->nthrds, ark->nasyncs, ark->basyncs,
ark->bcount, ark->blkbits);
}
rc = pthread_mutex_init(&ark->mainmutex,NULL);
if (rc != 0)
{
KV_TRC_FFDC(pAT, "pthread_mutex_init for main mutex failed: %d", rc);
goto ark_create_pth_mutex_err;
}
ark->rtags = tag_new(ark->nasyncs);
if ( NULL == ark->rtags )
{
rc = ENOMEM;
KV_TRC_FFDC(pAT, "Tag initialization for requests failed: %d", rc);
goto ark_create_rtag_err;
}
ark->ttags = tag_new(ark->ntasks);
if ( NULL == ark->ttags )
{
rc = ENOMEM;
KV_TRC_FFDC(pAT, "Tag initialization for tasks failed: %d", rc);
goto ark_create_ttag_err;
}
ark->rcbs = am_malloc(ark->nasyncs * sizeof(rcb_t));
if ( NULL == ark->rcbs )
{
rc = ENOMEM;
KV_TRC_FFDC(pAT, "Out of memory allocation of %"PRIu64" bytes for request control blocks", (ark->nasyncs * sizeof(rcb_t)));
goto ark_create_rcbs_err;
}
ark->tcbs = am_malloc(ark->ntasks * sizeof(tcb_t));
if ( NULL == ark->tcbs )
{
rc = ENOMEM;
KV_TRC_FFDC(pAT, "Out of memory allocation of %"PRIu64" bytes for task control blocks", (ark->ntasks * sizeof(rcb_t)));
goto ark_create_tcbs_err;
}
ark->iocbs = am_malloc(ark->ntasks * sizeof(iocb_t));
if ( NULL == ark->iocbs )
{
rc = ENOMEM;
KV_TRC_FFDC(pAT, "Out of memory allocation of %"PRIu64" bytes for io control blocks", (ark->ntasks * sizeof(iocb_t)));
goto ark_create_iocbs_err;
}
ark->poolthreads = am_malloc(ark->nthrds * sizeof(scb_t));
if ( NULL == ark->poolthreads )
{
rc = ENOMEM;
KV_TRC_FFDC(pAT, "Out of memory allocation of %"PRIu64" bytes for server thread control blocks", (ark->nthrds * sizeof(scb_t)));
goto ark_create_poolthreads_err;
}
for ( rnum = 0; rnum < ark->nasyncs ; rnum++ )
{
ark->rcbs[rnum].stat = A_NULL;
pthread_cond_init(&(ark->rcbs[rnum].acond), NULL);
pthread_mutex_init(&(ark->rcbs[rnum].alock), NULL);
}
for ( tnum = 0; tnum < ark->ntasks; tnum++ )
{
ark->tcbs[tnum].inb = bt_new(0, ark->vlimit, sizeof(uint64_t),
&(ark->tcbs[tnum].inblen),
&(ark->tcbs[tnum].inb_orig));
if (ark->tcbs[tnum].inb == NULL)
{
if (!errno) {KV_TRC_FFDC(pAT, "UNSET_ERRNO"); errno=ENOMEM;}
rc = errno;
KV_TRC_FFDC(pAT, "Bucket allocation for inbuffer failed: %d", rc);
goto ark_create_taskloop_err;
}
ark->tcbs[tnum].oub = bt_new(0, ark->vlimit, sizeof(uint64_t),
&(ark->tcbs[tnum].oublen),
&(ark->tcbs[tnum].oub_orig));
if (ark->tcbs[tnum].oub == NULL)
{
if (!errno) {KV_TRC_FFDC(pAT, "UNSET_ERRNO"); errno=ENOMEM;}
rc = errno;
KV_TRC_FFDC(pAT, "Bucket allocation for outbuffer failed: %d", rc);
goto ark_create_taskloop_err;
}
//ark->tcbs[tnum].vbsize = bsize * 1024;
ark->tcbs[tnum].vbsize = bsize * 256;
ark->tcbs[tnum].vb_orig = am_malloc(ark->tcbs[tnum].vbsize);
if (ark->tcbs[tnum].vb_orig == NULL)
{
rc = ENOMEM;
KV_TRC_FFDC(pAT, "Out of memory allocation for %"PRIu64" bytes for variable size buffer", (bsize * 1024));
goto ark_create_taskloop_err;
}
ark->tcbs[tnum].vb = ptr_align(ark->tcbs[tnum].vb_orig);
}
*arkret = (void *)ark;
ark->pts = (PT *)am_malloc(sizeof(PT) * ark->nthrds);
if ( ark->pts == NULL )
{
rc = ENOMEM;
KV_TRC_FFDC(pAT, "Out of memory allocation for %"PRIu64" bytes for server thread data", (sizeof(PT) * ark->nthrds));
goto ark_create_taskloop_err;
}
for (i = 0; i < ark->nthrds; i++) {
PT *pt = &(ark->pts[i]);
scbp = &(ark->poolthreads[i]);
memset(scbp, 0, sizeof(scb_t));
// Start off the random start point for this thread
// at -1, to show that it has not been part of a
// ark_random call.
scbp->rlast = -1;
scbp->holds = 0;
scbp->poolstate = PT_RUN;
scbp->poolstats.io_cnt = 0;
scbp->poolstats.ops_cnt = 0;
scbp->poolstats.kv_cnt = 0;
scbp->poolstats.blk_cnt = 0;
scbp->poolstats.byte_cnt = 0;
pthread_mutex_init(&(scbp->poolmutex), NULL);
pthread_cond_init(&(scbp->poolcond), NULL);
scbp->rqueue = queue_new(ark->nasyncs);
scbp->tqueue = queue_new(ark->ntasks);
scbp->ioqueue = queue_new(ark->ntasks);
pt->id = i;
pt->ark = ark;
rc = pthread_create(&(scbp->pooltid), NULL, pool_function, pt);
if (rc != 0)
{
KV_TRC_FFDC(pAT, "pthread_create of server thread failed: %d", rc);
goto ark_create_poolloop_err;
}
}
#if 0
while (ark->nactive < ark->nthrds) {
usleep(1);
//printf("Create waiting %d/%d\n", ark->nactive, ark->nthrds);
}
#endif
ark->pcmd = PT_RUN;
goto ark_create_return;
ark_create_poolloop_err:
for (; i >= 0; i--)
{
scbp = &(ark->poolthreads[i]);
if (scbp->pooltid != 0)
{
queue_lock(scbp->rqueue);
queue_wakeup(scbp->rqueue);
queue_unlock(scbp->rqueue);
pthread_join(scbp->pooltid, NULL);
pthread_mutex_destroy(&(scbp->poolmutex));
pthread_cond_destroy(&(scbp->poolcond));
if ( scbp->rqueue != NULL )
{
queue_free(scbp->rqueue);
}
if ( scbp->tqueue != NULL )
{
queue_free(scbp->tqueue);
}
if ( scbp->ioqueue != NULL )
{
queue_free(scbp->ioqueue);
}
}
}
if ( ark->pts != NULL )
{
am_free(ark->pts);
}
ark_create_taskloop_err:
for ( tnum = 0; tnum < ark->ntasks; tnum++ )
{
if (ark->tcbs[tnum].inb)
{
bt_delete(ark->tcbs[tnum].inb);
}
if (ark->tcbs[tnum].oub)
{
bt_delete(ark->tcbs[tnum].oub);
}
if (ark->tcbs[tnum].vb_orig)
{
am_free(ark->tcbs[tnum].vb_orig);
}
}
for (rnum = 0; rnum < ark->nasyncs; rnum++)
{
pthread_cond_destroy(&(ark->rcbs[rnum].acond));
pthread_mutex_destroy(&(ark->rcbs[rnum].alock));
}
if ( ark->poolthreads != NULL )
{
am_free(ark->poolthreads);
}
ark_create_poolthreads_err:
if (ark->iocbs)
{
am_free(ark->iocbs);
}
ark_create_iocbs_err:
if (ark->tcbs)
{
am_free(ark->tcbs);
}
ark_create_tcbs_err:
if (ark->rcbs)
{
am_free(ark->rcbs);
}
ark_create_rcbs_err:
if (ark->ttags)
{
tag_free(ark->ttags);
}
ark_create_ttag_err:
if (ark->rtags)
{
tag_free(ark->rtags);
}
ark_create_rtag_err:
pthread_mutex_destroy(&ark->mainmutex);
ark_create_pth_mutex_err:
bl_delete(ark->bl);
ark_create_bl_err:
hash_free(ark->ht);
ark_create_ht_err:
ark_create_persist_err:
ea_delete(ark->ea);
ark_create_ea_err:
am_free(ark);
*arkret = NULL;
ark_create_ark_err:
KV_TRC_CLOSE(pAT);
ark_create_return:
return rc;
}
int ark_check_persistence(_ARK *_arkp, uint64_t flags)
{
int32_t rc = -1;
char *p_data_orig = NULL;
char *p_data = NULL;
ark_io_list_t *bl_array = NULL;
p_cntr_t *pptr = NULL;
P_ARK_t *pcfg = NULL;
hash_t *htp = NULL;
BL *blp = NULL;
uint64_t rdblks = 0;
if (flags & ARK_KV_PERSIST_LOAD) {KV_TRC(pAT, "PERSIST_LOAD");}
// Ignore the persistence data and load from scratch
if ( (!(flags & ARK_KV_PERSIST_LOAD)) || (flags & ARK_KV_VIRTUAL_LUN) )
{
return -1;
}
p_data_orig = am_malloc(_arkp->bsize);
if (p_data_orig == NULL)
{
KV_TRC_FFDC(pAT, "Out of memory allocating %"PRIu64" bytes for the first "
"persistence block", _arkp->bsize);
rc = ENOMEM;
}
else
{
p_data = ptr_align(p_data_orig);
bl_array = bl_chain_no_bl(0, 1);
rc = ea_async_io(_arkp->ea, ARK_EA_READ, (void *)p_data,
bl_array, 1, 1);
am_free(bl_array);
}
if (rc == 0)
{
// We've read the first block. We check to see if
// persistence data is present and if so, then
// read the rest of the data from the flash.
pptr = (p_cntr_t *)p_data;
_arkp->persdata = p_data_orig;
if ( memcmp(pptr->p_cntr_magic, ARK_P_MAGIC,
sizeof(pptr->p_cntr_magic) != 0))
{
KV_TRC_FFDC(pAT, "No magic number found in persistence data: %d", EINVAL);
// The magic number does not match so data is either
// not present or is corrupted.
rc = -1;
}
else
{
// Now we check version and the first persistence data
// needs to be the ARK_PERSIST_CONFIG block
if (pptr->p_cntr_version != ARK_P_VERSION_1 &&
pptr->p_cntr_version != ARK_P_VERSION_2)
{
KV_TRC_FFDC(pAT, "Invalid / unsupported version: %"PRIu64"",
pptr->p_cntr_version);
rc = EINVAL;
}
else
{
// Read in the rest of the persistence data
pcfg = (P_ARK_t *)(pptr->p_cntr_data + pptr->p_cntr_cfg_offset);
rdblks = pcfg->pblocks;
if (rdblks > 1)
{
p_data_orig = am_realloc(p_data_orig, (rdblks * _arkp->bsize));
if (p_data_orig == NULL)
{
KV_TRC_FFDC(pAT, "Out of memory allocating %"PRIu64" bytes for "
"full persistence block",
(rdblks * _arkp->bsize));
rc = ENOMEM;
}
else
{
p_data = ptr_align(p_data_orig);
bl_array = bl_chain_no_bl(0, rdblks);
if (bl_array == NULL)
{
KV_TRC_FFDC(pAT, "Out of memory allocating %"PRIu64" blocks for "
"full persistence data", rdblks);
rc = ENOMEM;
}
}
// We are still good to read the rest of the data
// from the flash
if (rc == 0)
{
KV_TRC(pAT, "PERSIST_RD rdblks:%ld", rdblks);
rc = ea_async_io(_arkp->ea, ARK_EA_READ, (void *)p_data,
bl_array, rdblks, 1);
am_free(bl_array);
pptr = (p_cntr_t *)p_data;
pcfg = (P_ARK_t *)(pptr->p_cntr_data + pptr->p_cntr_cfg_offset);
_arkp->persdata = p_data_orig;
}
}
}
}
}
// If rc == 0, that means we have persistence data
if (rc == 0)
{
KV_TRC(pAT, "PERSIST_META size %ld bsize %ld hcount %ld bcount %ld "
"nthrds %d nasyncs %d basyncs %d blkbits %ld version:%ld",
pcfg->size, pcfg->bsize, pcfg->hcount, pcfg->bcount,
pcfg->nthrds, pcfg->nasyncs, pcfg->basyncs, pcfg->blkbits,
pptr->p_cntr_version);
_arkp->persload = 1;
_arkp->size = pcfg->size;
_arkp->flags = flags;
_arkp->bsize = pcfg->bsize;
_arkp->bcount = pcfg->bcount;
_arkp->blkbits = pcfg->blkbits;
_arkp->grow = pcfg->grow;
_arkp->hcount = pcfg->hcount;
_arkp->vlimit = pcfg->vlimit;
_arkp->blkused = pcfg->blkused;
_arkp->pers_stats.kv_cnt = pcfg->pstats.kv_cnt;
_arkp->pers_stats.blk_cnt = pcfg->pstats.blk_cnt;
_arkp->pers_stats.byte_cnt = pcfg->pstats.byte_cnt;
KV_TRC(pAT, "ARK_META size %ld bsize %ld hcount %ld bcount %ld "
"nthrds %d nasyncs %ld basyncs %d blkbits %ld",
_arkp->size, _arkp->bsize, _arkp->hcount, _arkp->bcount,
_arkp->nthrds, _arkp->nasyncs, _arkp->basyncs, _arkp->blkbits);
htp = (hash_t *)(pptr->p_cntr_data + pptr->p_cntr_ht_offset);
_arkp->ht = hash_new(htp->n);
if (_arkp->ht == NULL)
{
if (!errno) {KV_TRC_FFDC(pAT, "UNSET_ERRNO"); errno=ENOMEM;}
rc = errno;
KV_TRC_FFDC(pAT, "ht_new failed: n:%ld rc:%d", htp->n, rc);
goto error_exit;
}
memcpy(_arkp->ht, htp, pptr->p_cntr_ht_size);
blp = (BL *)(pptr->p_cntr_data + pptr->p_cntr_bl_offset);
_arkp->bl = bl_new(blp->n, blp->w);
if (_arkp->bl == NULL)
{
if (!errno) {KV_TRC_FFDC(pAT, "UNSET_ERRNO"); errno=ENOMEM;}
rc = errno;
KV_TRC_FFDC(pAT, "bl_new failed: n:%ld w:%ld rc:%d",
blp->n, blp->w, rc);
goto error_exit;
}
_arkp->bl->count = blp->count;
_arkp->bl->head = blp->head;
_arkp->bl->hold = blp->hold;
_arkp->bl->top = blp->top;
if (pptr->p_cntr_version == ARK_P_VERSION_1)
{
IV *piv = (IV *)(pptr->p_cntr_data + pptr->p_cntr_bliv_offset);
KV_TRC(pAT, "PERSIST_VERSION_1 LOADED");
_arkp->bl->top = _arkp->bl->n;
// copy IV->data from piv->data
memcpy(_arkp->bl->list->data,
piv->data,
pptr->p_cntr_bliv_size);
}
else if (pptr->p_cntr_version == ARK_P_VERSION_2)
{
KV_TRC(pAT, "PERSIST_VERSION_2 LOADED");
// copy IV->data from bliv_offset
memcpy(_arkp->bl->list->data,
pptr->p_cntr_data + pptr->p_cntr_bliv_offset,
pptr->p_cntr_bliv_size);
}
else
{
rc = EINVAL;
KV_TRC_FFDC(pAT, "bad persistent version number: ver:%ld",
pptr->p_cntr_version);
goto error_exit;
}
KV_TRC(pAT, "BL_META: n:%ld count:%ld head:%ld hold:%ld top:%ld",
_arkp->bl->n, _arkp->bl->count, _arkp->bl->head, _arkp->bl->hold,
_arkp->bl->top);
}
error_exit:
am_free(p_data_orig);
return rc;
}
int ark_persist(_ARK *_arkp)
{
int32_t rc = 0;
uint64_t tot_bytes = 0;
uint64_t wrblks = 0;
char *p_data_orig = NULL;
char *p_data = NULL;
p_cntr_t *pptr = NULL;
char *dptr = NULL;
P_ARK_t *pcfg = NULL;
ark_io_list_t *bl_array = NULL;
if ( (_arkp->ea->st_type == EA_STORE_TYPE_MEMORY) ||
!(_arkp->flags & ARK_KV_PERSIST_STORE) )
{
return 0;
}
ark_persistence_calc(_arkp);
// allocate write buffer
tot_bytes = _arkp->pers_max_blocks * _arkp->bsize;
p_data_orig = am_malloc(tot_bytes);
if (p_data_orig == NULL)
{
KV_TRC_FFDC(pAT, "Out of memory allocating %"PRIu64" bytes for "
"persistence data", tot_bytes);
return ENOMEM;
}
memset(p_data_orig, 0, tot_bytes);
p_data = ptr_align(p_data_orig);
// Record cntr data
pptr = (p_cntr_t *)p_data;
memcpy(pptr->p_cntr_magic, ARK_P_MAGIC, sizeof(pptr->p_cntr_magic));
pptr->p_cntr_version = ARK_P_VERSION_2;
pptr->p_cntr_size = sizeof(p_cntr_t);
// Record configuration info
pcfg = (P_ARK_t*)pptr->p_cntr_data;
pcfg->flags = _arkp->flags;
pcfg->size = _arkp->ea->size;
pcfg->bsize = _arkp->bsize;
pcfg->bcount = _arkp->bcount;
pcfg->blkbits = _arkp->blkbits;
pcfg->grow = _arkp->blkbits;
pcfg->hcount = _arkp->hcount;
pcfg->vlimit = _arkp->vlimit;
pcfg->blkused = _arkp->blkused;
pcfg->nasyncs = _arkp->nasyncs;
pcfg->basyncs = _arkp->basyncs;
pcfg->ntasks = _arkp->ntasks;
pcfg->nthrds = _arkp->nthrds;
ark_persist_stats(_arkp, &(pcfg->pstats));
pptr->p_cntr_cfg_offset = 0;
pptr->p_cntr_cfg_size = sizeof(P_ARK_t);
dptr = pptr->p_cntr_data;
// Record hash info
dptr += pptr->p_cntr_cfg_size;
pptr->p_cntr_ht_offset = dptr - pptr->p_cntr_data;
pptr->p_cntr_ht_size = sizeof(hash_t) + (_arkp->ht->n * sizeof(uint64_t));
memcpy(dptr, _arkp->ht, pptr->p_cntr_ht_size);
// Record block list info
dptr += pptr->p_cntr_ht_size;
pptr->p_cntr_bl_offset = dptr - pptr->p_cntr_data;
pptr->p_cntr_bl_size = sizeof(BL);
memcpy(dptr, _arkp->bl, pptr->p_cntr_bl_size);
// Record IV list info
dptr += pptr->p_cntr_bl_size;
pptr->p_cntr_bliv_offset = dptr - pptr->p_cntr_data;
// bliv_size = bytes in bl->list->data[cs_blocks + kvdata_blocks]
// add 2 to top because of how IV->data chaining works
pptr->p_cntr_bliv_size = divup((_arkp->bl->top+2) * _arkp->bl->w, 8);
memcpy(dptr, _arkp->bl->list->data, pptr->p_cntr_bliv_size);
// Calculate wrblks: number of persist metadata blocks to write
tot_bytes = _arkp->pers_cs_bytes + pptr->p_cntr_bliv_size;
wrblks = pcfg->pblocks = divup(tot_bytes, _arkp->bsize);
KV_TRC(pAT, "PERSIST_WR dev:%s top:%ld wrblks:%ld vs pers_max_blocks:%ld",
_arkp->ea->st_device, _arkp->bl->top, pcfg->pblocks,
_arkp->pers_max_blocks);
bl_array = bl_chain_blocks(_arkp->bl, 0, wrblks);
if ( NULL == bl_array )
{
KV_TRC_FFDC(pAT, "Out of memory allocating %"PRIu64" blocks for block list",
wrblks);
rc = ENOMEM;
}
else
{
rc = ea_async_io(_arkp->ea, ARK_EA_WRITE, (void *)p_data,
bl_array, wrblks, _arkp->nthrds);
am_free(bl_array);
}
KV_TRC(pAT, "PERSIST_DATA_STORED rc:%d", rc);
am_free(p_data_orig);
return rc;
}
/**
* Same as am_mysql_query_return_row_0
* but given the mysql handle and the formatted query
* Return NULL on error, the values of row[0][0] on success
* The caller will need to free the result.
*/
char *
_am_mysql_query_return_row_0 (MYSQL *mysql, char *fmt_query){
MYSQL_RES *result = NULL;
MYSQL_ROW row;
unsigned long *lengths;
char *res = NULL;
my_ulonglong num_rows;
/* Now for the query itself */
if (mysql_query(mysql, fmt_query))
{
/* query failed */
res = NULL;
LOGERROR ("Failed to execute query: Error (%d): %s\n", mysql_errno(mysql), mysql_error(mysql));
goto _query_return_row_0_free;
}
if ((result = mysql_store_result(mysql)) == NULL)
{
/*
* Failed to get result
* 2 cases:
* - no result expected (after an insert statement...
* - error getting result
*/
if (mysql_field_count (mysql) == 0){
/* insert like statement, should not happen!; */
res = NULL;
goto _query_return_row_0_free;
}else{
res = NULL;
LOGERROR ("Failed to store query results: Error (%d): %s\n", mysql_errno(mysql), mysql_error(mysql));
goto _query_return_row_0_free;
}
}
num_rows = mysql_num_rows(result);
/* If num_rows > 1, we consider it is an error */
if (num_rows > 1 ){
res = NULL;
LOGERROR ("Many results were found while only 1 was expected!\n");
goto _query_return_row_0_free;
}
if (num_rows == 0){
/* no result found */
res = NULL;
goto _query_return_row_0_free;
}
/* If we dont have any fields, we should fail too */
if (mysql_num_fields(result) < 1 ){
res = NULL;
LOGERROR ("No field encountered in MySQL result!\n");
goto _query_return_row_0_free;
}
row = mysql_fetch_row(result);
if (row == NULL){
/* An error occurred */
res = NULL;
LOGERROR ("Failed to return first row, Error (%d): %s\n", mysql_errno(mysql), mysql_error(mysql));
goto _query_return_row_0_free;
}
lengths = mysql_fetch_lengths(result);
if ( lengths[0] != 0 ){
res = am_malloc (lengths[0] + 1);
if (res == NULL){
LOGERROR ("Could not allocate memory for result\n");
}else{
memcpy (res, row[0], lengths[0]);
res[lengths[0]] = '\0';
}
}else{
res = NULL;
}
_query_return_row_0_free:
if (result != NULL)
mysql_free_result (result);
return res;
}
EA *ea_new(const char *path, uint64_t bsize, int basyncs,
uint64_t *size, uint64_t *bcount, uint64_t vlun)
{
int rc = 0;
size_t plen = 0;
uint8_t *store = NULL;
EA *ea = NULL;
chunk_id_t chkid = NULL_CHUNK_ID;
chunk_ext_arg_t ext = 0;
if (!(fetch_and_or(&cflsh_blk_lib_init,1)))
{
// We need to call cblk_init once before
// we use any other cblk_ interfaces
rc = cblk_init(NULL,0);
if (rc)
{
KV_TRC_FFDC(pAT, "cblk_init failed path %s bsize %"PRIu64" "
"size %"PRIu64" bcount %"PRIu64", errno = %d",
path, bsize, *size, *bcount, errno);
goto error_exit;
}
}
ea = am_malloc(sizeof(EA));
if (NULL == ea)
{
KV_TRC_FFDC(pAT, "Out of memory path %s bsize %"PRIu64" size %"PRIu64" "
"bcount %"PRIu64", errno = %d",
path, bsize, *size, *bcount, errno);
goto error_exit;
}
// We need to check the path parameter to see if
// we are going to use memory or a file/capi
// device (to be determined by the block layer)
if ( (NULL == path) || (strlen(path) == 0) )
{
KV_TRC(pAT, "EA_STORE_TYPE_MEMORY");
// Using memory for store
ea->st_type = EA_STORE_TYPE_MEMORY;
store = malloc(*size);
if (NULL == store)
{
errno = ENOMEM;
KV_TRC_FFDC(pAT, "Out of memory for store path %s bsize %"PRIu64" "
"size %"PRIu64" bcount %"PRIu64", errno = %d",
path, bsize, *size, *bcount, errno);
goto error_exit;
}
*bcount = ((*size) / bsize);
ea->st_memory = store;
}
else
{
KV_TRC(pAT, "EA_STORE_TYPE_FILE(%s)", path);
// Using a file. We don't care if it's an actual
// file or a CAPI device, we let block layer
// decide and we just use the chunk ID that is
// passed back from the cblk_open call.
ea->st_type = EA_STORE_TYPE_FILE;
// Check to see if we need to create the store on a
// physical or virtual LUN. Previously, in GA1,
// we keyed off the size and if it was 0, then we
// asked for the LUN to be physical. Now, the user
// can specify with a flag.
if ( vlun == 0 )
{
KV_TRC(pAT, "cblk_open PHYSICAL LUN: %s", path);
chkid = cblk_open(path, basyncs, O_RDWR, ext,
CBLK_OPN_NO_INTRP_THREADS);
if (NULL_CHUNK_ID == chkid)
{
printf("cblk_open physical lun failed\n");
KV_TRC_FFDC(pAT, "cblk_open phys lun failed path:%s bsize:%ld "
"size:%ld bcount:%ld, errno:%d",
path, bsize, *size, *bcount, errno);
goto error_exit;
}
rc = cblk_get_size(chkid, (size_t *)bcount, 0);
if ( (rc != 0) || (*bcount == 0) )
{
// An error was encountered, close the chunk
cblk_close(chkid, 0);
chkid = NULL_CHUNK_ID;
KV_TRC_FFDC(pAT, "cblk_get_size failed path %s bsize %"PRIu64" "
"size %"PRIu64" bcount %"PRIu64", errno = %d",
path, bsize, *size, *bcount, errno);
goto error_exit;
}
// Set the size to be returned
*size = *bcount * bsize;
}
else
{
KV_TRC(pAT, "cblk_open VIRTUAL LUN: %s", path);
chkid = cblk_open(path, basyncs, O_RDWR, ext,
CBLK_OPN_VIRT_LUN|CBLK_OPN_NO_INTRP_THREADS);
if (NULL_CHUNK_ID == chkid)
{
printf("cblk_open virtual lun failed\n");
KV_TRC_FFDC(pAT, "cblk_open virt lun failed path:%s bsize:%ld "
"size:%ld bcount:%ld, errno:%d",
path, bsize, *size, *bcount, errno);
goto error_exit;
}
// A specific size was passed in so we try to set the
// size of the chunk.
*bcount = *size / bsize;
rc = cblk_set_size(chkid, (size_t)*bcount, 0);
if ( rc != 0 )
{
printf("cblk_set_size failed for %ld\n", *bcount);
// An error was encountered, close the chunk
cblk_close(chkid, 0);
chkid = NULL_CHUNK_ID;
KV_TRC_FFDC(pAT, "cblk_set_size failed path %s bsize %"PRIu64" "
"size %"PRIu64" bcount %"PRIu64", errno = %d",
path, bsize, *size, *bcount, errno);
goto error_exit;
}
}
// Save off the chunk ID and the device name
ea->st_flash = chkid;
plen = strlen(path) + 1;
ea->st_device = (char *)am_malloc(plen);
if (!ea->st_device)
{
cblk_close(chkid, 0);
KV_TRC_FFDC(pAT, "MALLOC st_device failed (%s) plen=%ld errno:%d",
path, plen, errno);
goto error_exit;
}
memset(ea->st_device, 0, plen);
strncpy(ea->st_device, path, plen);
}
// Fill in the EA struct
pthread_rwlock_init(&(ea->ea_rwlock), NULL);
ea->bsize = bsize;
ea->bcount = *bcount;
ea->size = *size;
KV_TRC(pAT, "path %s bsize %"PRIu64" size %"PRIu64" bcount %"PRIu64"",
path, bsize, *size, *bcount);
goto done;
error_exit:
am_free(ea);
ea = NULL;
if (!errno) {KV_TRC_FFDC(pAT, "UNSET_ERRNO"); errno=ENOSPC;}
done:
return ea;
}