luarest_status invoke_application(application* apps, UT_string* url, luarest_method m, luarest_response* res_code,
luarest_content_type* con_type, UT_string* res_buf)
{
application* app = NULL;
service *service;
char* pch = NULL;
char* tmp = utstring_body(url);
UT_string* app_name;
UT_string* key;
utstring_new(app_name);
pch = strchr(++tmp, '/');
if (pch == NULL) {
return(LUAREST_ERROR);
}
utstring_bincpy(app_name, tmp, pch-tmp);
HASH_FIND(hh, apps, utstring_body(app_name), utstring_len(app_name), app);
utstring_free(app_name);
if (app == NULL) {
return(LUAREST_ERROR);
}
utstring_new(key);
utstring_printf(key, "M%d#P%s", m, pch);
HASH_FIND(hh, app->s, utstring_body(key), utstring_len(key), service);
if (service == NULL) {
return(LUAREST_ERROR);
}
invoke_lua(app->lua_state, service->callback_ref, res_code, con_type, res_buf);
return(LUAREST_SUCCESS);
}
int main() {
UT_string *s,*t;
char V_TestStr[] = "There are two needle\0s in this \0haystack with needle\0s.";
char V_NeedleStr[] = "needle\0s";
long *V_KMP_Table;
long V_FindPos;
size_t V_StartPos;
size_t V_FindCnt;
utstring_new(s);
utstring_new(t);
utstring_bincpy(s, V_TestStr, sizeof(V_TestStr)-1);
printf("\"%s\" len=%u\n", utstring_body(s), utstring_len(s));
utstring_bincpy(t, V_NeedleStr, sizeof(V_NeedleStr)-1);
printf("\"%s\" len=%u\n", utstring_body(t), utstring_len(t));
V_KMP_Table = (long *)malloc(sizeof(long) * (utstring_len(t) + 1));
if (V_KMP_Table != NULL)
{
_utstring_BuildTable(utstring_body(t), utstring_len(t), V_KMP_Table);
V_FindCnt = 0;
V_FindPos = 0;
V_StartPos = 0;
do
{
V_FindPos = _utstring_find(utstring_body(s) + V_StartPos,
utstring_len(s) - V_StartPos,
utstring_body(t),
utstring_len(t),
V_KMP_Table);
if (V_FindPos >= 0)
{
V_FindPos += V_StartPos;
V_FindCnt++;
V_StartPos = V_FindPos + 1;
}
printf("utstring_find()=%ld\n", V_FindPos);
} while (V_FindPos >= 0);
printf("FindCnt=%u\n", V_FindCnt);
free(V_KMP_Table);
}
else
{
printf("malloc() failed...\n");
}
utstring_free(s);
utstring_free(t);
return 0;
}
static void JsonParser_internalCreate(struct ParserInternal *pi)
{
pi->error = JSON_ERR_NONE;
pi->line = 0;
utstring_new(pi->key);
utstring_new(pi->value);
pi->stack.v = calloc(JSON_STACK_SIZE, sizeof(char*));
pi->stack.num = 0;
pi->stack.max = JSON_STACK_SIZE;
pi->stack.elem_size = sizeof(char*);
pi->startElem = NULL;
pi->endElem = NULL;
pi->elemData = NULL;
}
void initialize_oh_prepared(OM_DEFINE_OBJECT(o_handler,oh)) {
OM_MUTEX_LOCK(o_prepared, oprep);
oh->service_protocol = -1;
oh->connection_prepared = FALSE;
oh->result = NULL;
OM_NEW(o_ac, oh->admin);
utstring_new(oh->admin->username);
utstring_new(oh->admin->password);
OM_NEW(o_ac, oh->user);
utstring_new(oh->user->username);
utstring_new(oh->user->password);
oh->och_ht = NULL;
OM_MUTEX_UNLOCK(o_prepared, oprep);
}
struct pkg_repo_it *
pkg_repo_binary_require(struct pkg_repo *repo, const char *provide)
{
sqlite3_stmt *stmt;
sqlite3 *sqlite = PRIV_GET(repo);
UT_string *sql = NULL;
int ret;
const char basesql[] = ""
"SELECT p.id, p.origin, p.name, p.version, p.comment, "
"p.name as uniqueid, "
"p.prefix, p.desc, p.arch, p.maintainer, p.www, "
"p.licenselogic, p.flatsize, p.pkgsize, "
"p.cksum, p.manifestdigest, p.path AS repopath, '%s' AS dbname "
"FROM packages AS p INNER JOIN pkg_requires AS ps ON "
"p.id = ps.package_id "
"WHERE ps.require_id = (SELECT id FROM requires WHERE require=?1);";
utstring_new(sql);
utstring_printf(sql, basesql, repo->name);
pkg_debug(4, "Pkgdb: running '%s'", utstring_body(sql));
ret = sqlite3_prepare_v2(sqlite, utstring_body(sql), -1, &stmt, NULL);
if (ret != SQLITE_OK) {
ERROR_SQLITE(sqlite, utstring_body(sql));
utstring_free(sql);
return (NULL);
}
utstring_free(sql);
sqlite3_bind_text(stmt, 1, provide, -1, SQLITE_TRANSIENT);
return (pkg_repo_binary_it_new(repo, stmt, PKGDB_IT_FLAG_ONCE));
}
int o_bin_shutdown(orientdb *o, orientdb_con *c, struct timeval *timeout, int nonblocking) {
UT_string *s = NULL;
int i;
debug_note(o, ORIENT_INFO, "executing binary method: SHUTDOWN\n");
i = get_con_fd(o, c);
if (i == -1) return -1;
// send the operation we want to execute to the server
i = o_sendrequest(o, c, O_SHUTDOWN);
i = o_sendcredentials(o, c, ORIENT_ADMIN);
i = o_flushsendbuffer(o, c, timeout);
i = o_getresponse(o, c, timeout);
if (i == 1) { /* EXCEPTION!! */
debug_note(o, ORIENT_INFO, "received an exception\n");
utstring_new(s);
i = o_getexceptions(o, c, timeout, s);
debug_note(o, ORIENT_INFO, "E: %s\n", utstring_body(s));
utstring_free(s);
return -1;
}
return 0;
}
char *mtex2MML_combine_row_data(UT_array **environment_data_stack)
{
/* if no information was provided, give a standard sizing */
if (utarray_len(*environment_data_stack) == 0) {
const char* s = "rowspacing=\"0.5ex\" rowlines=\"none\"";
char* c = (char*)malloc(strlen(s) + 1);
strcpy(c, s);
return c;
}
envdata_t *row_data_elem = (envdata_t*) utarray_front(*environment_data_stack);
char *row_spacing_data = row_data_elem->rowspacing,
*row_lines_data = row_data_elem->rowlines,
*row_attr;
UT_string *row_attr_data;
utstring_new(row_attr_data);
/* combine the row spacing and row lines data */
utstring_printf(row_attr_data, "%s%s\" %s\"", "rowspacing=\"", row_spacing_data, row_lines_data);
row_attr = string_dup(utstring_body(row_attr_data));
utarray_erase(*environment_data_stack, 0, 1);
utstring_free(row_attr_data);
return row_attr;
}
char *mtex2MML_remove_excess_pipe_chars(char *string)
{
UT_string *columnalign;
utstring_new(columnalign);
char *dupe = string_dup(string);
char *token = strtok(dupe, " ");
char *attr_columnalign;
while (token != NULL) {
if (strncmp(token, "s", 1) != 0 && strncmp(token, "d", 1) != 0) {
utstring_printf(columnalign, "%s ", token);
}
token = strtok(NULL, " ");
}
attr_columnalign = string_dup(utstring_body(columnalign));
free(dupe);
utstring_free(columnalign);
if (strlen(attr_columnalign) > 0) {
mtex2MML_remove_last_char(attr_columnalign); /* remove the final space */
}
return attr_columnalign;
}
int main(int argc, char *argv[]) {
void *sp=NULL;
void *set=NULL;
int opt,rc=-1;
char *config_file;
set = kv_set_new();
utarray_new(output_keys, &ut_str_icd);
utarray_new(output_defaults, &ut_str_icd);
utarray_new(output_types,&ut_int_icd);
utstring_new(tmp);
while ( (opt = getopt(argc, argv, "v+d:b:s")) != -1) {
switch (opt) {
case 'v': verbose++; break;
case 's': push_mode++; break;
case 'd': spool=strdup(optarg); break;
case 'b': config_file=strdup(optarg); break;
default: usage(argv[0]); break;
}
}
if (optind < argc) pub_transport = argv[optind++];
if (!pub_transport) usage(argv[0]);
if (spool == NULL) usage(argv[0]);
if (parse_config(config_file) < 0) goto done;
if ( !(pub_context = zmq_init(1))) goto done;
if ( !(pub_socket = zmq_socket(pub_context, push_mode?ZMQ_PUSH:ZMQ_PUB))) goto done;
if (zmq_setsockopt(pub_socket, ZMQ_SNDHWM, &hwm, sizeof(hwm))) goto done;
if (zmq_bind(pub_socket, pub_transport) == -1) goto done;
sp = kv_spoolreader_new(spool);
if (!sp) goto done;
while (kv_spool_read(sp,set,1) > 0) { /* read til interrupted by signal */
zmq_msg_t part;
if (set_to_binary(set,&part) < 0) goto done;
rc = zmq_sendmsg(pub_socket, &part, 0);
zmq_msg_close(&part);
if (rc == -1) goto done;
}
rc = 0;
done:
if (rc) fprintf(stderr,"zmq: %s %s\n", pub_transport, zmq_strerror(errno));
if (pub_socket) zmq_close(pub_socket);
if (pub_context) zmq_term(pub_context);
if (sp) kv_spoolreader_free(sp);
kv_set_free(set);
utarray_free(output_keys);
utarray_free(output_defaults);
utarray_free(output_types);
utstring_free(tmp);
return 0;
}
// TODO: finish to test this one
int o_bin_dbcreate(orientdb *o, orientdb_con *c, struct timeval *timeout, int nonblocking, const char *dbname, int dbtype) {
UT_string *s;
int i;
debug_note(o, ORIENT_INFO, "executing binary method: DB_CREATE\n");
i = get_con_fd(o, c);
if (i == -1) return -1;
// check the dbname and dbtype parameters
if ((dbname == NULL) || (strlen(dbname) == 0) || ((dbtype != O_DBTYPE_LOCAL) && (dbtype != O_DBTYPE_MEMORY))) return -1;
utstring_new(s);
// send the operation we want to execute to the server
i = o_sendrequest(o, c, O_DB_CREATE);
// send db name
utstring_printf(s, dbname);
i = o_putstring(o, c, s);
utstring_clear(s);
switch (dbtype) {
case O_DBTYPE_LOCAL:
utstring_printf(s, "local");
break;
case O_DBTYPE_MEMORY:
utstring_printf(s, "memory");
break;
default:
break;
}
// send db type
i = o_putstring(o, c, s);
utstring_clear(s);
/*
i = o_sendcredentials(o, c, ORIENT_USER);
*/
i = o_flushsendbuffer(o, c, timeout);
i = o_getresponse(o, c, timeout);
if (i == 1) { /* EXCEPTION!! */
debug_note(o, ORIENT_INFO, "received an exception\n");
i = o_getexceptions(o, c, timeout, s);
debug_note(o, ORIENT_INFO, "E: %s\n", utstring_body(s));
utstring_free(s);
return -1;
}
utstring_free(s);
return 0;
}
int main()
{
UT_string *s;
utstring_new(s);
utstring_printf(s, "hello world!" );
printf("%s\n", utstring_body(s));
utstring_free(s);
return 0;
}
int main(int argc, char * argv[]) {
int opt,verbose=0;
long dirmax=0;
/* input spool */
char *dir=NULL,unit,*sz="10GB";
UT_string *s;
utstring_new(s);
while ( (opt = getopt(argc, argv, "v+s:")) != -1) {
switch (opt) {
default: usage(argv[0]); break;
case 'v': verbose++; break;
case 's':
sz = strdup(optarg);
switch (sscanf(sz, "%ld%c", &dirmax, &unit)) {
case 2: /* check unit */
switch (unit) {
case 't': case 'T': break;
case 'g': case 'G': break;
case 'm': case 'M': break;
case 'k': case 'K': break;
case '\r': case '\n': case ' ': case '\t': break;
default: usage(argv[0]); break;
}
case 1: /* just a number in bytes */ break;
default: usage(argv[0]); break;
}
break;
}
}
if (optind >= argc) usage(argv[0]);
if (!dirmax) usage(argv[0]);
kv_spool_options.dir_max = dirmax;
while (optind < argc) {
dir = argv[optind++];
utstring_clear(s);
utstring_printf(s,"%s/limits", dir);
char *p = utstring_body(s);
FILE *f = fopen(p, "w");
if (f == NULL) {
fprintf(stderr,"cannot open %s: %s\n", p, strerror(errno));
continue;
}
fprintf(f, "%s", sz);
fclose(f);
sp_attrition(dir);
}
done:
utstring_free(s);
return 0;
}
static PyObject* binaryplist_encode(PyObject *self, PyObject *args, PyObject *kwargs)
{
static char *kwlist[] = {"obj", "unique", "debug", "convert_nulls",
"max_recursion", "object_hook", "as_ascii", NULL};
PyObject *newobj = NULL;
PyObject *oinput = NULL;
PyObject *ounique = NULL;
PyObject *odebug = NULL;
PyObject *orecursion = NULL;
binaryplist_encoder encoder;
memset(&encoder, 0, sizeof(binaryplist_encoder));
encoder.convert_nulls = Py_False;
encoder.as_ascii = Py_False;
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|OOOOOO", kwlist, &oinput, &ounique,
&odebug, &(encoder.convert_nulls), &orecursion, &(encoder.object_hook),
&(encoder.as_ascii))) {
return NULL;
}
if (encoder.object_hook && !PyCallable_Check(encoder.object_hook)) {
PyErr_SetString(PLIST_Error, "object_hook is not callable");
return NULL;
}
encoder.ref_table = PyDict_New();
encoder.objects = PyList_New(0);
utstring_new(encoder.output);
if (!ounique || (ounique && PyObject_IsTrue(ounique))) {
/* default to True */
encoder.dounique = 1;
encoder.uniques = PyDict_New();
}
if (odebug && PyObject_IsTrue(odebug)) {
encoder.debug = 1;
}
if (orecursion && PyInt_Check(orecursion)) {
encoder.max_recursion = PyInt_AsLong(orecursion);
} else {
encoder.max_recursion = 1024*16;
}
if (encoder_encode_object(&encoder, oinput) == BINARYPLIST_OK
&& encoder_write(&encoder) == BINARYPLIST_OK) {
newobj = PyString_FromStringAndSize(utstring_body(encoder.output),
utstring_len(encoder.output));
}
Py_DECREF(encoder.ref_table);
Py_DECREF(encoder.objects);
Py_XDECREF(encoder.uniques);
utstring_free(encoder.output);
return newobj;
}
int main() {
UT_string *err;
utstring_new(err);
void *c60;
c60 = c60_client_init_fromfile("config.txt",err);
if (!c60) fprintf(stderr,"init failed: %s",utstring_body(err));
c60_send(c60, "recipient", "hello!", 7);
c60_client_close(c60);
return 0;
}
/**
* LUA syntax: application.register(method, url, callback)
*
* Return: boolean true on success
*
*/
static int l_register(lua_State* state)
{
service* s;
application* a = (application*)luaL_checkudata(state, 1, LUA_USERDATA_APPLICATION);
int method = luaL_checkint(state, 2);
const char* url = luaL_checkstring(state, 3);
int ref = luaL_ref(state, LUA_REGISTRYINDEX);
s = (service*)malloc(sizeof(service));
utstring_new(s->key);
utstring_printf(s->key, "M%d#P%s", method, url);
switch (method)
{
case 1:
s->method = HTTP_METHOD_GET;
break;
case 2:
s->method = HTTP_METHOD_POST;
break;
case 3:
s->method = HTTP_METHOD_PUT;
break;
case 4:
s->method = HTTP_METHOD_DELETE;
break;
case 5:
s->method = HTTP_METHOD_OPTION;
break;
case 6:
s->method = HTTP_METHOD_HEAD;
break;
}
utstring_new(s->path);
utstring_printf(s->path, url);
s->callback_ref = ref;
HASH_ADD_KEYPTR(hh, a->s, utstring_body(s->key), utstring_len(s->key), s);
return(1);
}
short o_bin_dataclusteradd(orientdb *o, orientdb_con *c, struct timeval *timeout, int nonblocking, const char *type, const char *name, const char *filename, int initialsize) {
UT_string *s;
int i;
short x;
debug_note(o, ORIENT_INFO, "executing binary method: DATACLUSTER_ADD\n");
i = get_con_fd(o, c);
if (i == -1) return -1;
utstring_new(s);
// send the operation we want to execute to the server
i = o_sendrequest(o, c, O_DATACLUSTER_ADD);
// send cluster type
utstring_printf(s, type);
i = o_putstring(o, c, s);
utstring_clear(s);
// send cluster name
utstring_printf(s, name);
i = o_putstring(o, c, s);
utstring_clear(s);
if (strcmp(type, O_CLUSTER_PHYSICAL) == 0) { // the physical cluster need filename and initialsize, while memory and logical do not need them.
// send cluster filename
utstring_printf(s, filename);
i = o_putstring(o, c, s);
utstring_clear(s);
// send cluster initialsize in bytes
i = o_putint(o, c, initialsize);
}
i = o_flushsendbuffer(o, c, timeout);
i = o_getresponse(o, c, timeout);
if (i == 1) { /* EXCEPTION!! */
debug_note(o, ORIENT_INFO, "received an exception\n");
i = o_getexceptions(o, c, timeout, s);
debug_note(o, ORIENT_INFO, "E: %s\n", utstring_body(s));
utstring_free(s);
return -1;
}
// get the new cluster id
x = o_getshort(o, c, timeout);
utstring_free(s);
return x;
}
int
packing_append_tree(struct packing *pack, const char *treepath,
const char *newroot)
{
FTS *fts = NULL;
FTSENT *fts_e = NULL;
size_t treelen;
UT_string *sb;
char *paths[2] = { __DECONST(char *, treepath), NULL };
treelen = strlen(treepath);
fts = fts_open(paths, FTS_PHYSICAL | FTS_XDEV, NULL);
if (fts == NULL)
goto cleanup;
utstring_new(sb);
while ((fts_e = fts_read(fts)) != NULL) {
switch(fts_e->fts_info) {
case FTS_D:
case FTS_DEFAULT:
case FTS_F:
case FTS_SL:
case FTS_SLNONE:
/* Entries not within this tree are irrelevant. */
if (fts_e->fts_pathlen <= treelen)
break;
utstring_clear(sb);
/* Strip the prefix to obtain the target path */
if (newroot) /* Prepend a root if one is specified */
utstring_printf(sb, "%s", newroot);
/* +1 = skip trailing slash */
utstring_printf(sb, "%s", fts_e->fts_path + treelen + 1);
packing_append_file_attr(pack, fts_e->fts_name,
utstring_body(sb), NULL, NULL, 0, 0);
break;
case FTS_DC:
case FTS_DNR:
case FTS_ERR:
case FTS_NS:
/* XXX error cases, check fts_e->fts_errno and
* bubble up the call chain */
break;
default:
break;
}
}
utstring_free(sb);
cleanup:
fts_close(fts);
return EPKG_OK;
}
int64_t
pkg_repo_binary_stat(struct pkg_repo *repo, pkg_stats_t type)
{
sqlite3 *sqlite = PRIV_GET(repo);
sqlite3_stmt *stmt = NULL;
int64_t stats = 0;
UT_string *sql = NULL;
int ret;
utstring_new(sql);
switch(type) {
case PKG_STATS_LOCAL_COUNT:
goto out;
break;
case PKG_STATS_LOCAL_SIZE:
goto out;
break;
case PKG_STATS_REMOTE_UNIQUE:
utstring_printf(sql, "SELECT COUNT(id) FROM main.packages;");
break;
case PKG_STATS_REMOTE_COUNT:
utstring_printf(sql, "SELECT COUNT(id) FROM main.packages;");
break;
case PKG_STATS_REMOTE_SIZE:
utstring_printf(sql, "SELECT SUM(pkgsize) FROM main.packages;");
break;
case PKG_STATS_REMOTE_REPOS:
goto out;
break;
}
pkg_debug(4, "binary_repo: running '%s'", utstring_body(sql));
ret = sqlite3_prepare_v2(sqlite, utstring_body(sql), -1, &stmt, NULL);
if (ret != SQLITE_OK) {
ERROR_SQLITE(sqlite, utstring_body(sql));
goto out;
}
while (sqlite3_step(stmt) != SQLITE_DONE) {
stats = sqlite3_column_int64(stmt, 0);
}
out:
utstring_free(sql);
if (stmt != NULL)
sqlite3_finalize(stmt);
return (stats);
}
static UT_string* escape_fields(CURL* ch, const char* fields, va_list ap)
{
UT_string* res = 0;
utstring_new(res);
for (;*fields;*fields++)
{
if (*fields == '?')
{
char* f,*arg;
arg = va_arg(ap,char*);
f = curl_easy_escape(ch,arg,0);
utstring_bincpy(res,f,strlen(f));
curl_free(f);
}
else
struct pkg_repo_it *
pkg_repo_binary_query(struct pkg_repo *repo, const char *pattern, match_t match)
{
sqlite3 *sqlite = PRIV_GET(repo);
sqlite3_stmt *stmt = NULL;
UT_string *sql = NULL;
const char *comp = NULL;
int ret;
char basesql[BUFSIZ] = ""
"SELECT id, origin, name, name as uniqueid, version, comment, "
"prefix, desc, arch, maintainer, www, "
"licenselogic, flatsize, pkgsize, "
"cksum, manifestdigest, path AS repopath, '%s' AS dbname "
"FROM packages AS p";
if (match != MATCH_ALL && (pattern == NULL || pattern[0] == '\0'))
return (NULL);
utstring_new(sql);
comp = pkgdb_get_pattern_query(pattern, match);
if (comp && comp[0])
strlcat(basesql, comp, sizeof(basesql));
utstring_printf(sql, basesql, repo->name);
utstring_printf(sql, "%s", " ORDER BY name;");
pkg_debug(4, "Pkgdb: running '%s' query for %s", utstring_body(sql),
pattern == NULL ? "all": pattern);
ret = sqlite3_prepare_v2(sqlite, utstring_body(sql), utstring_len(sql), &stmt,
NULL);
if (ret != SQLITE_OK) {
ERROR_SQLITE(sqlite, utstring_body(sql));
utstring_free(sql);
return (NULL);
}
utstring_free(sql);
if (match != MATCH_ALL && match != MATCH_CONDITION)
sqlite3_bind_text(stmt, 1, pattern, -1, SQLITE_TRANSIENT);
return (pkg_repo_binary_it_new(repo, stmt, PKGDB_IT_FLAG_ONCE));
}
static luarest_status verify_application(application** apps, const char* appName, UT_string* path)
{
int ret;
lua_State* ls = luaL_newstate();
application* app;
luaL_openlibs(ls);
luaopen_luarestlibs(ls);
ret = luaL_loadfile(ls, utstring_body(path));
if (ret != 0) {
printf("Couldn't load file: %s\n", lua_tostring(ls, -1));
lua_close(ls);
return(LUAREST_ERROR);
}
ret = lua_pcall(ls, 0, 0, 0);
if (ret != 0) {
printf("Couldn't execute LUA Script %s\n", lua_tostring(ls, -1));
lua_close(ls);
return(LUAREST_ERROR);
}
lua_getglobal(ls, "luarest_init");
if (lua_isfunction(ls, -1) == 0) {
printf("Couln'd find 'luarest_init' table in LUA script!\n");
lua_close(ls);
return(LUAREST_ERROR);
}
app = (application*)lua_newuserdata(ls, sizeof(application));
luaL_getmetatable(ls, LUA_USERDATA_APPLICATION);
lua_setmetatable(ls, -2);
app->s = NULL;
utstring_new(app->name);
utstring_printf(app->name, appName);
if (lua_pcall(ls, 1, 0, 0) != 0) {
printf("Error calling luarest_init: %s\n!", lua_tostring(ls, -1));
lua_close(ls);
return(LUAREST_ERROR);
}
app->lua_state = ls;
HASH_ADD_KEYPTR(hh, *apps, utstring_body(app->name), utstring_len(app->name), app);
return(LUAREST_SUCCESS);
}
struct pkg_repo_it *
pkg_repo_binary_search(struct pkg_repo *repo, const char *pattern, match_t match,
pkgdb_field field, pkgdb_field sort)
{
sqlite3 *sqlite = PRIV_GET(repo);
sqlite3_stmt *stmt = NULL;
UT_string *sql = NULL;
int ret;
const char *multireposql = ""
"SELECT id, origin, name, version, comment, "
"prefix, desc, arch, maintainer, www, "
"licenselogic, flatsize, pkgsize, "
"cksum, path AS repopath, '%1$s' AS dbname, '%2$s' AS repourl "
"FROM packages ";
if (pattern == NULL || pattern[0] == '\0')
return (NULL);
utstring_new(sql);
utstring_printf(sql, multireposql, repo->name, repo->url);
/* close the UNIONs and build the search query */
utstring_printf(sql, "%s", "WHERE ");
pkg_repo_binary_build_search_query(sql, match, field, sort);
utstring_printf(sql, "%s", ";");
pkg_debug(4, "Pkgdb: running '%s'", utstring_body(sql));
ret = sqlite3_prepare_v2(sqlite, utstring_body(sql), -1, &stmt, NULL);
if (ret != SQLITE_OK) {
ERROR_SQLITE(sqlite, utstring_body(sql));
utstring_free(sql);
return (NULL);
}
utstring_free(sql);
sqlite3_bind_text(stmt, 1, pattern, -1, SQLITE_TRANSIENT);
return (pkg_repo_binary_it_new(repo, stmt, PKGDB_IT_FLAG_ONCE));
}
/* return allocated UT_string */
static UT_string *JsonNode_getJSON_UT(JsonNode *node, String last)
{
asize_t i, nPairs, nChilds;
UT_string *buff;
utstring_new(buff);
if (buff == NULL) return NULL;
if (!isNullorEmpty(node->m_name)) {
utstring_printf(buff, "\"%s\":", node->m_name);
}
utstring_printf(buff, "%s\n", JSON_IS_OBJ(node) ? "{" : "[");
nPairs = JsonNode_getPairCount(node);
nChilds = JsonNode_getChildCount(node);
for (i=0; i < nPairs; i++ ) {
JsonPair *pair = JsonNode_getPair(node, i);
if (JSON_IS_ARRAY(node)) {
utstring_printf(buff, "\"%s\"", pair->key);
} else {
utstring_printf(buff, "\"%s\":\"%s\"", pair->key, pair->value);
}
utstring_printf(buff, "%s\n", (i < nPairs -1 || nChilds > 0) ? "," : "");
}
for (i = 0; i < nChilds; i++) {
JsonNode* child = JsonNode_getChild(node, i);
UT_string *childJSON = JsonNode_getJSON_UT(child, i == nChilds - 1 ? "\n" : ",\n");
if (childJSON != NULL) {
utstring_concat(buff, childJSON);
utstring_free(childJSON);
}
}
utstring_printf(buff, "%s%s", JSON_IS_OBJ(node) ? "}" : "]", last);
return buff;
}
int get_files(UT_array *files, UT_array *stats) {
struct dirent *dent;
char *name, *path;
file_stat_t fsb;
int rc=-1,i=0;
DIR *d;
UT_string *s;
utstring_new(s);
utarray_clear(files);
utarray_clear(stats);
if ( (d = opendir(cf.dir)) == NULL) {
syslog(LOG_ERR,"failed to opendir [%s]: %s\n", cf.dir, strerror(errno));
goto done;
}
while ( (dent = readdir(d)) != NULL) {
if (dent->d_type != DT_REG) continue;
if (dent->d_name[0] == '.') continue; /* skip dot files */
// ok, fully qualify it and push it
utstring_clear(s);
utstring_printf(s, "%s/%s", cf.dir, dent->d_name);
path = utstring_body(s);
if (stat(path,&fsb.sb) == -1) {
syslog(LOG_ERR,"can't stat %s: %s", path, strerror(errno));
continue;
}
fsb.file_idx = i++;
utarray_push_back(files, &path);
utarray_push_back(stats, &fsb);
}
rc = 0; // success
done:
utstring_free(s);
if (d) closedir(d);
return rc;
}
int accept_client() {
int rc=-1, i=0;
struct sockaddr_in cin;
socklen_t cin_sz = sizeof(cin);
int fd = accept(cfg.listener_fd,(struct sockaddr*)&cin, &cin_sz);
if (fd == -1) {
fprintf(stderr,"accept: %s\n", strerror(errno));
goto done;
}
if (sizeof(cin)==cin_sz) fprintf(stderr,"connection from %s:%d\n",
inet_ntoa(cin.sin_addr), (int)ntohs(cin.sin_port));
utarray_push_back(cfg.clients,&fd);
/* set up client output buffer. reserve space for full buffer */
UT_string *s; utstring_new(s); utstring_reserve(s,cfg.mb_per_client*1024*1024);
utarray_push_back(cfg.outbufs,&s);
utarray_push_back(cfg.outidxs,&i);
new_epoll(EPOLLIN, fd);
rc=0;
done:
return rc;
}
long readSegmentFromFileSystem(
const char* nid,unsigned long long csn,size_t leftByte, size_t segmentSize,void* buffer)
{
size_t bytesRead=0; size_t fileNameSize;
size_t bytesToRead;
FILE *fp; //file pointer
//variables to update cache_table
UT_string* utnid;
UT_string* filename; utstring_new(filename);
calculate_filename( (char*)nid,csn,filename);
if(CONET_DEBUG)
fprintf(stderr,"[cacheEngine.c:%d]Opening file \"%s\" \n",__LINE__, utstring_body(filename) );
fp = fopen( utstring_body(filename), "rb");
if (fp == NULL)
{
fprintf(stderr, "[cacheEngine.c:%d]Error: Failed to open the file %s; maybe the chunk does not exist\n\n",
__LINE__, filename);
if (CONET_SEVERE_DEBUG) exit(-651);
return ( (size_t) - 1);
}
//!!!Questo fa andare in buffer overflow, il primo parametro dovrebbe essere
//buffer senza +leftByte
//Go to the leftByte location
int fseek_return = fseek(fp,leftByte,SEEK_SET);
if(fseek_return!=0)
{
fprintf(stderr,"[cacheEngine.c:%d]Error reading file %s. fseek returned with %d (ferror() tests the error indicator for the stream pointed to by stream, returning nonzero if it is set.)\n",
__LINE__,utstring_body(filename),fseek_return );
//if(CONET_SEVERE_DEBUG) exit(-652);
return ( (size_t) -0);
}
if(feof(fp))
{
fprintf(stderr,"[cacheEngine.c:%d]Error reading file %s. feof(fp)=%d\n",
__LINE__,utstring_body(filename), feof(fp));
//if(CONET_SEVERE_DEBUG) exit(-644);
return ( (size_t) -0);
}
clearerr(fp);
bytesRead = fread(buffer, 1, segmentSize, fp);
if (bytesRead <= 0)
{
fprintf(stderr,"%s\n",utstring_body(filename) );
fprintf(
stderr,
"[cacheEngine.c:%d] Error: in reading file %s. fread returned with %u. ferror=%d, feof(fp)=%d (ferror() tests the error indicator for the stream pointed to by stream, returning nonzero if it is set.)\n\n",
__LINE__,utstring_body(filename), bytesRead,ferror(fp), feof(fp)
);
if (CONET_SEVERE_DEBUG) exit(-652);
return bytesRead;
}
fclose(fp);
if(CONET_DEBUG)
fprintf(stderr,"[cacheEngine.c:%d]Successfully read %d bytes\n",
__LINE__,(int) bytesRead);
utstring_free(filename);
return bytesRead;
}
//size_t handleChunk(char* nid,char* csn,long tag,size_t left_edge, size_t segment_size,void* buffer)
size_t handleChunk(char* nid,unsigned long long csn,long tag,unsigned long long chunk_size,void* buffer)
{
size_t writtenBytes,fileNameSize;
char* fileName;
char csnToDelete[64];
FILE *fp;
UT_string* folderName;
utstring_new(folderName);
UT_string* tempHexCsn;
utstring_new(tempHexCsn);
UT_string* filename; utstring_new(filename);
//variables for cache table
CacheEntry_t *entry, *tmp_entry;
CacheEntry_t *entryToDelete=NULL;
UT_string *utnid, *filenameToDelete;
// long csnlong;
//folderName=NULL;
if(CONET_DEBUG) fprintf(stderr,"\n[cacheEngine.c:%d]New segment nid=%s, csn=%llu received.\n",
__LINE__,nid,csn);
calculate_filename(nid,csn,filename);
utstring_printf(folderName,CONTAINING_FOLDER);
utstring_printf(folderName,nid);
mkdir(utstring_body(folderName));
fp = fopen(utstring_body(filename) , "w+b");
if (fp == NULL) {
fprintf(stderr, "Error: Failed to open the file %s for writing;\n\n", fileName);
if (CONET_SEVERE_DEBUG) exit(-863);
return ( (size_t) - 1);
}
writtenBytes = fwrite(buffer, 1,chunk_size, fp);
fclose(fp);
if (writtenBytes!=chunk_size) {
fprintf(stderr, "Error in writing %s: writtenBytes=%d, while chunk_size=%d\n",
fileName,writtenBytes,chunk_size);
if (CONET_SEVERE_DEBUG) exit(-8763);
return ( (size_t) - 1);
}
utstring_new(utnid); utstring_printf(utnid,nid);
//utstring_printf(tempHexCsn,"0x");utstring_printf(tempHexCsn,csn);
ADD_TO_CACHE_TABLE(cache_table,entry,tmp_entry,tag,utnid,csn,chunk_size,0,entryToDelete);
//creates json message with nid and of type "stored", and store it to msg_to_controller_buffer;
fill_message(tag, "stored", nid, csn, msg_to_controller_buffer);
//BUFFERLEN is defined in c_json.h
int bytesSentToController=send(socket_to_controller, msg_to_controller_buffer, BUFFERLEN, 0);
if(CONET_DEBUG )
fprintf(stderr,"[cacheEngine.c:%d]Message sent to controller = %s\n",__LINE__,msg_to_controller_buffer);
if (bytesSentToController <=strlen(msg_to_controller_buffer) )
{
fprintf(stderr,"[cacheEngine.c: %d] bytes sent to controller=%d\n",__LINE__, bytesSentToController);
if (CONET_SEVERE_DEBUG) exit(-982);
}
if (entryToDelete!=NULL)
{
sprintf(csnToDelete,"%x",(unsigned int) entryToDelete->csn);
CALCULATE_UTSTRING_FILE_NAME(entryToDelete->nid,csnToDelete,filenameToDelete);
if (remove(utstring_body(filenameToDelete) )==0)
fprintf(stderr,"File %s deleted.\n",utstring_body(filenameToDelete));
else
fprintf(stderr,"Error deleting file %s,\n",utstring_body(filenameToDelete));
//creates json message with nid and of type "stored", and store it to msg_to_controller_buffer;
fill_message(entryToDelete->tag, "deleted", entryToDelete->nid, entryToDelete->csn, msg_to_controller_buffer);
//BUFFERLEN is defined in c_json.h
if (send(socket_to_controller, msg_to_controller_buffer, BUFFERLEN, 0) !=
strlen(msg_to_controller_buffer)
)
fprintf(stderr,"send() sent a different number of bytes than expected\n");
}
fprintf(stderr,"[cacheEngine.c:%d]New chunk added: nid=%s, csn=%llu, tag=%ld\n",__LINE__,nid,csn,tag);
// utstring_free(utnid); utstring_free(filenameToDelete);
utstring_free(filename);
return writtenBytes;
}
/**
* Searches the chunk.
* If the chunk is found, puts segment bytes in buffer and returns the size of the extracted segment
* If no dataCIU is found, returns a negative number
* @param nid network identifier (must be NULL-TERMINATED)
* @param csn chunk sequence number (must be NULL-TERMINATED)
* @param leftByte, segmentSize: the segment start at leftByte and ends segmentSize bytes after.
* @param buffer memory area where to put the segment
* @param is_final: it will be set to 1 if the segment is the final, 0 otherwise
*/
long retrieveSegment(
const char* nid,unsigned long long csn,long tag,size_t leftByte, size_t segmentSize,
void* buffer,unsigned int* is_final, unsigned long long* chunk_size)
{
if(CONET_SEVERE_DEBUG && csn>=100)
{
fprintf(stderr,"[cacheEngine.c:%d] csn=%llu. This is very large. It is not necessarily an error but ... are you sure?\n",__LINE__, csn);
exit(-8776);
}
int bytesRead=0; char* fileName; size_t fileNameSize;
size_t bytesToRead;
FILE *fp; //file pointer
//variables to update cache_table
UT_string* utnid;
char** conversion_pointer=NULL;
if (CONET_DEBUG)
fprintf(stderr,"\n[cacheEngine.c: %d] Request for nid=%s, csn=%ull, tag=%ld\n",
__LINE__,nid,csn,tag);
//update table
utstring_new(utnid);
utstring_printf(utnid, nid );
if(CONET_DEBUG)
fprintf(stderr,"[cacheEngine.c: %d] Updating cache table. Now the chunk nid=%s, csn=%llu is at the head\n",
__LINE__,nid,csn);
*chunk_size=find_on_cache_table(cache_table,utnid,csn);
if (CONET_DEBUG)
fprintf(stderr,"[cacheEngine.c: %d] chunk_size=%llu\n",__LINE__, *chunk_size);
if (*chunk_size<0)
{
if (CONET_SEVERE_DEBUG) exit(-642);
return (long) *chunk_size;
}
bytesRead=readSegmentFromFileSystem(nid,csn,leftByte, segmentSize,buffer);
if (CONET_DEBUG)
fprintf(stderr,"[cacheEngine.c: %d] chunk size is is %llu and %d(leftByte) + %d(bytesRead)=%d\n",
__LINE__, *chunk_size,leftByte ,bytesRead,leftByte+bytesRead);
if (bytesRead<=0)
{
fprintf(stderr,"[cacheEngine.c:%d] reading file: bytesRead=%d\n",
__LINE__,bytesRead);
//if (CONET_SEVERE_DEBUG) exit(-989);
return bytesRead;
}else
{
if (leftByte+bytesRead== *chunk_size)
*is_final=1;
else if (leftByte+bytesRead > *chunk_size)
{
fprintf(stderr,"[cacheEngine.c: %d] Error: Error chunk_size(%d)<leftByte(%d)+bytesRead(%d). This is a nonsense \n",
__LINE__,*chunk_size, leftByte, bytesRead);
if (CONET_SEVERE_DEBUG) exit(-3);
return -3;
}
else *is_final=0;
}
return bytesRead;
}
int main() {
UT_string *s,*t;
char V_TestStr[] = "There are two needle\0s in this \0haystack with needle\0s.";
char V_NeedleStr[] = "needle\0s";
long *V_KMP_Table;
long V_FindPos;
size_t V_StartPos;
size_t V_FindCnt;
utstring_new(s);
utstring_new(t);
utstring_bincpy(s, V_TestStr, sizeof(V_TestStr)-1);
printf("\"%s\" len=%u\n", utstring_body(s), utstring_len(s));
utstring_bincpy(t, V_NeedleStr, sizeof(V_NeedleStr)-1);
printf("\"%s\" len=%u\n", utstring_body(t), utstring_len(t));
V_KMP_Table = (long *)malloc(sizeof(long) * (utstring_len(t) + 1));
if (V_KMP_Table != NULL)
{
_utstring_BuildTableR(utstring_body(t), utstring_len(t), V_KMP_Table);
V_FindCnt = 0;
V_FindPos = 0;
V_StartPos = utstring_len(s) - 1;
do
{
V_FindPos = _utstring_findR(utstring_body(s),
V_StartPos + 1,
utstring_body(t),
utstring_len(t),
V_KMP_Table);
if (V_FindPos >= 0)
{
V_FindCnt++;
V_StartPos = V_FindPos - 1;
}
printf("utstring_find()=%ld\n", V_FindPos);
} while (V_FindPos >= 0);
printf("FindCnt=%u\n", V_FindCnt);
free(V_KMP_Table);
}
else
{
printf("malloc() failed...\n");
}
utstring_free(t);
utstring_clear(s);
utstring_printf(s,"ABC ABCDAB ABCDABCDABDE");
int o;
o=utstring_find( s, -9, "ABC", 3 ) ; printf("expect 15 %d\n",o);
o=utstring_find( s, 3, "ABC", 3 ) ; printf("expect 4 %d\n",o);
o=utstring_find( s, 16, "ABC", 3 ) ; printf("expect -1 %d\n",o);
o=utstring_findR( s, -9, "ABC", 3 ) ; printf("expect 11 %d\n",o);
o=utstring_findR( s, 12, "ABC", 3 ) ; printf("expect 4 %d\n",o);
o=utstring_findR( s, 13, "ABC", 3 ) ; printf("expect 11 %d\n",o);
o=utstring_findR( s, 2, "ABC", 3 ) ; printf("expect 0 %d\n",o);
utstring_free(s);
return 0;
}
int main(int argc, char *argv[]) {
#ifndef _OPENMP
fprintf(stderr, "\nERROR: Program built with compiler lacking OpenMP support.\n");
fprintf(stderr, "See SEAStAR README file for information about suitable compilers.\n");
exit(EXIT_FAILURE);
#endif
///////////////////////////
// Variable declarations
///////////////////////////
// Input filenames
UT_string *in_read1_fq_fn, *in_read2_fq_fn, *in_single1_fq_fn, *in_single2_fq_fn;
utstring_new(in_read1_fq_fn);
utstring_new(in_read2_fq_fn);
utstring_new(in_single1_fq_fn);
utstring_new(in_single2_fq_fn);
// Output filenames
UT_string *out_read1_fn, *out_read2_fn, *out_single1_fn, *out_single2_fn, *out_mates_fn, *out_filetype;
utstring_new(out_filetype);
utstring_new(out_read1_fn);
utstring_new(out_read2_fn);
utstring_new(out_single1_fn);
utstring_new(out_single2_fn);
utstring_new(out_mates_fn);
// Read name prefix
UT_string *out_read_prefix;
utstring_new(out_read_prefix);
// Flags
int singles_flag = 0; // 1 when two output singles files being written
int num_input_singles_files = 0;
// Read counters
unsigned long int mp_org = 0, R1_org = 0, R2_org = 0, singlet1_org = 0, singlet2_org = 0;
unsigned long int mp_cnt = 0, R1_cnt = 0, R2_cnt = 0, singlet1_cnt = 0, singlet2_cnt = 0, s1_cnt = 0, s2_cnt = 0;
unsigned long int comp_r1 = 0, comp_r2 = 0, comp_s1 = 0, comp_s2 = 0;
unsigned long int read1_singlet_cnt = 0, read2_singlet_cnt = 0;
////////////////////////////////////////////////////////////////////////
// All done with variable declarations!!
///////////////////////////////////
// Command line argtable settings
///////////////////////////////////
struct arg_lit *gzip = arg_lit0("z", "gzip", "Output converted files in gzip compressed format. [NULL]");
struct arg_lit *inv_singles = arg_lit0("v", "invert_singles", "Causes singles output to be the inverse of the input. 2->1 or 1->2 [NULL]");
struct arg_lit *num_singles = arg_lit0("s", "singles", "Write two singlet files, one for each mate-paired input file. [NULL]");
struct arg_rem *sing_rem = arg_rem(NULL, "Note! -v is only valid when there are input singlet reads. -s is only valid when there are NO input singlet reads.");
struct arg_str *pre_read_id = arg_str0(NULL, "prefix", "<string>", "Prefix to add to read identifiers. [out_prefix]");
struct arg_lit *no_pre = arg_lit0(NULL, "no_prefix", "Do not change the read names in any way. [NULL]");
struct arg_lit *pre_read_len = arg_lit0(NULL, "add_len", "Add the final trimmed length value to the read id prefix. [length not added]");
struct arg_dbl *prob = arg_dbl0("p","correct_prob","<d>","Probability that output reads are correct. 0.0 disables quality trimming. [0.5]");
struct arg_int *fixed_len = arg_int0("f","fixed_len","<u>","Trim all reads to a fixed length, still filtering on quality [no fixed length]");
struct arg_int *len = arg_int0("l","min_read_len","<u>","Minimum length of a singlet or longest-mate in nucleotides [24]");
struct arg_int *mate_len = arg_int0("m","min_mate_len","<u>","Minimum length of the shortest mate in nucleotides [min_read_len]");
struct arg_dbl *entropy = arg_dbl0("e","entropy_filter","<d>","Remove reads with per position information below given value (in bits per dinucleotide) [No filter]");
struct arg_lit *entropy_strict = arg_lit0(NULL, "entropy_strict", "Reject reads for low entropy overall, not just the retained part after trimming [NULL]");
struct arg_lit *mates = arg_lit0(NULL, "mates_file", "Produce a Velvet compatible interleaved paired read output file (e.g. <out_prefix>_mates.fastq). [NULL]");
struct arg_lit *no_rev = arg_lit0(NULL, "no_rev", "By default, the second read in each pair is reversed for colorspace --mate-file output. --no_rev disables reversing. [rev]");
struct arg_lit *only_mates = arg_lit0(NULL, "only_mates", "Supress writing .read1 and .read2 outputs. Requires --mates_file. [NULL]");
struct arg_lit *fasta = arg_lit0(NULL, "fasta", "Write FASTA format files instead of FASTQ for all outputs (e.g. <out_prefix>.<read_type>.fasta). [FASTQ]");
struct arg_file *input = arg_file1(NULL, NULL, "<in_prefix>", "Input file prefix: (e.g. <in_prefix>_single.fastq [<in_prefix>_read1.fastq <in_prefix>_read2.fastq]) ");
struct arg_file *output = arg_file1(NULL, NULL, "<out_prefix>", "Output file prefix: (e.g. <out_prefix>_single.fastq [<out_prefix>_read1.fastq <out_prefix>_read2.fastq]) ");
struct arg_lit *version = arg_lit0(NULL,"version","Print the build version and exit.");
struct arg_lit *h = arg_lit0("h", "help", "Request help.");
struct arg_end *end = arg_end(20);
void *argtable[] = {h,version,gzip,inv_singles,num_singles,sing_rem,prob,len,mate_len,fixed_len,pre_read_id,pre_read_len,no_pre,entropy,entropy_strict,mates,no_rev,only_mates,fasta,input,output,end};
int arg_errors = 0;
////////////////////////////////////////////////////////////////////////
// Handle command line processing (via argtable2 library)
////////////////////////////////////////////////////////////////////////
arg_errors = arg_parse(argc, argv, argtable);
if (version->count) {
fprintf(stderr, "%s version: %s\n", argv[0], SS_BUILD_VERSION);
exit(EXIT_SUCCESS);
}
if (h->count) {
fprintf(stderr,"\ntrim_fastq is a utility for performing quality and information-based\n");
fprintf(stderr,"trimming on paired or unpaired, nucleotide or SOLiD colorspace reads. \n\n");
arg_print_syntaxv(stderr, argtable, "\n\n");
arg_print_glossary(stderr, argtable, "%-25s %s\n");
fprintf(stderr, "\nInput and output \"prefixes\" are the part of the filename before:\n");
fprintf(stderr, "_single.fastq [_read1.fastq _read2.fastq] A singlets (single) file\n");
fprintf(stderr, "is required. Mate-paired read files are automatically used if present.\n");
fprintf(stderr, "Multiple output files only produced for mate-paired inputs.\n");
fprintf(stderr, "\nNote! Input and output files may be gzipped, and outputs can be written\n");
fprintf(stderr, "as either FASTQ or FASTA format files.\n");
exit(EXIT_FAILURE);
}
if (arg_errors) {
arg_print_errors(stderr, end, "trimfastq");
arg_print_syntaxv(stderr, argtable, "\n");
exit(EXIT_FAILURE);
}
// Validate entropy
if (entropy->count) {
entropy_cutoff = entropy->dval[0];
if ((entropy_cutoff < 0.0) || (entropy_cutoff > 4.0)) {
fprintf(stderr, "entropy_filter must be [0.0 - 4.0] \n");
exit(EXIT_FAILURE);
}
strict_ent = entropy_strict->count;
} else {
if (entropy_strict->count) {
fprintf(stderr, "Error: --entropy_strict requires --entropy_filter.\n");
exit(EXIT_FAILURE);
}
entropy_cutoff = -1.0;
}
// Validate error_prob
if (prob->count) {
err_prob = prob->dval[0];
if ((err_prob < 0.0) || (err_prob > 1.0)) {
fprintf(stderr, "--correct_prob (-p) must be 0.0 - 1.0 inclusive\n");
exit(EXIT_FAILURE);
}
} else {
err_prob = 0.5;
}
// Validate min read len
if (len->count) {
min_len = len->ival[0];
if (min_len <= 0) {
fprintf(stderr, "min_read_len must be > 0\n");
exit(EXIT_FAILURE);
}
} else {
min_len = 24;
}
// Validate min mate len
if (mate_len->count) {
min_mate_len = mate_len->ival[0];
if (min_mate_len <= 0) {
fprintf(stderr, "min_mate_len must be > 0\n");
exit(EXIT_FAILURE);
}
if (min_mate_len > min_len) {
fprintf(stderr, "min_mate_len must be <= min_len\n");
exit(EXIT_FAILURE);
}
} else {
min_mate_len = min_len;
}
if (fixed_len->count) {
fix_len = min_mate_len = min_len = fixed_len->ival[0];
if ((mate_len->count) || (len->count)) {
fprintf(stderr, "fixed_len cannot be used with min_read_len or min_mate_len\n");
exit(EXIT_FAILURE);
}
if (fix_len <= 0) {
fprintf(stderr, "fixed_len must be > 0\n");
exit(EXIT_FAILURE);
}
} else {
fix_len = 0;
}
if (pre_read_id->count) {
if (no_pre->count) {
fprintf(stderr, "Error: Both --prefix and --no_prefix were specified.\n");
exit(EXIT_FAILURE);
}
if (! strlen(pre_read_id->sval[0])) {
fprintf(stderr, "Read ID prefix may not be zero length.\n");
exit(EXIT_FAILURE);
}
if (strchr(pre_read_id->sval[0], ':') || strchr(pre_read_id->sval[0], '|') || strchr(pre_read_id->sval[0], '+') || strchr(pre_read_id->sval[0], '/')) {
fprintf(stderr, "Read ID prefix '%s' may not contain the characters ':', '|', '+' or '/'.\n", pre_read_id->sval[0]);
exit(EXIT_FAILURE);
}
// Build default read ID prefix
ss_strcat_utstring(out_read_prefix, pre_read_id->sval[0]);
} else {
if (!no_pre->count) {
if (strchr(output->filename[0], ':') || strchr(output->filename[0], '|') || strchr(output->filename[0], '+') || strchr(output->filename[0], '/')) {
fprintf(stderr, "Read ID prefix '%s' (from output prefix) may not contain the characters ':', '|', '+' or '/'.\n", output->filename[0]);
fprintf(stderr, "Hint: Use the --prefix parameter if the output file prefix contains path information.\n");
exit(EXIT_FAILURE);
}
// Build default read ID prefix
ss_strcat_utstring(out_read_prefix, output->filename[0]);
}
}
if ((only_mates->count) && (!mates->count)) {
fprintf(stderr, "--only_mates requires --mates.\n");
exit(EXIT_FAILURE);
}
if ((no_rev->count) && (!mates->count)) {
fprintf(stderr, "--no_rev requires --mates.\n");
exit(EXIT_FAILURE);
}
// Check for null string prefixes
if (!(strlen(input->filename[0]) && strlen(output->filename[0]))) {
fprintf(stderr, "Error: NULL prefix strings are not permitted.\n");
exit(EXIT_FAILURE);
}
// Construct input filenames
utstring_printf(in_read1_fq_fn, "%s.read1.fastq", input->filename[0]);
utstring_printf(in_read2_fq_fn, "%s.read2.fastq", input->filename[0]);
utstring_printf(in_single1_fq_fn, "%s.single.fastq", input->filename[0]);
FILE *in_read_file = NULL;
num_input_singles_files = 1;
// Try to open a singlet fastq file
// Check singlet output options -s and -v
// Set input singlet names to
// - *.single.fastq or
// - *.single1.fastq and *.single2.fastq
if (!(in_read_file = ss_get_gzFile(utstring_body(in_single1_fq_fn), "r"))) {
utstring_clear(in_single1_fq_fn);
utstring_printf(in_single1_fq_fn, "%s.single1.fastq", input->filename[0]);
utstring_printf(in_single2_fq_fn, "%s.single2.fastq", input->filename[0]);
num_input_singles_files = 2;
if ((in_read_file = ss_get_gzFile(utstring_body(in_single1_fq_fn), "r")) || (in_read_file = ss_get_gzFile(utstring_body(in_single2_fq_fn), "r"))) {
singles_flag = 1; // Two singlet outputs
} else {
singles_flag = num_singles->count; // Number of singlet outputs set by -s parm
if (inv_singles->count) {
fprintf(stderr, "Error: Invalid option -v, No input singlet file(s) found. Use -s to select multiple output singlet files.\n");
exit(EXIT_FAILURE);
}
}
}
if (in_read_file) {
gzclose(in_read_file);
if (num_singles->count) {
fprintf(stderr, "Error: Invalid option -s, Input singlet file(s) found, use -v to change the number of output singlet files.\n");
exit(EXIT_FAILURE);
}
}
// singles->count inverts the current singles file input scheme
singles_flag = (singles_flag ^ inv_singles->count);
// Check if input fastq is colorspace
// If some files are colorspace and some are basespace, throw an error
int fcount = 0;
int cscount = 0;
fcount += ss_is_fastq(utstring_body(in_read1_fq_fn));
fcount += ss_is_fastq(utstring_body(in_read2_fq_fn));
fcount += ss_is_fastq(utstring_body(in_single1_fq_fn));
fcount += ss_is_fastq(utstring_body(in_single2_fq_fn));
cscount += (ss_is_fastq(utstring_body(in_read1_fq_fn)) && ss_is_colorspace_fastq(utstring_body(in_read1_fq_fn)));
cscount += (ss_is_fastq(utstring_body(in_read2_fq_fn)) && ss_is_colorspace_fastq(utstring_body(in_read2_fq_fn)));
cscount += (ss_is_fastq(utstring_body(in_single1_fq_fn)) && ss_is_colorspace_fastq(utstring_body(in_single1_fq_fn)));
cscount += (ss_is_fastq(utstring_body(in_single2_fq_fn)) && ss_is_colorspace_fastq(utstring_body(in_single2_fq_fn)));
if (cscount && (cscount != fcount)) {
printf("Error: Mixed colorspace and basespace FASTQ files detected\n");
exit(EXIT_FAILURE);
}
colorspace_flag = cscount ? 1 : 0;
// Output filenames
if (fasta->count) {
ss_strcat_utstring(out_filetype, "fasta");
read_count_divisor = 2;
} else {
ss_strcat_utstring(out_filetype, "fastq");
read_count_divisor = 4;
}
if (!only_mates->count) {
utstring_printf(out_read1_fn, "%s.read1.%s", output->filename[0], utstring_body(out_filetype));
utstring_printf(out_read2_fn, "%s.read2.%s", output->filename[0], utstring_body(out_filetype));
}
if (singles_flag == 1) {
utstring_printf(out_single1_fn, "%s.single1.%s", output->filename[0], utstring_body(out_filetype));
utstring_printf(out_single2_fn, "%s.single2.%s", output->filename[0], utstring_body(out_filetype));
} else {
utstring_printf(out_single1_fn, "%s.single.%s", output->filename[0], utstring_body(out_filetype));
}
if (mates->count) {
utstring_printf(out_mates_fn, "%s.mates.%s", output->filename[0], utstring_body(out_filetype));
}
////////////////////////////////////////////////////////////////////////////////////////////////
// Begin processing!
#ifdef _OPENMP
omp_set_num_threads(10);
#endif
// This is the value of a non-valid pipe descriptor
#define NO_PIPE 0
int r1_pipe[2];
int r2_pipe[2];
int s1_pipe[2];
int s2_pipe[2];
pipe(r1_pipe);
pipe(r2_pipe);
pipe(s1_pipe);
pipe(s2_pipe);
int r1_out_pipe[2];
int r2_out_pipe[2];
int mates_out_pipe[2];
int s1_out_pipe[2];
int s2_out_pipe[2];
pipe(r1_out_pipe);
pipe(r2_out_pipe);
pipe(mates_out_pipe);
pipe(s1_out_pipe);
pipe(s2_out_pipe);
#pragma omp parallel sections default(shared)
{
#pragma omp section
{ // Read1 reader
fq_stream_trimmer(in_read1_fq_fn, r1_pipe[1], out_read_prefix, no_pre->count, pre_read_len->count, &comp_r1, &R1_org, '\0', fasta->count);
}
#pragma omp section
{ // Read1 writer
R1_cnt = ss_stream_writer(out_read1_fn, r1_out_pipe[0], gzip->count) / read_count_divisor;
}
#pragma omp section
{ // Read2 reader
fq_stream_trimmer(in_read2_fq_fn, r2_pipe[1], out_read_prefix, no_pre->count, pre_read_len->count, &comp_r2, &R2_org, '\0', fasta->count);
}
#pragma omp section
{ // Read2 writer
R2_cnt = ss_stream_writer(out_read2_fn, r2_out_pipe[0], gzip->count) / read_count_divisor;
}
#pragma omp section
{ // Single1 reader
// When there is only one input singles file, but two output singles files, then supply which mate to use for this stream in the split parameter
if ((singles_flag) && (num_input_singles_files == 1)) {
singlet1_cnt = fq_stream_trimmer(in_single1_fq_fn, s1_pipe[1], out_read_prefix, no_pre->count, pre_read_len->count, &comp_s1, &singlet1_org, '1', fasta->count);
} else {
singlet1_cnt = fq_stream_trimmer(in_single1_fq_fn, s1_pipe[1], out_read_prefix, no_pre->count, pre_read_len->count, &comp_s1, &singlet1_org, '\0', fasta->count);
}
}
#pragma omp section
{ // Single1 writer
s1_cnt = ss_stream_writer(out_single1_fn, s1_out_pipe[0], gzip->count) / read_count_divisor;
}
#pragma omp section
{ // Single2 reader
// When there is only one input singles file, but two output singles files, then supply which mate to use for this stream in the split parameter
if ((singles_flag) && (num_input_singles_files == 1)) {
singlet2_cnt = fq_stream_trimmer(in_single1_fq_fn, s2_pipe[1], out_read_prefix, no_pre->count, pre_read_len->count, &comp_s2, &singlet2_org, '2', fasta->count);
} else {
singlet2_cnt = fq_stream_trimmer(in_single2_fq_fn, s2_pipe[1], out_read_prefix, no_pre->count, pre_read_len->count, &comp_s2, &singlet2_org, '\0', fasta->count);
}
}
#pragma omp section
{ // Single2 writer
s2_cnt = ss_stream_writer(out_single2_fn, s2_out_pipe[0], gzip->count) / read_count_divisor;
}
#pragma omp section
{ // Velvet mates writer
// Divide count by 2 because both R1 and R2 reads go through this writer
mp_cnt = ss_stream_writer(out_mates_fn, mates_out_pipe[0], gzip->count) / 2 / read_count_divisor;
}
#pragma omp section
{ // Dispatcher
// Allocate data buffer strings
UT_string *r1_data;
utstring_new(r1_data);
UT_string *r2_data;
utstring_new(r2_data);
UT_string *s1_data;
utstring_new(s1_data);
UT_string *s2_data;
utstring_new(s2_data);
UT_string *rev_tmp;
utstring_new(rev_tmp);
UT_string *rev_data;
utstring_new(rev_data);
// Pipes
FILE *r1_in = fdopen(r1_pipe[0],"r");
FILE *r2_in = fdopen(r2_pipe[0],"r");
FILE *s1_in = fdopen(s1_pipe[0],"r");
FILE *s2_in = fdopen(s2_pipe[0],"r");
FILE *mates_out = fdopen(mates_out_pipe[1],"w");
FILE *r1_out = fdopen(r1_out_pipe[1],"w");
FILE *r2_out = fdopen(r2_out_pipe[1],"w");
FILE *s1_out = fdopen(s1_out_pipe[1],"w");
FILE *s2_out = fdopen(s2_out_pipe[1],"w");
if (!singles_flag) {
fclose(s2_out);
s2_out = s1_out;
}
// Flags for data left in single files
int single1_hungry = 1;
int single2_hungry = 1;
// Handle read1 and read2 files
while (ss_get_utstring(r1_in, r1_data)) {
if (!ss_get_utstring(r2_in, r2_data)) {
fprintf(stderr, "Error: Input read1 and read2 files are not synced\n");
exit(EXIT_FAILURE);
}
if (keep_read(r1_data)) {
if (keep_read(r2_data)) {
// Output both read1 and read2
if (mates->count) {
if (only_mates->count) {
// Interleaved velvet output only
output_read(r1_data, NULL, NULL, r1_in, NULL, mates_out, fasta->count);
if (no_rev->count || !colorspace_flag) {
output_read(r2_data, NULL, NULL, r2_in, NULL, mates_out, fasta->count);
} else {
output_read(r2_data, rev_data, rev_tmp, r2_in, NULL, mates_out, fasta->count);
}
} else {
// Interleaved velvet output and normal read file output
output_read(r1_data, NULL, NULL, r1_in, r1_out, mates_out, fasta->count);
if (no_rev->count || !colorspace_flag) {
output_read(r2_data, NULL, NULL, r2_in, r2_out, mates_out, fasta->count);
} else {
output_read(r2_data, rev_data, rev_tmp, r2_in, r2_out, mates_out, fasta->count);
}
}
} else {
// No interleaved velvet output
output_read(r1_data, NULL, NULL, r1_in, r1_out, NULL, fasta->count);
output_read(r2_data, NULL, NULL, r2_in, r2_out, NULL, fasta->count);
}
} else {
// Discard read2, output read1 as singlet
output_read(r1_data, NULL, NULL, r1_in, s1_out, NULL, fasta->count);
read1_singlet_cnt++;
}
} else {
if (keep_read(r2_data)) {
// Discard read1, output read2 as singlet
output_read(r2_data, NULL, NULL, r2_in, s2_out, NULL, fasta->count);
read2_singlet_cnt++;
}
}
// Process reads from singles here to take advantage of
// parallelism
if (single1_hungry || single2_hungry) {
if (single1_hungry) {
if (ss_get_utstring(s1_in, s1_data)) {
if (keep_read(s1_data)) {
output_read(s1_data, NULL, NULL, s1_in, s1_out, NULL, fasta->count);
}
} else {
single1_hungry = 0;
}
}
if (single2_hungry) {
if (ss_get_utstring(s2_in, s2_data)) {
if (keep_read(s2_data)) {
output_read(s2_data, NULL, NULL, s2_in, s2_out, NULL, fasta->count);
}
} else {
single2_hungry = 0;
}
}
}
}
while (single1_hungry || single2_hungry) {
if (single1_hungry) {
if (ss_get_utstring(s1_in, s1_data)) {
if (keep_read(s1_data)) {
output_read(s1_data, NULL, NULL, s1_in, s1_out, NULL, fasta->count);
}
} else {
single1_hungry = 0;
}
}
if (single2_hungry) {
if (ss_get_utstring(s2_in, s2_data)) {
if (keep_read(s2_data)) {
output_read(s2_data, NULL, NULL, s2_in, s2_out, NULL, fasta->count);
}
} else {
single2_hungry = 0;
}
}
}
fclose(r1_in);
fclose(r2_in);
fclose(s1_in);
fclose(s2_in);
fclose(mates_out);
fclose(r1_out);
fclose(r2_out);
fclose(s1_out);
if (singles_flag) {
fclose(s2_out);
}
// Free buffers
utstring_free(r1_data);
utstring_free(r2_data);
utstring_free(s1_data);
utstring_free(s2_data);
utstring_free(rev_tmp);
utstring_free(rev_data);
}
}
if (!(R1_org+singlet1_org+singlet2_org)) {
fprintf(stderr, "ERROR! No reads found in input files, or input(s) not found.\n");
exit(EXIT_FAILURE);
}
if (R1_org != R2_org) {
fprintf(stderr, "\nWarning! read1 and read2 fastq files did not contain an equal number of reads. %lu %lu\n", R1_org, R2_org);
}
if ((R1_org + R2_org) && !(singlet1_cnt + singlet2_cnt)) {
fprintf(stderr, "\nWarning! read1/read2 files were processed, but no corresponding input singlets were found.\n");
}
if (entropy->count) {
printf("\nLow complexity reads discarded: Read1: %lu, Read2: %lu, Singlets: %lu %lu\n", comp_r1, comp_r2, comp_s1, comp_s2);
}
mp_org = R1_org;
if (!only_mates->count) {
mp_cnt = R1_cnt;
}
printf("\nMatepairs: Before: %lu, After: %lu\n", mp_org, mp_cnt);
printf("Singlets: Before: %lu %lu After: %lu %lu\n", singlet1_org, singlet2_org, s1_cnt, s2_cnt);
printf("Read1 singlets: %lu, Read2 singlets: %lu, Original singlets: %lu %lu\n", read1_singlet_cnt, read2_singlet_cnt, singlet1_cnt, singlet2_cnt);
printf("Total Reads Processed: %lu, Reads retained: %lu\n", 2*mp_org+singlet1_org+singlet2_org, 2*mp_cnt+s1_cnt+s2_cnt);
utstring_free(in_read1_fq_fn);
utstring_free(in_read2_fq_fn);
utstring_free(in_single1_fq_fn);
utstring_free(in_single2_fq_fn);
utstring_free(out_read1_fn);
utstring_free(out_read2_fn);
utstring_free(out_single1_fn);
utstring_free(out_single2_fn);
utstring_free(out_mates_fn);
utstring_free(out_filetype);
utstring_free(out_read_prefix);
exit(EXIT_SUCCESS);
}
