struct pkg_repo_it *
pkg_repo_binary_require(struct pkg_repo *repo, const char *provide)
{
sqlite3_stmt *stmt;
sqlite3 *sqlite = PRIV_GET(repo);
UT_string *sql = NULL;
int ret;
const char basesql[] = ""
"SELECT p.id, p.origin, p.name, p.version, p.comment, "
"p.name as uniqueid, "
"p.prefix, p.desc, p.arch, p.maintainer, p.www, "
"p.licenselogic, p.flatsize, p.pkgsize, "
"p.cksum, p.manifestdigest, p.path AS repopath, '%s' AS dbname "
"FROM packages AS p INNER JOIN pkg_requires AS ps ON "
"p.id = ps.package_id "
"WHERE ps.require_id = (SELECT id FROM requires WHERE require=?1);";
utstring_new(sql);
utstring_printf(sql, basesql, repo->name);
pkg_debug(4, "Pkgdb: running '%s'", utstring_body(sql));
ret = sqlite3_prepare_v2(sqlite, utstring_body(sql), -1, &stmt, NULL);
if (ret != SQLITE_OK) {
ERROR_SQLITE(sqlite, utstring_body(sql));
utstring_free(sql);
return (NULL);
}
utstring_free(sql);
sqlite3_bind_text(stmt, 1, provide, -1, SQLITE_TRANSIENT);
return (pkg_repo_binary_it_new(repo, stmt, PKGDB_IT_FLAG_ONCE));
}
int main() {
UT_string *s,*t;
char V_TestStr[] = "There are two needle\0s in this \0haystack with needle\0s.";
char V_NeedleStr[] = "needle\0s";
long *V_KMP_Table;
long V_FindPos;
size_t V_StartPos;
size_t V_FindCnt;
utstring_new(s);
utstring_new(t);
utstring_bincpy(s, V_TestStr, sizeof(V_TestStr)-1);
printf("\"%s\" len=%u\n", utstring_body(s), utstring_len(s));
utstring_bincpy(t, V_NeedleStr, sizeof(V_NeedleStr)-1);
printf("\"%s\" len=%u\n", utstring_body(t), utstring_len(t));
V_KMP_Table = (long *)malloc(sizeof(long) * (utstring_len(t) + 1));
if (V_KMP_Table != NULL)
{
_utstring_BuildTable(utstring_body(t), utstring_len(t), V_KMP_Table);
V_FindCnt = 0;
V_FindPos = 0;
V_StartPos = 0;
do
{
V_FindPos = _utstring_find(utstring_body(s) + V_StartPos,
utstring_len(s) - V_StartPos,
utstring_body(t),
utstring_len(t),
V_KMP_Table);
if (V_FindPos >= 0)
{
V_FindPos += V_StartPos;
V_FindCnt++;
V_StartPos = V_FindPos + 1;
}
printf("utstring_find()=%ld\n", V_FindPos);
} while (V_FindPos >= 0);
printf("FindCnt=%u\n", V_FindCnt);
free(V_KMP_Table);
}
else
{
printf("malloc() failed...\n");
}
utstring_free(s);
utstring_free(t);
return 0;
}
// TODO: finish to test this one
int o_bin_dbcreate(orientdb *o, orientdb_con *c, struct timeval *timeout, int nonblocking, const char *dbname, int dbtype) {
UT_string *s;
int i;
debug_note(o, ORIENT_INFO, "executing binary method: DB_CREATE\n");
i = get_con_fd(o, c);
if (i == -1) return -1;
// check the dbname and dbtype parameters
if ((dbname == NULL) || (strlen(dbname) == 0) || ((dbtype != O_DBTYPE_LOCAL) && (dbtype != O_DBTYPE_MEMORY))) return -1;
utstring_new(s);
// send the operation we want to execute to the server
i = o_sendrequest(o, c, O_DB_CREATE);
// send db name
utstring_printf(s, dbname);
i = o_putstring(o, c, s);
utstring_clear(s);
switch (dbtype) {
case O_DBTYPE_LOCAL:
utstring_printf(s, "local");
break;
case O_DBTYPE_MEMORY:
utstring_printf(s, "memory");
break;
default:
break;
}
// send db type
i = o_putstring(o, c, s);
utstring_clear(s);
/*
i = o_sendcredentials(o, c, ORIENT_USER);
*/
i = o_flushsendbuffer(o, c, timeout);
i = o_getresponse(o, c, timeout);
if (i == 1) { /* EXCEPTION!! */
debug_note(o, ORIENT_INFO, "received an exception\n");
i = o_getexceptions(o, c, timeout, s);
debug_note(o, ORIENT_INFO, "E: %s\n", utstring_body(s));
utstring_free(s);
return -1;
}
utstring_free(s);
return 0;
}
short o_bin_dataclusteradd(orientdb *o, orientdb_con *c, struct timeval *timeout, int nonblocking, const char *type, const char *name, const char *filename, int initialsize) {
UT_string *s;
int i;
short x;
debug_note(o, ORIENT_INFO, "executing binary method: DATACLUSTER_ADD\n");
i = get_con_fd(o, c);
if (i == -1) return -1;
utstring_new(s);
// send the operation we want to execute to the server
i = o_sendrequest(o, c, O_DATACLUSTER_ADD);
// send cluster type
utstring_printf(s, type);
i = o_putstring(o, c, s);
utstring_clear(s);
// send cluster name
utstring_printf(s, name);
i = o_putstring(o, c, s);
utstring_clear(s);
if (strcmp(type, O_CLUSTER_PHYSICAL) == 0) { // the physical cluster need filename and initialsize, while memory and logical do not need them.
// send cluster filename
utstring_printf(s, filename);
i = o_putstring(o, c, s);
utstring_clear(s);
// send cluster initialsize in bytes
i = o_putint(o, c, initialsize);
}
i = o_flushsendbuffer(o, c, timeout);
i = o_getresponse(o, c, timeout);
if (i == 1) { /* EXCEPTION!! */
debug_note(o, ORIENT_INFO, "received an exception\n");
i = o_getexceptions(o, c, timeout, s);
debug_note(o, ORIENT_INFO, "E: %s\n", utstring_body(s));
utstring_free(s);
return -1;
}
// get the new cluster id
x = o_getshort(o, c, timeout);
utstring_free(s);
return x;
}
char *mtex2MML_combine_row_data(UT_array **environment_data_stack)
{
/* if no information was provided, give a standard sizing */
if (utarray_len(*environment_data_stack) == 0) {
const char* s = "rowspacing=\"0.5ex\" rowlines=\"none\"";
char* c = (char*)malloc(strlen(s) + 1);
strcpy(c, s);
return c;
}
envdata_t *row_data_elem = (envdata_t*) utarray_front(*environment_data_stack);
char *row_spacing_data = row_data_elem->rowspacing,
*row_lines_data = row_data_elem->rowlines,
*row_attr;
UT_string *row_attr_data;
utstring_new(row_attr_data);
/* combine the row spacing and row lines data */
utstring_printf(row_attr_data, "%s%s\" %s\"", "rowspacing=\"", row_spacing_data, row_lines_data);
row_attr = string_dup(utstring_body(row_attr_data));
utarray_erase(*environment_data_stack, 0, 1);
utstring_free(row_attr_data);
return row_attr;
}
char *mtex2MML_remove_excess_pipe_chars(char *string)
{
UT_string *columnalign;
utstring_new(columnalign);
char *dupe = string_dup(string);
char *token = strtok(dupe, " ");
char *attr_columnalign;
while (token != NULL) {
if (strncmp(token, "s", 1) != 0 && strncmp(token, "d", 1) != 0) {
utstring_printf(columnalign, "%s ", token);
}
token = strtok(NULL, " ");
}
attr_columnalign = string_dup(utstring_body(columnalign));
free(dupe);
utstring_free(columnalign);
if (strlen(attr_columnalign) > 0) {
mtex2MML_remove_last_char(attr_columnalign); /* remove the final space */
}
return attr_columnalign;
}
/* clean up the client output buffers and slots in fd/buf arrays */
void discard_client_buffers(int pos) {
UT_string **s = (UT_string**)utarray_eltptr(cfg.outbufs,pos);
utstring_free(*s); // deep free string
utarray_erase(cfg.outbufs,pos,1); // erase string pointer
utarray_erase(cfg.outidxs,pos,1); // erase write index
utarray_erase(cfg.clients,pos,1); // erase client descriptor
}
luarest_status invoke_application(application* apps, UT_string* url, luarest_method m, luarest_response* res_code,
luarest_content_type* con_type, UT_string* res_buf)
{
application* app = NULL;
service *service;
char* pch = NULL;
char* tmp = utstring_body(url);
UT_string* app_name;
UT_string* key;
utstring_new(app_name);
pch = strchr(++tmp, '/');
if (pch == NULL) {
return(LUAREST_ERROR);
}
utstring_bincpy(app_name, tmp, pch-tmp);
HASH_FIND(hh, apps, utstring_body(app_name), utstring_len(app_name), app);
utstring_free(app_name);
if (app == NULL) {
return(LUAREST_ERROR);
}
utstring_new(key);
utstring_printf(key, "M%d#P%s", m, pch);
HASH_FIND(hh, app->s, utstring_body(key), utstring_len(key), service);
if (service == NULL) {
return(LUAREST_ERROR);
}
invoke_lua(app->lua_state, service->callback_ref, res_code, con_type, res_buf);
return(LUAREST_SUCCESS);
}
int o_bin_shutdown(orientdb *o, orientdb_con *c, struct timeval *timeout, int nonblocking) {
UT_string *s = NULL;
int i;
debug_note(o, ORIENT_INFO, "executing binary method: SHUTDOWN\n");
i = get_con_fd(o, c);
if (i == -1) return -1;
// send the operation we want to execute to the server
i = o_sendrequest(o, c, O_SHUTDOWN);
i = o_sendcredentials(o, c, ORIENT_ADMIN);
i = o_flushsendbuffer(o, c, timeout);
i = o_getresponse(o, c, timeout);
if (i == 1) { /* EXCEPTION!! */
debug_note(o, ORIENT_INFO, "received an exception\n");
utstring_new(s);
i = o_getexceptions(o, c, timeout, s);
debug_note(o, ORIENT_INFO, "E: %s\n", utstring_body(s));
utstring_free(s);
return -1;
}
return 0;
}
struct pkg_repo_it *
pkg_repo_binary_query(struct pkg_repo *repo, const char *pattern, match_t match)
{
sqlite3 *sqlite = PRIV_GET(repo);
sqlite3_stmt *stmt = NULL;
UT_string *sql = NULL;
const char *comp = NULL;
int ret;
char basesql[BUFSIZ] = ""
"SELECT id, origin, name, name as uniqueid, version, comment, "
"prefix, desc, arch, maintainer, www, "
"licenselogic, flatsize, pkgsize, "
"cksum, manifestdigest, path AS repopath, '%s' AS dbname "
"FROM packages AS p";
if (match != MATCH_ALL && (pattern == NULL || pattern[0] == '\0'))
return (NULL);
utstring_new(sql);
comp = pkgdb_get_pattern_query(pattern, match);
if (comp && comp[0])
strlcat(basesql, comp, sizeof(basesql));
utstring_printf(sql, basesql, repo->name);
utstring_printf(sql, "%s", " ORDER BY name;");
pkg_debug(4, "Pkgdb: running '%s' query for %s", utstring_body(sql),
pattern == NULL ? "all": pattern);
ret = sqlite3_prepare_v2(sqlite, utstring_body(sql), utstring_len(sql), &stmt,
NULL);
if (ret != SQLITE_OK) {
ERROR_SQLITE(sqlite, utstring_body(sql));
utstring_free(sql);
return (NULL);
}
utstring_free(sql);
if (match != MATCH_ALL && match != MATCH_CONDITION)
sqlite3_bind_text(stmt, 1, pattern, -1, SQLITE_TRANSIENT);
return (pkg_repo_binary_it_new(repo, stmt, PKGDB_IT_FLAG_ONCE));
}
int main(int argc, char *argv[]) {
void *sp=NULL;
void *set=NULL;
int opt,rc=-1;
char *config_file;
set = kv_set_new();
utarray_new(output_keys, &ut_str_icd);
utarray_new(output_defaults, &ut_str_icd);
utarray_new(output_types,&ut_int_icd);
utstring_new(tmp);
while ( (opt = getopt(argc, argv, "v+d:b:s")) != -1) {
switch (opt) {
case 'v': verbose++; break;
case 's': push_mode++; break;
case 'd': spool=strdup(optarg); break;
case 'b': config_file=strdup(optarg); break;
default: usage(argv[0]); break;
}
}
if (optind < argc) pub_transport = argv[optind++];
if (!pub_transport) usage(argv[0]);
if (spool == NULL) usage(argv[0]);
if (parse_config(config_file) < 0) goto done;
if ( !(pub_context = zmq_init(1))) goto done;
if ( !(pub_socket = zmq_socket(pub_context, push_mode?ZMQ_PUSH:ZMQ_PUB))) goto done;
if (zmq_setsockopt(pub_socket, ZMQ_SNDHWM, &hwm, sizeof(hwm))) goto done;
if (zmq_bind(pub_socket, pub_transport) == -1) goto done;
sp = kv_spoolreader_new(spool);
if (!sp) goto done;
while (kv_spool_read(sp,set,1) > 0) { /* read til interrupted by signal */
zmq_msg_t part;
if (set_to_binary(set,&part) < 0) goto done;
rc = zmq_sendmsg(pub_socket, &part, 0);
zmq_msg_close(&part);
if (rc == -1) goto done;
}
rc = 0;
done:
if (rc) fprintf(stderr,"zmq: %s %s\n", pub_transport, zmq_strerror(errno));
if (pub_socket) zmq_close(pub_socket);
if (pub_context) zmq_term(pub_context);
if (sp) kv_spoolreader_free(sp);
kv_set_free(set);
utarray_free(output_keys);
utarray_free(output_defaults);
utarray_free(output_types);
utstring_free(tmp);
return 0;
}
int main()
{
UT_string *s;
utstring_new(s);
utstring_printf(s, "hello world!" );
printf("%s\n", utstring_body(s));
utstring_free(s);
return 0;
}
int main(int argc, char * argv[]) {
int opt,verbose=0;
long dirmax=0;
/* input spool */
char *dir=NULL,unit,*sz="10GB";
UT_string *s;
utstring_new(s);
while ( (opt = getopt(argc, argv, "v+s:")) != -1) {
switch (opt) {
default: usage(argv[0]); break;
case 'v': verbose++; break;
case 's':
sz = strdup(optarg);
switch (sscanf(sz, "%ld%c", &dirmax, &unit)) {
case 2: /* check unit */
switch (unit) {
case 't': case 'T': break;
case 'g': case 'G': break;
case 'm': case 'M': break;
case 'k': case 'K': break;
case '\r': case '\n': case ' ': case '\t': break;
default: usage(argv[0]); break;
}
case 1: /* just a number in bytes */ break;
default: usage(argv[0]); break;
}
break;
}
}
if (optind >= argc) usage(argv[0]);
if (!dirmax) usage(argv[0]);
kv_spool_options.dir_max = dirmax;
while (optind < argc) {
dir = argv[optind++];
utstring_clear(s);
utstring_printf(s,"%s/limits", dir);
char *p = utstring_body(s);
FILE *f = fopen(p, "w");
if (f == NULL) {
fprintf(stderr,"cannot open %s: %s\n", p, strerror(errno));
continue;
}
fprintf(f, "%s", sz);
fclose(f);
sp_attrition(dir);
}
done:
utstring_free(s);
return 0;
}
struct pkg_repo_it *
pkg_repo_binary_search(struct pkg_repo *repo, const char *pattern, match_t match,
pkgdb_field field, pkgdb_field sort)
{
sqlite3 *sqlite = PRIV_GET(repo);
sqlite3_stmt *stmt = NULL;
UT_string *sql = NULL;
int ret;
const char *multireposql = ""
"SELECT id, origin, name, version, comment, "
"prefix, desc, arch, maintainer, www, "
"licenselogic, flatsize, pkgsize, "
"cksum, path AS repopath, '%1$s' AS dbname, '%2$s' AS repourl "
"FROM packages ";
if (pattern == NULL || pattern[0] == '\0')
return (NULL);
utstring_new(sql);
utstring_printf(sql, multireposql, repo->name, repo->url);
/* close the UNIONs and build the search query */
utstring_printf(sql, "%s", "WHERE ");
pkg_repo_binary_build_search_query(sql, match, field, sort);
utstring_printf(sql, "%s", ";");
pkg_debug(4, "Pkgdb: running '%s'", utstring_body(sql));
ret = sqlite3_prepare_v2(sqlite, utstring_body(sql), -1, &stmt, NULL);
if (ret != SQLITE_OK) {
ERROR_SQLITE(sqlite, utstring_body(sql));
utstring_free(sql);
return (NULL);
}
utstring_free(sql);
sqlite3_bind_text(stmt, 1, pattern, -1, SQLITE_TRANSIENT);
return (pkg_repo_binary_it_new(repo, stmt, PKGDB_IT_FLAG_ONCE));
}
static PyObject* binaryplist_encode(PyObject *self, PyObject *args, PyObject *kwargs)
{
static char *kwlist[] = {"obj", "unique", "debug", "convert_nulls",
"max_recursion", "object_hook", "as_ascii", NULL};
PyObject *newobj = NULL;
PyObject *oinput = NULL;
PyObject *ounique = NULL;
PyObject *odebug = NULL;
PyObject *orecursion = NULL;
binaryplist_encoder encoder;
memset(&encoder, 0, sizeof(binaryplist_encoder));
encoder.convert_nulls = Py_False;
encoder.as_ascii = Py_False;
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|OOOOOO", kwlist, &oinput, &ounique,
&odebug, &(encoder.convert_nulls), &orecursion, &(encoder.object_hook),
&(encoder.as_ascii))) {
return NULL;
}
if (encoder.object_hook && !PyCallable_Check(encoder.object_hook)) {
PyErr_SetString(PLIST_Error, "object_hook is not callable");
return NULL;
}
encoder.ref_table = PyDict_New();
encoder.objects = PyList_New(0);
utstring_new(encoder.output);
if (!ounique || (ounique && PyObject_IsTrue(ounique))) {
/* default to True */
encoder.dounique = 1;
encoder.uniques = PyDict_New();
}
if (odebug && PyObject_IsTrue(odebug)) {
encoder.debug = 1;
}
if (orecursion && PyInt_Check(orecursion)) {
encoder.max_recursion = PyInt_AsLong(orecursion);
} else {
encoder.max_recursion = 1024*16;
}
if (encoder_encode_object(&encoder, oinput) == BINARYPLIST_OK
&& encoder_write(&encoder) == BINARYPLIST_OK) {
newobj = PyString_FromStringAndSize(utstring_body(encoder.output),
utstring_len(encoder.output));
}
Py_DECREF(encoder.ref_table);
Py_DECREF(encoder.objects);
Py_XDECREF(encoder.uniques);
utstring_free(encoder.output);
return newobj;
}
int
packing_append_tree(struct packing *pack, const char *treepath,
const char *newroot)
{
FTS *fts = NULL;
FTSENT *fts_e = NULL;
size_t treelen;
UT_string *sb;
char *paths[2] = { __DECONST(char *, treepath), NULL };
treelen = strlen(treepath);
fts = fts_open(paths, FTS_PHYSICAL | FTS_XDEV, NULL);
if (fts == NULL)
goto cleanup;
utstring_new(sb);
while ((fts_e = fts_read(fts)) != NULL) {
switch(fts_e->fts_info) {
case FTS_D:
case FTS_DEFAULT:
case FTS_F:
case FTS_SL:
case FTS_SLNONE:
/* Entries not within this tree are irrelevant. */
if (fts_e->fts_pathlen <= treelen)
break;
utstring_clear(sb);
/* Strip the prefix to obtain the target path */
if (newroot) /* Prepend a root if one is specified */
utstring_printf(sb, "%s", newroot);
/* +1 = skip trailing slash */
utstring_printf(sb, "%s", fts_e->fts_path + treelen + 1);
packing_append_file_attr(pack, fts_e->fts_name,
utstring_body(sb), NULL, NULL, 0, 0);
break;
case FTS_DC:
case FTS_DNR:
case FTS_ERR:
case FTS_NS:
/* XXX error cases, check fts_e->fts_errno and
* bubble up the call chain */
break;
default:
break;
}
}
utstring_free(sb);
cleanup:
fts_close(fts);
return EPKG_OK;
}
void
pkg_free(struct pkg *pkg)
{
if (pkg == NULL)
return;
free(pkg->name);
free(pkg->origin);
free(pkg->old_version);
free(pkg->maintainer);
free(pkg->www);
free(pkg->arch);
free(pkg->abi);
free(pkg->uid);
free(pkg->digest);
free(pkg->old_digest);
free(pkg->prefix);
free(pkg->comment);
free(pkg->desc);
free(pkg->sum);
free(pkg->repopath);
free(pkg->repourl);
free(pkg->reason);
free(pkg->dep_formula);
for (int i = 0; i < PKG_NUM_SCRIPTS; i++)
if (pkg->scripts[i])
utstring_free(pkg->scripts[i]);
pkg_list_free(pkg, PKG_DEPS);
pkg_list_free(pkg, PKG_RDEPS);
pkg_list_free(pkg, PKG_FILES);
pkg_list_free(pkg, PKG_DIRS);
pkg_list_free(pkg, PKG_OPTIONS);
pkg_list_free(pkg, PKG_USERS);
pkg_list_free(pkg, PKG_GROUPS);
pkg_list_free(pkg, PKG_SHLIBS_REQUIRED);
pkg_list_free(pkg, PKG_SHLIBS_PROVIDED);
pkg_list_free(pkg, PKG_PROVIDES);
pkg_list_free(pkg, PKG_REQUIRES);
pkg_list_free(pkg, PKG_CATEGORIES);
pkg_list_free(pkg, PKG_LICENSES);
LL_FREE(pkg->message, pkg_message_free);
LL_FREE(pkg->annotations, pkg_kv_free);
if (pkg->rootfd != -1)
close(pkg->rootfd);
free(pkg);
}
int64_t
pkg_repo_binary_stat(struct pkg_repo *repo, pkg_stats_t type)
{
sqlite3 *sqlite = PRIV_GET(repo);
sqlite3_stmt *stmt = NULL;
int64_t stats = 0;
UT_string *sql = NULL;
int ret;
utstring_new(sql);
switch(type) {
case PKG_STATS_LOCAL_COUNT:
goto out;
break;
case PKG_STATS_LOCAL_SIZE:
goto out;
break;
case PKG_STATS_REMOTE_UNIQUE:
utstring_printf(sql, "SELECT COUNT(id) FROM main.packages;");
break;
case PKG_STATS_REMOTE_COUNT:
utstring_printf(sql, "SELECT COUNT(id) FROM main.packages;");
break;
case PKG_STATS_REMOTE_SIZE:
utstring_printf(sql, "SELECT SUM(pkgsize) FROM main.packages;");
break;
case PKG_STATS_REMOTE_REPOS:
goto out;
break;
}
pkg_debug(4, "binary_repo: running '%s'", utstring_body(sql));
ret = sqlite3_prepare_v2(sqlite, utstring_body(sql), -1, &stmt, NULL);
if (ret != SQLITE_OK) {
ERROR_SQLITE(sqlite, utstring_body(sql));
goto out;
}
while (sqlite3_step(stmt) != SQLITE_DONE) {
stats = sqlite3_column_int64(stmt, 0);
}
out:
utstring_free(sql);
if (stmt != NULL)
sqlite3_finalize(stmt);
return (stats);
}
void cleanup_oh_prepared(OM_DEFINE_OBJECT(o_handler,oh)) {
OM_MUTEX_LOCK(o_prepared, oprep);
if (oh->connection_prepared == TRUE) {
switch (oh->service_protocol) {
case ORIENT_PROTO_BINARY:
freeaddrinfo(oh->result);
break;
case ORIENT_PROTO_HTTP:
break;
default:
break;
}
}
oh->connection_prepared = FALSE;
oh->service_protocol = -1;
utstring_free(oh->admin->username);
utstring_free(oh->admin->password);
OM_FREE(o_ac, oh->admin);
utstring_free(oh->user->username);
utstring_free(oh->user->password);
OM_FREE(o_ac, oh->user);
OM_MUTEX_UNLOCK(o_prepared, oprep);
}
/* return allocated UT_string */
static UT_string *JsonNode_getJSON_UT(JsonNode *node, String last)
{
asize_t i, nPairs, nChilds;
UT_string *buff;
utstring_new(buff);
if (buff == NULL) return NULL;
if (!isNullorEmpty(node->m_name)) {
utstring_printf(buff, "\"%s\":", node->m_name);
}
utstring_printf(buff, "%s\n", JSON_IS_OBJ(node) ? "{" : "[");
nPairs = JsonNode_getPairCount(node);
nChilds = JsonNode_getChildCount(node);
for (i=0; i < nPairs; i++ ) {
JsonPair *pair = JsonNode_getPair(node, i);
if (JSON_IS_ARRAY(node)) {
utstring_printf(buff, "\"%s\"", pair->key);
} else {
utstring_printf(buff, "\"%s\":\"%s\"", pair->key, pair->value);
}
utstring_printf(buff, "%s\n", (i < nPairs -1 || nChilds > 0) ? "," : "");
}
for (i = 0; i < nChilds; i++) {
JsonNode* child = JsonNode_getChild(node, i);
UT_string *childJSON = JsonNode_getJSON_UT(child, i == nChilds - 1 ? "\n" : ",\n");
if (childJSON != NULL) {
utstring_concat(buff, childJSON);
utstring_free(childJSON);
}
}
utstring_printf(buff, "%s%s", JSON_IS_OBJ(node) ? "}" : "]", last);
return buff;
}
int get_files(UT_array *files, UT_array *stats) {
struct dirent *dent;
char *name, *path;
file_stat_t fsb;
int rc=-1,i=0;
DIR *d;
UT_string *s;
utstring_new(s);
utarray_clear(files);
utarray_clear(stats);
if ( (d = opendir(cf.dir)) == NULL) {
syslog(LOG_ERR,"failed to opendir [%s]: %s\n", cf.dir, strerror(errno));
goto done;
}
while ( (dent = readdir(d)) != NULL) {
if (dent->d_type != DT_REG) continue;
if (dent->d_name[0] == '.') continue; /* skip dot files */
// ok, fully qualify it and push it
utstring_clear(s);
utstring_printf(s, "%s/%s", cf.dir, dent->d_name);
path = utstring_body(s);
if (stat(path,&fsb.sb) == -1) {
syslog(LOG_ERR,"can't stat %s: %s", path, strerror(errno));
continue;
}
fsb.file_idx = i++;
utarray_push_back(files, &path);
utarray_push_back(stats, &fsb);
}
rc = 0; // success
done:
utstring_free(s);
if (d) closedir(d);
return rc;
}
long readSegmentFromFileSystem(
const char* nid,unsigned long long csn,size_t leftByte, size_t segmentSize,void* buffer)
{
size_t bytesRead=0; size_t fileNameSize;
size_t bytesToRead;
FILE *fp; //file pointer
//variables to update cache_table
UT_string* utnid;
UT_string* filename; utstring_new(filename);
calculate_filename( (char*)nid,csn,filename);
if(CONET_DEBUG)
fprintf(stderr,"[cacheEngine.c:%d]Opening file \"%s\" \n",__LINE__, utstring_body(filename) );
fp = fopen( utstring_body(filename), "rb");
if (fp == NULL)
{
fprintf(stderr, "[cacheEngine.c:%d]Error: Failed to open the file %s; maybe the chunk does not exist\n\n",
__LINE__, filename);
if (CONET_SEVERE_DEBUG) exit(-651);
return ( (size_t) - 1);
}
//!!!Questo fa andare in buffer overflow, il primo parametro dovrebbe essere
//buffer senza +leftByte
//Go to the leftByte location
int fseek_return = fseek(fp,leftByte,SEEK_SET);
if(fseek_return!=0)
{
fprintf(stderr,"[cacheEngine.c:%d]Error reading file %s. fseek returned with %d (ferror() tests the error indicator for the stream pointed to by stream, returning nonzero if it is set.)\n",
__LINE__,utstring_body(filename),fseek_return );
//if(CONET_SEVERE_DEBUG) exit(-652);
return ( (size_t) -0);
}
if(feof(fp))
{
fprintf(stderr,"[cacheEngine.c:%d]Error reading file %s. feof(fp)=%d\n",
__LINE__,utstring_body(filename), feof(fp));
//if(CONET_SEVERE_DEBUG) exit(-644);
return ( (size_t) -0);
}
clearerr(fp);
bytesRead = fread(buffer, 1, segmentSize, fp);
if (bytesRead <= 0)
{
fprintf(stderr,"%s\n",utstring_body(filename) );
fprintf(
stderr,
"[cacheEngine.c:%d] Error: in reading file %s. fread returned with %u. ferror=%d, feof(fp)=%d (ferror() tests the error indicator for the stream pointed to by stream, returning nonzero if it is set.)\n\n",
__LINE__,utstring_body(filename), bytesRead,ferror(fp), feof(fp)
);
if (CONET_SEVERE_DEBUG) exit(-652);
return bytesRead;
}
fclose(fp);
if(CONET_DEBUG)
fprintf(stderr,"[cacheEngine.c:%d]Successfully read %d bytes\n",
__LINE__,(int) bytesRead);
utstring_free(filename);
return bytesRead;
}
//size_t handleChunk(char* nid,char* csn,long tag,size_t left_edge, size_t segment_size,void* buffer)
size_t handleChunk(char* nid,unsigned long long csn,long tag,unsigned long long chunk_size,void* buffer)
{
size_t writtenBytes,fileNameSize;
char* fileName;
char csnToDelete[64];
FILE *fp;
UT_string* folderName;
utstring_new(folderName);
UT_string* tempHexCsn;
utstring_new(tempHexCsn);
UT_string* filename; utstring_new(filename);
//variables for cache table
CacheEntry_t *entry, *tmp_entry;
CacheEntry_t *entryToDelete=NULL;
UT_string *utnid, *filenameToDelete;
// long csnlong;
//folderName=NULL;
if(CONET_DEBUG) fprintf(stderr,"\n[cacheEngine.c:%d]New segment nid=%s, csn=%llu received.\n",
__LINE__,nid,csn);
calculate_filename(nid,csn,filename);
utstring_printf(folderName,CONTAINING_FOLDER);
utstring_printf(folderName,nid);
mkdir(utstring_body(folderName));
fp = fopen(utstring_body(filename) , "w+b");
if (fp == NULL) {
fprintf(stderr, "Error: Failed to open the file %s for writing;\n\n", fileName);
if (CONET_SEVERE_DEBUG) exit(-863);
return ( (size_t) - 1);
}
writtenBytes = fwrite(buffer, 1,chunk_size, fp);
fclose(fp);
if (writtenBytes!=chunk_size) {
fprintf(stderr, "Error in writing %s: writtenBytes=%d, while chunk_size=%d\n",
fileName,writtenBytes,chunk_size);
if (CONET_SEVERE_DEBUG) exit(-8763);
return ( (size_t) - 1);
}
utstring_new(utnid); utstring_printf(utnid,nid);
//utstring_printf(tempHexCsn,"0x");utstring_printf(tempHexCsn,csn);
ADD_TO_CACHE_TABLE(cache_table,entry,tmp_entry,tag,utnid,csn,chunk_size,0,entryToDelete);
//creates json message with nid and of type "stored", and store it to msg_to_controller_buffer;
fill_message(tag, "stored", nid, csn, msg_to_controller_buffer);
//BUFFERLEN is defined in c_json.h
int bytesSentToController=send(socket_to_controller, msg_to_controller_buffer, BUFFERLEN, 0);
if(CONET_DEBUG )
fprintf(stderr,"[cacheEngine.c:%d]Message sent to controller = %s\n",__LINE__,msg_to_controller_buffer);
if (bytesSentToController <=strlen(msg_to_controller_buffer) )
{
fprintf(stderr,"[cacheEngine.c: %d] bytes sent to controller=%d\n",__LINE__, bytesSentToController);
if (CONET_SEVERE_DEBUG) exit(-982);
}
if (entryToDelete!=NULL)
{
sprintf(csnToDelete,"%x",(unsigned int) entryToDelete->csn);
CALCULATE_UTSTRING_FILE_NAME(entryToDelete->nid,csnToDelete,filenameToDelete);
if (remove(utstring_body(filenameToDelete) )==0)
fprintf(stderr,"File %s deleted.\n",utstring_body(filenameToDelete));
else
fprintf(stderr,"Error deleting file %s,\n",utstring_body(filenameToDelete));
//creates json message with nid and of type "stored", and store it to msg_to_controller_buffer;
fill_message(entryToDelete->tag, "deleted", entryToDelete->nid, entryToDelete->csn, msg_to_controller_buffer);
//BUFFERLEN is defined in c_json.h
if (send(socket_to_controller, msg_to_controller_buffer, BUFFERLEN, 0) !=
strlen(msg_to_controller_buffer)
)
fprintf(stderr,"send() sent a different number of bytes than expected\n");
}
fprintf(stderr,"[cacheEngine.c:%d]New chunk added: nid=%s, csn=%llu, tag=%ld\n",__LINE__,nid,csn,tag);
// utstring_free(utnid); utstring_free(filenameToDelete);
utstring_free(filename);
return writtenBytes;
}
int main(void)
{
UT_array * dictionary;
UT_array * specialChar;
UT_array * WholeString = NULL;
CharProfile NewCharProfile;
CharProbability NewCharProbability;
/*utarray_new(dictionary, &ut_str_icd);
utarray_new(specialChar, &ut_str_icd);*/
dictionary = postProcessingInitializeDictionary();
specialChar = postProcessingInitializeSpecialChar();
/*initialize string profile*/
UT_icd StringProfile_icd = {sizeof(CharProfile), NULL, NULL, CharProfile_free};
utarray_new(WholeString, &StringProfile_icd);
/*first char spot*/
UT_icd CharProbability_icd = {sizeof(CharProbability), NULL, NULL, NULL};
utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
/*first possibility*/
NewCharProbability.Char = 'f';
NewCharProbability.Probability = 25;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*second probability*/
NewCharProbability.Char = 't';
NewCharProbability.Probability = 75;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*push to the whole string*/
utarray_push_back(WholeString, &NewCharProfile);
/*second char spot*/
utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
/*first possibility*/
NewCharProbability.Char = 'o';
NewCharProbability.Probability = 35;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*second probability*/
NewCharProbability.Char = '0';
NewCharProbability.Probability = 85;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*push to the whole string*/
utarray_push_back(WholeString, &NewCharProfile);
/*third char spot*/
utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
/*first possibility*/
NewCharProbability.Char = 't';
NewCharProbability.Probability = 50;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*second probability*/
NewCharProbability.Char = 'r';
NewCharProbability.Probability = 100;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*push to the whole string*/
utarray_push_back(WholeString, &NewCharProfile);
/*third char spot*/
utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
/*first possibility*/
NewCharProbability.Char = '(';
NewCharProbability.Probability = 50;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*second probability*/
NewCharProbability.Char = '=';
NewCharProbability.Probability = 100;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*push to the whole string*/
utarray_push_back(WholeString, &NewCharProfile);
/*fifth char spot*/
utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
/*first possibility*/
NewCharProbability.Char = 'm';
NewCharProbability.Probability = 50;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*second probability*/
NewCharProbability.Char = 'n';
NewCharProbability.Probability = 100;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*push to the whole string*/
utarray_push_back(WholeString, &NewCharProfile);
/*sixth char spot*/
utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
/*first possibility*/
NewCharProbability.Char = 'y';
NewCharProbability.Probability = 50;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*second probability*/
NewCharProbability.Char = 't';
NewCharProbability.Probability = 100;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*push to the whole string*/
utarray_push_back(WholeString, &NewCharProfile);
/*seventh char spot*/
utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
/*first possibility*/
NewCharProbability.Char = ' ';
NewCharProbability.Probability = 50;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*second probability*/
NewCharProbability.Char = 'u';
NewCharProbability.Probability = 100;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*push to the whole string*/
utarray_push_back(WholeString, &NewCharProfile);
/*eigth char spot*/
utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
/*first possibility*/
NewCharProbability.Char = 'h';
NewCharProbability.Probability = 50;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*second probability*/
NewCharProbability.Char = 'w';
NewCharProbability.Probability = 100;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*push to the whole string*/
utarray_push_back(WholeString, &NewCharProfile);
/*ninth char spot*/
utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
/*first possibility*/
NewCharProbability.Char = ')';
NewCharProbability.Probability = 50;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*second probability*/
NewCharProbability.Char = '}';
NewCharProbability.Probability = 100;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*push to the whole string*/
utarray_push_back(WholeString, &NewCharProfile);
/*tenth char spot*/
utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
/*first possibility*/
NewCharProbability.Char = '\n';
NewCharProbability.Probability = 50;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*second probability*/
NewCharProbability.Char = 'u';
NewCharProbability.Probability = 100;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*push to the whole string*/
utarray_push_back(WholeString, &NewCharProfile);
/*sixth char spot*/
utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
/*first possibility*/
NewCharProbability.Char = '{';
NewCharProbability.Probability = 50;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*second probability*/
NewCharProbability.Char = '[';
NewCharProbability.Probability = 100;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*push to the whole string*/
utarray_push_back(WholeString, &NewCharProfile);
/*sixth char spot*/
utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
/*first possibility*/
NewCharProbability.Char = 'd';
NewCharProbability.Probability = 50;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*second probability*/
NewCharProbability.Char = 'a';
NewCharProbability.Probability = 100;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*push to the whole string*/
utarray_push_back(WholeString, &NewCharProfile);
/*sixth char spot*/
utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
/*first possibility*/
NewCharProbability.Char = 'u';
NewCharProbability.Probability = 50;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*second probability*/
NewCharProbability.Char = 'd';
NewCharProbability.Probability = 100;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*push to the whole string*/
utarray_push_back(WholeString, &NewCharProfile);
/*sixth char spot*/
utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
/*first possibility*/
NewCharProbability.Char = 't';
NewCharProbability.Probability = 50;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*second probability*/
NewCharProbability.Char = 'y';
NewCharProbability.Probability = 100;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*push to the whole string*/
utarray_push_back(WholeString, &NewCharProfile);
/*sixth char spot*/
utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
/*first possibility*/
NewCharProbability.Char = 'o';
NewCharProbability.Probability = 50;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*second probability*/
NewCharProbability.Char = 't';
NewCharProbability.Probability = 100;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*push to the whole string*/
utarray_push_back(WholeString, &NewCharProfile);
/*sixth char spot*/
utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
/*first possibility*/
NewCharProbability.Char = '}';
NewCharProbability.Probability = 50;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*second probability*/
NewCharProbability.Char = ']';
NewCharProbability.Probability = 100;
utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
/*push to the whole string*/
utarray_push_back(WholeString, &NewCharProfile);
UT_string * advance_string = postProcessingAdvance(WholeString,dictionary,specialChar);
UT_string * normal_string = postProcessing(WholeString);
printf("Result from highest probability catch is:\n%s\nResult from keyword match is:\n%s\n", utstring_body(normal_string), utstring_body(advance_string));
utstring_free(advance_string);
utstring_free(normal_string);
postProcessingCleanUP(dictionary, specialChar);
return 0;
}
int main() {
UT_string *s,*t;
char V_TestStr[] = "There are two needle\0s in this \0haystack with needle\0s.";
char V_NeedleStr[] = "needle\0s";
long *V_KMP_Table;
long V_FindPos;
size_t V_StartPos;
size_t V_FindCnt;
utstring_new(s);
utstring_new(t);
utstring_bincpy(s, V_TestStr, sizeof(V_TestStr)-1);
printf("\"%s\" len=%u\n", utstring_body(s), utstring_len(s));
utstring_bincpy(t, V_NeedleStr, sizeof(V_NeedleStr)-1);
printf("\"%s\" len=%u\n", utstring_body(t), utstring_len(t));
V_KMP_Table = (long *)malloc(sizeof(long) * (utstring_len(t) + 1));
if (V_KMP_Table != NULL)
{
_utstring_BuildTableR(utstring_body(t), utstring_len(t), V_KMP_Table);
V_FindCnt = 0;
V_FindPos = 0;
V_StartPos = utstring_len(s) - 1;
do
{
V_FindPos = _utstring_findR(utstring_body(s),
V_StartPos + 1,
utstring_body(t),
utstring_len(t),
V_KMP_Table);
if (V_FindPos >= 0)
{
V_FindCnt++;
V_StartPos = V_FindPos - 1;
}
printf("utstring_find()=%ld\n", V_FindPos);
} while (V_FindPos >= 0);
printf("FindCnt=%u\n", V_FindCnt);
free(V_KMP_Table);
}
else
{
printf("malloc() failed...\n");
}
utstring_free(t);
utstring_clear(s);
utstring_printf(s,"ABC ABCDAB ABCDABCDABDE");
int o;
o=utstring_find( s, -9, "ABC", 3 ) ; printf("expect 15 %d\n",o);
o=utstring_find( s, 3, "ABC", 3 ) ; printf("expect 4 %d\n",o);
o=utstring_find( s, 16, "ABC", 3 ) ; printf("expect -1 %d\n",o);
o=utstring_findR( s, -9, "ABC", 3 ) ; printf("expect 11 %d\n",o);
o=utstring_findR( s, 12, "ABC", 3 ) ; printf("expect 4 %d\n",o);
o=utstring_findR( s, 13, "ABC", 3 ) ; printf("expect 11 %d\n",o);
o=utstring_findR( s, 2, "ABC", 3 ) ; printf("expect 0 %d\n",o);
utstring_free(s);
return 0;
}
static luarest_status parse_apps(application** apps, char* directory_path)
{
#ifdef WIN32
WIN32_FIND_DATA ffd;
WIN32_FIND_DATA fap;
size_t length_of_arg;
TCHAR szDir[MAX_PATH];
TCHAR appFile[MAX_PATH];
HANDLE hFind = INVALID_HANDLE_VALUE;
HANDLE hApp = INVALID_HANDLE_VALUE;
luarest_status ret;
// Check that the input path plus 2 is not longer than MAX_PATH.
StringCchLength(directory_path, MAX_PATH, &length_of_arg);
if (length_of_arg > (MAX_PATH - 2)) {
_tprintf(TEXT("\nDirectory path is too long.\n"));
return(LUAREST_ERROR);
}
// Prepare string for use with FindFile functions. First, copy the
// string to a buffer, then append '\*' to the directory name.
StringCchCopy(szDir, MAX_PATH, directory_path);
StringCchCat(szDir, MAX_PATH, TEXT("\\*"));
// Find the first file in the directory.
hFind = FindFirstFile(szDir, &ffd);
if (INVALID_HANDLE_VALUE == hFind) {
return(LUAREST_ERROR);
}
// List all the files in the directory with some info about them.
do {
if (ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
StringCchCopy(appFile, MAX_PATH, directory_path);
StringCchCat(appFile, MAX_PATH, TEXT("\\"));
StringCchCat(appFile, MAX_PATH, ffd.cFileName);
StringCchCat(appFile, MAX_PATH, TEXT("\\"));
StringCchCat(appFile, MAX_PATH, APP_ENTRY_POINT);
hApp = FindFirstFile(appFile, &fap);
if (hApp != INVALID_HANDLE_VALUE) {
UT_string* app;
utstring_new(app);
utstring_printf(app, appFile);
/* verify application */
ret = verify_application(apps, ffd.cFileName, app);
utstring_free(app);
if (ret != LUAREST_SUCCESS) {
printf("Application %s couldn't be load due to errors!\n", ffd.cFileName);
}
}
FindClose(hApp);
}
}
while (FindNextFile(hFind, &ffd) != 0);
FindClose(hFind);
#else
DIR *dpdf;
struct dirent *epdf;
struct stat st;
int is_dir = 0;
dpdf = opendir("./");
if (dpdf != NULL){
while ( (epdf = readdir(dpdf)) != NULL) {
if (stat(epdf->d_name, &st) == -1) {
continue;
}
is_dir = (st.st_mode & S_IFDIR) != 0;
if (is_dir) {
continue;
}
/* re-do for unix */
}
}
closedir(dpdf);
#endif
return(LUAREST_SUCCESS);
}
int main(int argc, char *argv[]) {
#ifndef _OPENMP
fprintf(stderr, "\nERROR: Program built with compiler lacking OpenMP support.\n");
fprintf(stderr, "See SEAStAR README file for information about suitable compilers.\n");
exit(EXIT_FAILURE);
#endif
///////////////////////////
// Variable declarations
///////////////////////////
// Input filenames
UT_string *in_read1_fq_fn, *in_read2_fq_fn, *in_single1_fq_fn, *in_single2_fq_fn;
utstring_new(in_read1_fq_fn);
utstring_new(in_read2_fq_fn);
utstring_new(in_single1_fq_fn);
utstring_new(in_single2_fq_fn);
// Output filenames
UT_string *out_read1_fn, *out_read2_fn, *out_single1_fn, *out_single2_fn, *out_mates_fn, *out_filetype;
utstring_new(out_filetype);
utstring_new(out_read1_fn);
utstring_new(out_read2_fn);
utstring_new(out_single1_fn);
utstring_new(out_single2_fn);
utstring_new(out_mates_fn);
// Read name prefix
UT_string *out_read_prefix;
utstring_new(out_read_prefix);
// Flags
int singles_flag = 0; // 1 when two output singles files being written
int num_input_singles_files = 0;
// Read counters
unsigned long int mp_org = 0, R1_org = 0, R2_org = 0, singlet1_org = 0, singlet2_org = 0;
unsigned long int mp_cnt = 0, R1_cnt = 0, R2_cnt = 0, singlet1_cnt = 0, singlet2_cnt = 0, s1_cnt = 0, s2_cnt = 0;
unsigned long int comp_r1 = 0, comp_r2 = 0, comp_s1 = 0, comp_s2 = 0;
unsigned long int read1_singlet_cnt = 0, read2_singlet_cnt = 0;
////////////////////////////////////////////////////////////////////////
// All done with variable declarations!!
///////////////////////////////////
// Command line argtable settings
///////////////////////////////////
struct arg_lit *gzip = arg_lit0("z", "gzip", "Output converted files in gzip compressed format. [NULL]");
struct arg_lit *inv_singles = arg_lit0("v", "invert_singles", "Causes singles output to be the inverse of the input. 2->1 or 1->2 [NULL]");
struct arg_lit *num_singles = arg_lit0("s", "singles", "Write two singlet files, one for each mate-paired input file. [NULL]");
struct arg_rem *sing_rem = arg_rem(NULL, "Note! -v is only valid when there are input singlet reads. -s is only valid when there are NO input singlet reads.");
struct arg_str *pre_read_id = arg_str0(NULL, "prefix", "<string>", "Prefix to add to read identifiers. [out_prefix]");
struct arg_lit *no_pre = arg_lit0(NULL, "no_prefix", "Do not change the read names in any way. [NULL]");
struct arg_lit *pre_read_len = arg_lit0(NULL, "add_len", "Add the final trimmed length value to the read id prefix. [length not added]");
struct arg_dbl *prob = arg_dbl0("p","correct_prob","<d>","Probability that output reads are correct. 0.0 disables quality trimming. [0.5]");
struct arg_int *fixed_len = arg_int0("f","fixed_len","<u>","Trim all reads to a fixed length, still filtering on quality [no fixed length]");
struct arg_int *len = arg_int0("l","min_read_len","<u>","Minimum length of a singlet or longest-mate in nucleotides [24]");
struct arg_int *mate_len = arg_int0("m","min_mate_len","<u>","Minimum length of the shortest mate in nucleotides [min_read_len]");
struct arg_dbl *entropy = arg_dbl0("e","entropy_filter","<d>","Remove reads with per position information below given value (in bits per dinucleotide) [No filter]");
struct arg_lit *entropy_strict = arg_lit0(NULL, "entropy_strict", "Reject reads for low entropy overall, not just the retained part after trimming [NULL]");
struct arg_lit *mates = arg_lit0(NULL, "mates_file", "Produce a Velvet compatible interleaved paired read output file (e.g. <out_prefix>_mates.fastq). [NULL]");
struct arg_lit *no_rev = arg_lit0(NULL, "no_rev", "By default, the second read in each pair is reversed for colorspace --mate-file output. --no_rev disables reversing. [rev]");
struct arg_lit *only_mates = arg_lit0(NULL, "only_mates", "Supress writing .read1 and .read2 outputs. Requires --mates_file. [NULL]");
struct arg_lit *fasta = arg_lit0(NULL, "fasta", "Write FASTA format files instead of FASTQ for all outputs (e.g. <out_prefix>.<read_type>.fasta). [FASTQ]");
struct arg_file *input = arg_file1(NULL, NULL, "<in_prefix>", "Input file prefix: (e.g. <in_prefix>_single.fastq [<in_prefix>_read1.fastq <in_prefix>_read2.fastq]) ");
struct arg_file *output = arg_file1(NULL, NULL, "<out_prefix>", "Output file prefix: (e.g. <out_prefix>_single.fastq [<out_prefix>_read1.fastq <out_prefix>_read2.fastq]) ");
struct arg_lit *version = arg_lit0(NULL,"version","Print the build version and exit.");
struct arg_lit *h = arg_lit0("h", "help", "Request help.");
struct arg_end *end = arg_end(20);
void *argtable[] = {h,version,gzip,inv_singles,num_singles,sing_rem,prob,len,mate_len,fixed_len,pre_read_id,pre_read_len,no_pre,entropy,entropy_strict,mates,no_rev,only_mates,fasta,input,output,end};
int arg_errors = 0;
////////////////////////////////////////////////////////////////////////
// Handle command line processing (via argtable2 library)
////////////////////////////////////////////////////////////////////////
arg_errors = arg_parse(argc, argv, argtable);
if (version->count) {
fprintf(stderr, "%s version: %s\n", argv[0], SS_BUILD_VERSION);
exit(EXIT_SUCCESS);
}
if (h->count) {
fprintf(stderr,"\ntrim_fastq is a utility for performing quality and information-based\n");
fprintf(stderr,"trimming on paired or unpaired, nucleotide or SOLiD colorspace reads. \n\n");
arg_print_syntaxv(stderr, argtable, "\n\n");
arg_print_glossary(stderr, argtable, "%-25s %s\n");
fprintf(stderr, "\nInput and output \"prefixes\" are the part of the filename before:\n");
fprintf(stderr, "_single.fastq [_read1.fastq _read2.fastq] A singlets (single) file\n");
fprintf(stderr, "is required. Mate-paired read files are automatically used if present.\n");
fprintf(stderr, "Multiple output files only produced for mate-paired inputs.\n");
fprintf(stderr, "\nNote! Input and output files may be gzipped, and outputs can be written\n");
fprintf(stderr, "as either FASTQ or FASTA format files.\n");
exit(EXIT_FAILURE);
}
if (arg_errors) {
arg_print_errors(stderr, end, "trimfastq");
arg_print_syntaxv(stderr, argtable, "\n");
exit(EXIT_FAILURE);
}
// Validate entropy
if (entropy->count) {
entropy_cutoff = entropy->dval[0];
if ((entropy_cutoff < 0.0) || (entropy_cutoff > 4.0)) {
fprintf(stderr, "entropy_filter must be [0.0 - 4.0] \n");
exit(EXIT_FAILURE);
}
strict_ent = entropy_strict->count;
} else {
if (entropy_strict->count) {
fprintf(stderr, "Error: --entropy_strict requires --entropy_filter.\n");
exit(EXIT_FAILURE);
}
entropy_cutoff = -1.0;
}
// Validate error_prob
if (prob->count) {
err_prob = prob->dval[0];
if ((err_prob < 0.0) || (err_prob > 1.0)) {
fprintf(stderr, "--correct_prob (-p) must be 0.0 - 1.0 inclusive\n");
exit(EXIT_FAILURE);
}
} else {
err_prob = 0.5;
}
// Validate min read len
if (len->count) {
min_len = len->ival[0];
if (min_len <= 0) {
fprintf(stderr, "min_read_len must be > 0\n");
exit(EXIT_FAILURE);
}
} else {
min_len = 24;
}
// Validate min mate len
if (mate_len->count) {
min_mate_len = mate_len->ival[0];
if (min_mate_len <= 0) {
fprintf(stderr, "min_mate_len must be > 0\n");
exit(EXIT_FAILURE);
}
if (min_mate_len > min_len) {
fprintf(stderr, "min_mate_len must be <= min_len\n");
exit(EXIT_FAILURE);
}
} else {
min_mate_len = min_len;
}
if (fixed_len->count) {
fix_len = min_mate_len = min_len = fixed_len->ival[0];
if ((mate_len->count) || (len->count)) {
fprintf(stderr, "fixed_len cannot be used with min_read_len or min_mate_len\n");
exit(EXIT_FAILURE);
}
if (fix_len <= 0) {
fprintf(stderr, "fixed_len must be > 0\n");
exit(EXIT_FAILURE);
}
} else {
fix_len = 0;
}
if (pre_read_id->count) {
if (no_pre->count) {
fprintf(stderr, "Error: Both --prefix and --no_prefix were specified.\n");
exit(EXIT_FAILURE);
}
if (! strlen(pre_read_id->sval[0])) {
fprintf(stderr, "Read ID prefix may not be zero length.\n");
exit(EXIT_FAILURE);
}
if (strchr(pre_read_id->sval[0], ':') || strchr(pre_read_id->sval[0], '|') || strchr(pre_read_id->sval[0], '+') || strchr(pre_read_id->sval[0], '/')) {
fprintf(stderr, "Read ID prefix '%s' may not contain the characters ':', '|', '+' or '/'.\n", pre_read_id->sval[0]);
exit(EXIT_FAILURE);
}
// Build default read ID prefix
ss_strcat_utstring(out_read_prefix, pre_read_id->sval[0]);
} else {
if (!no_pre->count) {
if (strchr(output->filename[0], ':') || strchr(output->filename[0], '|') || strchr(output->filename[0], '+') || strchr(output->filename[0], '/')) {
fprintf(stderr, "Read ID prefix '%s' (from output prefix) may not contain the characters ':', '|', '+' or '/'.\n", output->filename[0]);
fprintf(stderr, "Hint: Use the --prefix parameter if the output file prefix contains path information.\n");
exit(EXIT_FAILURE);
}
// Build default read ID prefix
ss_strcat_utstring(out_read_prefix, output->filename[0]);
}
}
if ((only_mates->count) && (!mates->count)) {
fprintf(stderr, "--only_mates requires --mates.\n");
exit(EXIT_FAILURE);
}
if ((no_rev->count) && (!mates->count)) {
fprintf(stderr, "--no_rev requires --mates.\n");
exit(EXIT_FAILURE);
}
// Check for null string prefixes
if (!(strlen(input->filename[0]) && strlen(output->filename[0]))) {
fprintf(stderr, "Error: NULL prefix strings are not permitted.\n");
exit(EXIT_FAILURE);
}
// Construct input filenames
utstring_printf(in_read1_fq_fn, "%s.read1.fastq", input->filename[0]);
utstring_printf(in_read2_fq_fn, "%s.read2.fastq", input->filename[0]);
utstring_printf(in_single1_fq_fn, "%s.single.fastq", input->filename[0]);
FILE *in_read_file = NULL;
num_input_singles_files = 1;
// Try to open a singlet fastq file
// Check singlet output options -s and -v
// Set input singlet names to
// - *.single.fastq or
// - *.single1.fastq and *.single2.fastq
if (!(in_read_file = ss_get_gzFile(utstring_body(in_single1_fq_fn), "r"))) {
utstring_clear(in_single1_fq_fn);
utstring_printf(in_single1_fq_fn, "%s.single1.fastq", input->filename[0]);
utstring_printf(in_single2_fq_fn, "%s.single2.fastq", input->filename[0]);
num_input_singles_files = 2;
if ((in_read_file = ss_get_gzFile(utstring_body(in_single1_fq_fn), "r")) || (in_read_file = ss_get_gzFile(utstring_body(in_single2_fq_fn), "r"))) {
singles_flag = 1; // Two singlet outputs
} else {
singles_flag = num_singles->count; // Number of singlet outputs set by -s parm
if (inv_singles->count) {
fprintf(stderr, "Error: Invalid option -v, No input singlet file(s) found. Use -s to select multiple output singlet files.\n");
exit(EXIT_FAILURE);
}
}
}
if (in_read_file) {
gzclose(in_read_file);
if (num_singles->count) {
fprintf(stderr, "Error: Invalid option -s, Input singlet file(s) found, use -v to change the number of output singlet files.\n");
exit(EXIT_FAILURE);
}
}
// singles->count inverts the current singles file input scheme
singles_flag = (singles_flag ^ inv_singles->count);
// Check if input fastq is colorspace
// If some files are colorspace and some are basespace, throw an error
int fcount = 0;
int cscount = 0;
fcount += ss_is_fastq(utstring_body(in_read1_fq_fn));
fcount += ss_is_fastq(utstring_body(in_read2_fq_fn));
fcount += ss_is_fastq(utstring_body(in_single1_fq_fn));
fcount += ss_is_fastq(utstring_body(in_single2_fq_fn));
cscount += (ss_is_fastq(utstring_body(in_read1_fq_fn)) && ss_is_colorspace_fastq(utstring_body(in_read1_fq_fn)));
cscount += (ss_is_fastq(utstring_body(in_read2_fq_fn)) && ss_is_colorspace_fastq(utstring_body(in_read2_fq_fn)));
cscount += (ss_is_fastq(utstring_body(in_single1_fq_fn)) && ss_is_colorspace_fastq(utstring_body(in_single1_fq_fn)));
cscount += (ss_is_fastq(utstring_body(in_single2_fq_fn)) && ss_is_colorspace_fastq(utstring_body(in_single2_fq_fn)));
if (cscount && (cscount != fcount)) {
printf("Error: Mixed colorspace and basespace FASTQ files detected\n");
exit(EXIT_FAILURE);
}
colorspace_flag = cscount ? 1 : 0;
// Output filenames
if (fasta->count) {
ss_strcat_utstring(out_filetype, "fasta");
read_count_divisor = 2;
} else {
ss_strcat_utstring(out_filetype, "fastq");
read_count_divisor = 4;
}
if (!only_mates->count) {
utstring_printf(out_read1_fn, "%s.read1.%s", output->filename[0], utstring_body(out_filetype));
utstring_printf(out_read2_fn, "%s.read2.%s", output->filename[0], utstring_body(out_filetype));
}
if (singles_flag == 1) {
utstring_printf(out_single1_fn, "%s.single1.%s", output->filename[0], utstring_body(out_filetype));
utstring_printf(out_single2_fn, "%s.single2.%s", output->filename[0], utstring_body(out_filetype));
} else {
utstring_printf(out_single1_fn, "%s.single.%s", output->filename[0], utstring_body(out_filetype));
}
if (mates->count) {
utstring_printf(out_mates_fn, "%s.mates.%s", output->filename[0], utstring_body(out_filetype));
}
////////////////////////////////////////////////////////////////////////////////////////////////
// Begin processing!
#ifdef _OPENMP
omp_set_num_threads(10);
#endif
// This is the value of a non-valid pipe descriptor
#define NO_PIPE 0
int r1_pipe[2];
int r2_pipe[2];
int s1_pipe[2];
int s2_pipe[2];
pipe(r1_pipe);
pipe(r2_pipe);
pipe(s1_pipe);
pipe(s2_pipe);
int r1_out_pipe[2];
int r2_out_pipe[2];
int mates_out_pipe[2];
int s1_out_pipe[2];
int s2_out_pipe[2];
pipe(r1_out_pipe);
pipe(r2_out_pipe);
pipe(mates_out_pipe);
pipe(s1_out_pipe);
pipe(s2_out_pipe);
#pragma omp parallel sections default(shared)
{
#pragma omp section
{ // Read1 reader
fq_stream_trimmer(in_read1_fq_fn, r1_pipe[1], out_read_prefix, no_pre->count, pre_read_len->count, &comp_r1, &R1_org, '\0', fasta->count);
}
#pragma omp section
{ // Read1 writer
R1_cnt = ss_stream_writer(out_read1_fn, r1_out_pipe[0], gzip->count) / read_count_divisor;
}
#pragma omp section
{ // Read2 reader
fq_stream_trimmer(in_read2_fq_fn, r2_pipe[1], out_read_prefix, no_pre->count, pre_read_len->count, &comp_r2, &R2_org, '\0', fasta->count);
}
#pragma omp section
{ // Read2 writer
R2_cnt = ss_stream_writer(out_read2_fn, r2_out_pipe[0], gzip->count) / read_count_divisor;
}
#pragma omp section
{ // Single1 reader
// When there is only one input singles file, but two output singles files, then supply which mate to use for this stream in the split parameter
if ((singles_flag) && (num_input_singles_files == 1)) {
singlet1_cnt = fq_stream_trimmer(in_single1_fq_fn, s1_pipe[1], out_read_prefix, no_pre->count, pre_read_len->count, &comp_s1, &singlet1_org, '1', fasta->count);
} else {
singlet1_cnt = fq_stream_trimmer(in_single1_fq_fn, s1_pipe[1], out_read_prefix, no_pre->count, pre_read_len->count, &comp_s1, &singlet1_org, '\0', fasta->count);
}
}
#pragma omp section
{ // Single1 writer
s1_cnt = ss_stream_writer(out_single1_fn, s1_out_pipe[0], gzip->count) / read_count_divisor;
}
#pragma omp section
{ // Single2 reader
// When there is only one input singles file, but two output singles files, then supply which mate to use for this stream in the split parameter
if ((singles_flag) && (num_input_singles_files == 1)) {
singlet2_cnt = fq_stream_trimmer(in_single1_fq_fn, s2_pipe[1], out_read_prefix, no_pre->count, pre_read_len->count, &comp_s2, &singlet2_org, '2', fasta->count);
} else {
singlet2_cnt = fq_stream_trimmer(in_single2_fq_fn, s2_pipe[1], out_read_prefix, no_pre->count, pre_read_len->count, &comp_s2, &singlet2_org, '\0', fasta->count);
}
}
#pragma omp section
{ // Single2 writer
s2_cnt = ss_stream_writer(out_single2_fn, s2_out_pipe[0], gzip->count) / read_count_divisor;
}
#pragma omp section
{ // Velvet mates writer
// Divide count by 2 because both R1 and R2 reads go through this writer
mp_cnt = ss_stream_writer(out_mates_fn, mates_out_pipe[0], gzip->count) / 2 / read_count_divisor;
}
#pragma omp section
{ // Dispatcher
// Allocate data buffer strings
UT_string *r1_data;
utstring_new(r1_data);
UT_string *r2_data;
utstring_new(r2_data);
UT_string *s1_data;
utstring_new(s1_data);
UT_string *s2_data;
utstring_new(s2_data);
UT_string *rev_tmp;
utstring_new(rev_tmp);
UT_string *rev_data;
utstring_new(rev_data);
// Pipes
FILE *r1_in = fdopen(r1_pipe[0],"r");
FILE *r2_in = fdopen(r2_pipe[0],"r");
FILE *s1_in = fdopen(s1_pipe[0],"r");
FILE *s2_in = fdopen(s2_pipe[0],"r");
FILE *mates_out = fdopen(mates_out_pipe[1],"w");
FILE *r1_out = fdopen(r1_out_pipe[1],"w");
FILE *r2_out = fdopen(r2_out_pipe[1],"w");
FILE *s1_out = fdopen(s1_out_pipe[1],"w");
FILE *s2_out = fdopen(s2_out_pipe[1],"w");
if (!singles_flag) {
fclose(s2_out);
s2_out = s1_out;
}
// Flags for data left in single files
int single1_hungry = 1;
int single2_hungry = 1;
// Handle read1 and read2 files
while (ss_get_utstring(r1_in, r1_data)) {
if (!ss_get_utstring(r2_in, r2_data)) {
fprintf(stderr, "Error: Input read1 and read2 files are not synced\n");
exit(EXIT_FAILURE);
}
if (keep_read(r1_data)) {
if (keep_read(r2_data)) {
// Output both read1 and read2
if (mates->count) {
if (only_mates->count) {
// Interleaved velvet output only
output_read(r1_data, NULL, NULL, r1_in, NULL, mates_out, fasta->count);
if (no_rev->count || !colorspace_flag) {
output_read(r2_data, NULL, NULL, r2_in, NULL, mates_out, fasta->count);
} else {
output_read(r2_data, rev_data, rev_tmp, r2_in, NULL, mates_out, fasta->count);
}
} else {
// Interleaved velvet output and normal read file output
output_read(r1_data, NULL, NULL, r1_in, r1_out, mates_out, fasta->count);
if (no_rev->count || !colorspace_flag) {
output_read(r2_data, NULL, NULL, r2_in, r2_out, mates_out, fasta->count);
} else {
output_read(r2_data, rev_data, rev_tmp, r2_in, r2_out, mates_out, fasta->count);
}
}
} else {
// No interleaved velvet output
output_read(r1_data, NULL, NULL, r1_in, r1_out, NULL, fasta->count);
output_read(r2_data, NULL, NULL, r2_in, r2_out, NULL, fasta->count);
}
} else {
// Discard read2, output read1 as singlet
output_read(r1_data, NULL, NULL, r1_in, s1_out, NULL, fasta->count);
read1_singlet_cnt++;
}
} else {
if (keep_read(r2_data)) {
// Discard read1, output read2 as singlet
output_read(r2_data, NULL, NULL, r2_in, s2_out, NULL, fasta->count);
read2_singlet_cnt++;
}
}
// Process reads from singles here to take advantage of
// parallelism
if (single1_hungry || single2_hungry) {
if (single1_hungry) {
if (ss_get_utstring(s1_in, s1_data)) {
if (keep_read(s1_data)) {
output_read(s1_data, NULL, NULL, s1_in, s1_out, NULL, fasta->count);
}
} else {
single1_hungry = 0;
}
}
if (single2_hungry) {
if (ss_get_utstring(s2_in, s2_data)) {
if (keep_read(s2_data)) {
output_read(s2_data, NULL, NULL, s2_in, s2_out, NULL, fasta->count);
}
} else {
single2_hungry = 0;
}
}
}
}
while (single1_hungry || single2_hungry) {
if (single1_hungry) {
if (ss_get_utstring(s1_in, s1_data)) {
if (keep_read(s1_data)) {
output_read(s1_data, NULL, NULL, s1_in, s1_out, NULL, fasta->count);
}
} else {
single1_hungry = 0;
}
}
if (single2_hungry) {
if (ss_get_utstring(s2_in, s2_data)) {
if (keep_read(s2_data)) {
output_read(s2_data, NULL, NULL, s2_in, s2_out, NULL, fasta->count);
}
} else {
single2_hungry = 0;
}
}
}
fclose(r1_in);
fclose(r2_in);
fclose(s1_in);
fclose(s2_in);
fclose(mates_out);
fclose(r1_out);
fclose(r2_out);
fclose(s1_out);
if (singles_flag) {
fclose(s2_out);
}
// Free buffers
utstring_free(r1_data);
utstring_free(r2_data);
utstring_free(s1_data);
utstring_free(s2_data);
utstring_free(rev_tmp);
utstring_free(rev_data);
}
}
if (!(R1_org+singlet1_org+singlet2_org)) {
fprintf(stderr, "ERROR! No reads found in input files, or input(s) not found.\n");
exit(EXIT_FAILURE);
}
if (R1_org != R2_org) {
fprintf(stderr, "\nWarning! read1 and read2 fastq files did not contain an equal number of reads. %lu %lu\n", R1_org, R2_org);
}
if ((R1_org + R2_org) && !(singlet1_cnt + singlet2_cnt)) {
fprintf(stderr, "\nWarning! read1/read2 files were processed, but no corresponding input singlets were found.\n");
}
if (entropy->count) {
printf("\nLow complexity reads discarded: Read1: %lu, Read2: %lu, Singlets: %lu %lu\n", comp_r1, comp_r2, comp_s1, comp_s2);
}
mp_org = R1_org;
if (!only_mates->count) {
mp_cnt = R1_cnt;
}
printf("\nMatepairs: Before: %lu, After: %lu\n", mp_org, mp_cnt);
printf("Singlets: Before: %lu %lu After: %lu %lu\n", singlet1_org, singlet2_org, s1_cnt, s2_cnt);
printf("Read1 singlets: %lu, Read2 singlets: %lu, Original singlets: %lu %lu\n", read1_singlet_cnt, read2_singlet_cnt, singlet1_cnt, singlet2_cnt);
printf("Total Reads Processed: %lu, Reads retained: %lu\n", 2*mp_org+singlet1_org+singlet2_org, 2*mp_cnt+s1_cnt+s2_cnt);
utstring_free(in_read1_fq_fn);
utstring_free(in_read2_fq_fn);
utstring_free(in_single1_fq_fn);
utstring_free(in_single2_fq_fn);
utstring_free(out_read1_fn);
utstring_free(out_read2_fn);
utstring_free(out_single1_fn);
utstring_free(out_single2_fn);
utstring_free(out_mates_fn);
utstring_free(out_filetype);
utstring_free(out_read_prefix);
exit(EXIT_SUCCESS);
}
unsigned long int fq_stream_trimmer(UT_string *fq_fn, int pipe_fd, UT_string *out_prefix, int no_pre, int len_pre, unsigned long int *comp_cnt, unsigned long int *org, char split, int fmt_fasta) {
UT_string *new_head_data;
utstring_new(new_head_data);
UT_string *head_data;
utstring_new(head_data);
UT_string *seq_data;
utstring_new(seq_data);
UT_string *extra_data;
utstring_new(extra_data);
UT_string *qual_data;
utstring_new(qual_data);
unsigned long int cnt = 0;
char *start = NULL;
char *end = NULL;
char *suffix = NULL;
FILE *fq_file = NULL;
FILE *pipe_in = fdopen(pipe_fd, "w");
if (!(utstring_len(fq_fn))) {
fclose(pipe_in);
return(0);
}
// Try to open the fastq file
if (!(fq_file = gzopen(utstring_body(fq_fn), "r"))) {
utstring_printf(fq_fn, ".gz");
if (!(fq_file = gzopen(utstring_body(fq_fn), "r"))) {
fclose(pipe_in);
return(0);
}
}
int x = 0;
char head_char = '@';
if (fmt_fasta) {
head_char = '>';
}
while (ss_gzget_utstring(fq_file, head_data)) {
ss_gzget_utstring(fq_file, seq_data);
ss_gzget_utstring(fq_file, extra_data);
ss_gzget_utstring(fq_file, qual_data);
if (!split || ((suffix = strchr(utstring_body(head_data), '/')) && (suffix[1] == split))) {
(*org)++;
if ((x = trimmer(utstring_body(qual_data))) >= min_len) { // Keep at least some of read
// Reject read if complexity is too low
if ((entropy_cutoff < 0.0) || (entropy_calc(utstring_body(seq_data), x) >= entropy_cutoff)) {
// Truncate sequence
ss_trim_utstring(seq_data, x);
ss_strcat_utstring(seq_data, "\n");
if (!fmt_fasta) {
ss_trim_utstring(qual_data, x);
ss_strcat_utstring(qual_data, "\n");
}
// Fixup the read name
utstring_clear(new_head_data);
end = strchr(utstring_body(head_data), ':');
if (no_pre) {
if ((start = strchr(utstring_body(head_data), '|'))) {
start++;
} else {
if (colorspace_flag) {
start = utstring_body(head_data) + 4;
} else {
start = utstring_body(head_data) + 1;
}
}
*end = '\0';
} else {
start = utstring_body(out_prefix);
}
end++;
if (colorspace_flag) {
if (len_pre) {
utstring_printf(new_head_data, "%c%.2s+%u|%s:%s",head_char,utstring_body(head_data)+1,x,start,end);
} else {
utstring_printf(new_head_data, "%c%.2s+%s:%s",head_char,utstring_body(head_data)+1,start,end);
}
} else {
if (len_pre) {
utstring_printf(new_head_data, "%c%u|%s:%s",head_char,x,start,end);
} else {
utstring_printf(new_head_data, "%c%s:%s",head_char,start,end);
}
}
fputs(utstring_body(new_head_data), pipe_in);
fputs(utstring_body(seq_data), pipe_in);
if (!fmt_fasta) {
fputs(utstring_body(extra_data), pipe_in);
fputs(utstring_body(qual_data), pipe_in);
}
cnt++;
} else {
// rejected by entropy filter
// Send along placeholder read to be discarded, keeping read1 and read2 in sync
// Empty fastq header is a read to be rejected by consumer threads
(*comp_cnt)++;
fputs("\n", pipe_in);
}
} else {
// rejected by minimum length cutoff
// Send along placeholder read to be discarded, keeping read1 and read2 in sync
// Empty fastq header is a read to be rejected by consumer threads
fputs("\n", pipe_in);
}
}
}
fclose(pipe_in);
gzclose(fq_file);
utstring_free(new_head_data);
utstring_free(head_data);
utstring_free(seq_data);
utstring_free(extra_data);
utstring_free(qual_data);
return(cnt);
}
int
exec_rquery(int argc, char **argv)
{
struct pkgdb *db = NULL;
struct pkgdb_it *it = NULL;
struct pkg *pkg = NULL;
char *pkgname = NULL;
int query_flags = PKG_LOAD_BASIC;
match_t match = MATCH_EXACT;
int ch;
int ret = EPKG_OK;
int retcode = EX_OK;
int i;
char multiline = 0;
char *condition = NULL;
const char *portsdir;
UT_string *sqlcond = NULL;
const unsigned int q_flags_len = NELEM(accepted_rquery_flags);
const char *reponame = NULL;
bool onematched = false;
bool old_quiet;
bool index_output = false;
struct option longopts[] = {
{ "all", no_argument, NULL, 'a' },
{ "case-sensitive", no_argument, NULL, 'C' },
{ "evaluate", required_argument, NULL, 'e' },
{ "glob", no_argument, NULL, 'g' },
{ "case-insensitive", no_argument, NULL, 'i' },
{ "index-line", no_argument, NULL, 'I' },
{ "repository", required_argument, NULL, 'r' },
{ "no-repo-update", no_argument, NULL, 'U' },
{ "regex", no_argument, NULL, 'x' },
{ NULL, 0, NULL, 0 },
};
portsdir = pkg_object_string(pkg_config_get("PORTSDIR"));
while ((ch = getopt_long(argc, argv, "+aCgiIxe:r:U", longopts, NULL)) != -1) {
switch (ch) {
case 'a':
match = MATCH_ALL;
break;
case 'C':
pkgdb_set_case_sensitivity(true);
break;
case 'e':
match = MATCH_CONDITION;
condition = optarg;
break;
case 'g':
match = MATCH_GLOB;
break;
case 'i':
pkgdb_set_case_sensitivity(false);
break;
case 'I':
index_output = true;
break;
case 'r':
reponame = optarg;
break;
case 'U':
auto_update = false;
break;
case 'x':
match = MATCH_REGEX;
break;
default:
usage_rquery();
return (EX_USAGE);
}
}
argc -= optind;
argv += optind;
if (argc == 0 && !index_output) {
usage_rquery();
return (EX_USAGE);
}
/* Default to all packages if no pkg provided */
if (!index_output) {
if (argc == 1 && condition == NULL && match == MATCH_EXACT) {
match = MATCH_ALL;
} else if (((argc == 1) ^ (match == MATCH_ALL )) && condition == NULL) {
usage_rquery();
return (EX_USAGE);
}
} else {
if (argc == 0)
match = MATCH_ALL;
}
if (!index_output && analyse_query_string(argv[0], accepted_rquery_flags, q_flags_len, &query_flags, &multiline) != EPKG_OK)
return (EX_USAGE);
if (condition != NULL) {
utstring_new(sqlcond);
if (format_sql_condition(condition, sqlcond, true) != EPKG_OK) {
utstring_free(sqlcond);
return (EX_USAGE);
}
}
ret = pkgdb_access(PKGDB_MODE_READ, PKGDB_DB_REPO);
if (ret == EPKG_ENOACCESS) {
warnx("Insufficient privileges to query the package database");
if (sqlcond != NULL)
utstring_free(sqlcond);
return (EX_NOPERM);
} else if (ret != EPKG_OK) {
if (sqlcond != NULL)
utstring_free(sqlcond);
return (EX_IOERR);
}
/* first update the remote repositories if needed */
old_quiet = quiet;
quiet = true;
if (auto_update && (ret = pkgcli_update(false, false, reponame)) != EPKG_OK) {
if (sqlcond != NULL)
utstring_free(sqlcond);
return (ret);
}
quiet = old_quiet;
ret = pkgdb_open_all(&db, PKGDB_REMOTE, reponame);
if (ret != EPKG_OK) {
if (sqlcond != NULL)
utstring_free(sqlcond);
return (EX_IOERR);
}
drop_privileges();
if (index_output)
query_flags = PKG_LOAD_BASIC|PKG_LOAD_CATEGORIES|PKG_LOAD_DEPS;
if (match == MATCH_ALL || match == MATCH_CONDITION) {
const char *condition_sql = NULL;
if (match == MATCH_CONDITION && sqlcond)
condition_sql = utstring_body(sqlcond);
if ((it = pkgdb_repo_query(db, condition_sql, match, reponame)) == NULL) {
if (sqlcond != NULL)
utstring_free(sqlcond);
return (EX_IOERR);
}
while ((ret = pkgdb_it_next(it, &pkg, query_flags)) == EPKG_OK) {
if (index_output)
print_index(pkg, portsdir);
else
print_query(pkg, argv[0], multiline);
}
if (ret != EPKG_END)
retcode = EX_SOFTWARE;
pkgdb_it_free(it);
} else {
for (i = (index_output ? 0 : 1); i < argc; i++) {
pkgname = argv[i];
if ((it = pkgdb_repo_query(db, pkgname, match, reponame)) == NULL) {
if (sqlcond != NULL)
utstring_free(sqlcond);
return (EX_IOERR);
}
while ((ret = pkgdb_it_next(it, &pkg, query_flags)) == EPKG_OK) {
onematched = true;
if (index_output)
print_index(pkg, portsdir);
else
print_query(pkg, argv[0], multiline);
}
if (ret != EPKG_END) {
retcode = EX_SOFTWARE;
break;
}
pkgdb_it_free(it);
}
if (!onematched && retcode == EX_OK)
retcode = EX_UNAVAILABLE;
}
if (sqlcond != NULL)
utstring_free(sqlcond);
pkg_free(pkg);
pkgdb_close(db);
return (retcode);
}
static void
check_vulnerable(struct pkg_audit *audit, struct pkgdb *db, int sock)
{
struct pkgdb_it *it = NULL;
struct pkg *pkg = NULL;
kh_pkgs_t *check = NULL;
const char *uid;
UT_string *sb;
int ret;
FILE *out;
out = fdopen(sock, "w");
if (out == NULL) {
warn("unable to open stream");
return;
}
if ((it = pkgdb_query(db, NULL, MATCH_ALL)) == NULL) {
warnx("Error accessing the package database");
pkg_audit_free(audit);
fclose(out);
return;
}
else {
check = kh_init_pkgs();
while ((ret = pkgdb_it_next(it, &pkg, PKG_LOAD_BASIC|PKG_LOAD_RDEPS))
== EPKG_OK) {
add_to_check(check, pkg);
pkg = NULL;
}
ret = EX_OK;
}
if (db != NULL) {
pkgdb_it_free(it);
pkgdb_close(db);
}
if (ret != EX_OK) {
pkg_audit_free(audit);
kh_destroy_pkgs(check);
fclose(out);
return;
}
drop_privileges();
if (pkg_audit_load(audit, NULL) != EPKG_OK) {
warn("unable to open vulnxml file");
fclose(out);
pkg_audit_free(audit);
return;
}
#ifdef HAVE_CAPSICUM
if (cap_enter() < 0 && errno != ENOSYS) {
warn("cap_enter() failed");
pkg_audit_free(audit);
kh_destroy_pkgs(check);
fclose(out);
return;
}
#endif
if (pkg_audit_process(audit) == EPKG_OK) {
kh_foreach_value(check, pkg, {
if (pkg_audit_is_vulnerable(audit, pkg, true, &sb)) {
pkg_get(pkg, PKG_UNIQUEID, &uid);
fprintf(out, "%s\n", uid);
fflush(out);
utstring_free(sb);
}
pkg_free(pkg);
});
