예제 #1
0
파일: query.c 프로젝트: HardenedBSD/pkg
struct pkg_repo_it *
pkg_repo_binary_require(struct pkg_repo *repo, const char *provide)
{
	sqlite3_stmt	*stmt;
	sqlite3 *sqlite = PRIV_GET(repo);
	UT_string	*sql = NULL;
	int		 ret;
	const char	 basesql[] = ""
			"SELECT p.id, p.origin, p.name, p.version, p.comment, "
			"p.name as uniqueid, "
			"p.prefix, p.desc, p.arch, p.maintainer, p.www, "
			"p.licenselogic, p.flatsize, p.pkgsize, "
			"p.cksum, p.manifestdigest, p.path AS repopath, '%s' AS dbname "
			"FROM packages AS p INNER JOIN pkg_requires AS ps ON "
			"p.id = ps.package_id "
			"WHERE ps.require_id = (SELECT id FROM requires WHERE require=?1);";

	utstring_new(sql);
	utstring_printf(sql, basesql, repo->name);

	pkg_debug(4, "Pkgdb: running '%s'", utstring_body(sql));
	ret = sqlite3_prepare_v2(sqlite, utstring_body(sql), -1, &stmt, NULL);
	if (ret != SQLITE_OK) {
		ERROR_SQLITE(sqlite, utstring_body(sql));
		utstring_free(sql);
		return (NULL);
	}

	utstring_free(sql);

	sqlite3_bind_text(stmt, 1, provide, -1, SQLITE_TRANSIENT);

	return (pkg_repo_binary_it_new(repo, stmt, PKGDB_IT_FLAG_ONCE));
}
예제 #2
0
int main() {
    UT_string *s,*t;
    char V_TestStr[] = "There are two needle\0s in this \0haystack with needle\0s.";
    char V_NeedleStr[] = "needle\0s";
    long *V_KMP_Table;
    long V_FindPos;
    size_t V_StartPos;
    size_t V_FindCnt;


    utstring_new(s);
    utstring_new(t);

    utstring_bincpy(s, V_TestStr, sizeof(V_TestStr)-1);
    printf("\"%s\" len=%u\n", utstring_body(s), utstring_len(s));
    utstring_bincpy(t, V_NeedleStr, sizeof(V_NeedleStr)-1);
    printf("\"%s\" len=%u\n", utstring_body(t), utstring_len(t));

    V_KMP_Table = (long *)malloc(sizeof(long) * (utstring_len(t) + 1));
    if (V_KMP_Table != NULL)
    {
        _utstring_BuildTable(utstring_body(t), utstring_len(t), V_KMP_Table);

        V_FindCnt = 0;
        V_FindPos = 0;
        V_StartPos = 0;
        do
        {
            V_FindPos = _utstring_find(utstring_body(s) + V_StartPos, 
                                       utstring_len(s) - V_StartPos, 
                                       utstring_body(t), 
                                       utstring_len(t), 
                                       V_KMP_Table);
            if (V_FindPos >= 0)
            {
                V_FindPos += V_StartPos;
                V_FindCnt++;
                V_StartPos = V_FindPos + 1;
            }
            printf("utstring_find()=%ld\n", V_FindPos);
        } while (V_FindPos >= 0);
        printf("FindCnt=%u\n", V_FindCnt);

        free(V_KMP_Table);
    }
    else
    {
        printf("malloc() failed...\n");
    }

    utstring_free(s);
    utstring_free(t);

    return 0;
}
예제 #3
0
// TODO: finish to test this one
int o_bin_dbcreate(orientdb *o, orientdb_con *c, struct timeval *timeout, int nonblocking, const char *dbname, int dbtype) {
	UT_string *s;
	int i;
	
	debug_note(o, ORIENT_INFO, "executing binary method: DB_CREATE\n");
	i = get_con_fd(o, c);
	if (i == -1) return -1;

	
	// check the dbname and dbtype parameters
	if ((dbname == NULL) || (strlen(dbname) == 0) || ((dbtype != O_DBTYPE_LOCAL) && (dbtype != O_DBTYPE_MEMORY))) return -1;
	
	utstring_new(s);
	
	// send the operation we want to execute to the server
	i = o_sendrequest(o, c, O_DB_CREATE);
	
	// send db name
	utstring_printf(s, dbname);
	i = o_putstring(o, c, s);
	utstring_clear(s);
	
	switch (dbtype) {
	case O_DBTYPE_LOCAL:
		utstring_printf(s, "local");
		break;
	case O_DBTYPE_MEMORY:
		utstring_printf(s, "memory");
		break;
	default:
		break;
	}
	
	// send db type
	i = o_putstring(o, c, s);
	utstring_clear(s);
	
	/*
	i = o_sendcredentials(o, c, ORIENT_USER);
	*/
	i = o_flushsendbuffer(o, c, timeout);
	
	i = o_getresponse(o, c, timeout);
	if (i == 1) { /* EXCEPTION!! */
		debug_note(o, ORIENT_INFO, "received an exception\n");
		i = o_getexceptions(o, c, timeout, s);
		debug_note(o, ORIENT_INFO, "E: %s\n", utstring_body(s));
		utstring_free(s);
		return -1;
	}
	
	utstring_free(s);
	
	return 0;
}
예제 #4
0
short o_bin_dataclusteradd(orientdb *o, orientdb_con *c, struct timeval *timeout, int nonblocking, const char *type, const char *name, const char *filename, int initialsize) {
	UT_string *s;
	int i;
	short x;
	
	debug_note(o, ORIENT_INFO, "executing binary method: DATACLUSTER_ADD\n");
	i = get_con_fd(o, c);
	if (i == -1) return -1;
	
	utstring_new(s);
	// send the operation we want to execute to the server
	i = o_sendrequest(o, c, O_DATACLUSTER_ADD);
	
	// send cluster type
	utstring_printf(s, type);
	i = o_putstring(o, c, s);
	utstring_clear(s);
	
	// send cluster name
	utstring_printf(s, name);
	i = o_putstring(o, c, s);
	utstring_clear(s);
	
	if (strcmp(type, O_CLUSTER_PHYSICAL) == 0) { // the physical cluster need filename and initialsize, while memory and logical do not need them.
		// send cluster filename
		utstring_printf(s, filename);
		i = o_putstring(o, c, s);
		utstring_clear(s);
		
		// send cluster initialsize in bytes
		i = o_putint(o, c, initialsize);
	}
	
	i = o_flushsendbuffer(o, c, timeout);
	
	i = o_getresponse(o, c, timeout);
	if (i == 1) { /* EXCEPTION!! */
		debug_note(o, ORIENT_INFO, "received an exception\n");
		i = o_getexceptions(o, c, timeout, s);
		debug_note(o, ORIENT_INFO, "E: %s\n", utstring_body(s));
		utstring_free(s);
		return -1;
	}
	
	// get the new cluster id
	x = o_getshort(o, c, timeout);
	
	utstring_free(s);
	
	return x;
}
예제 #5
0
char *mtex2MML_combine_row_data(UT_array **environment_data_stack)
{
  /* if no information was provided, give a standard sizing */
  if (utarray_len(*environment_data_stack) == 0) {
    const char* s = "rowspacing=\"0.5ex\" rowlines=\"none\"";
    char* c = (char*)malloc(strlen(s) + 1);
    strcpy(c, s);
    return c;
  }
  envdata_t *row_data_elem = (envdata_t*) utarray_front(*environment_data_stack);

  char *row_spacing_data = row_data_elem->rowspacing,
        *row_lines_data = row_data_elem->rowlines,
         *row_attr;

  UT_string *row_attr_data;
  utstring_new(row_attr_data);

  /* combine the row spacing and row lines data */
  utstring_printf(row_attr_data, "%s%s\" %s\"", "rowspacing=\"", row_spacing_data, row_lines_data);

  row_attr = string_dup(utstring_body(row_attr_data));
  utarray_erase(*environment_data_stack, 0, 1);

  utstring_free(row_attr_data);

  return row_attr;
}
예제 #6
0
char *mtex2MML_remove_excess_pipe_chars(char *string)
{
  UT_string *columnalign;
  utstring_new(columnalign);

  char *dupe = string_dup(string);
  char *token = strtok(dupe, " ");
  char *attr_columnalign;

  while (token != NULL) {
    if (strncmp(token, "s", 1) != 0 && strncmp(token, "d", 1) != 0) {
      utstring_printf(columnalign, "%s ", token);
    }
    token = strtok(NULL, " ");
  }

  attr_columnalign = string_dup(utstring_body(columnalign));
  free(dupe);
  utstring_free(columnalign);

  if (strlen(attr_columnalign) > 0) {
    mtex2MML_remove_last_char(attr_columnalign); /* remove the final space */
  }

  return attr_columnalign;
}
예제 #7
0
파일: kvsp-tpub.c 프로젝트: JHUAPL/kvspool
/* clean up the client output buffers and slots in fd/buf arrays */
void discard_client_buffers(int pos) {
  UT_string **s = (UT_string**)utarray_eltptr(cfg.outbufs,pos);
  utstring_free(*s);                // deep free string 
  utarray_erase(cfg.outbufs,pos,1); // erase string pointer
  utarray_erase(cfg.outidxs,pos,1); // erase write index
  utarray_erase(cfg.clients,pos,1); // erase client descriptor
}
예제 #8
0
파일: app.c 프로젝트: stoni/luarest
luarest_status invoke_application(application* apps, UT_string* url, luarest_method m, luarest_response* res_code, 
	luarest_content_type* con_type, UT_string* res_buf)
{
	application* app = NULL;
	service *service;
	char* pch = NULL;
	char* tmp = utstring_body(url);
	UT_string* app_name;
	UT_string* key;

	utstring_new(app_name);
	pch = strchr(++tmp, '/');
	if (pch == NULL) {
		return(LUAREST_ERROR);
	}
	utstring_bincpy(app_name, tmp, pch-tmp);
	HASH_FIND(hh, apps, utstring_body(app_name), utstring_len(app_name), app);
	utstring_free(app_name);
	if (app == NULL) {
		return(LUAREST_ERROR);
	}
	utstring_new(key);
	utstring_printf(key, "M%d#P%s", m, pch);
	HASH_FIND(hh, app->s, utstring_body(key), utstring_len(key), service);
	if (service == NULL) {
		return(LUAREST_ERROR);
	}
	invoke_lua(app->lua_state, service->callback_ref, res_code, con_type, res_buf);
	return(LUAREST_SUCCESS);
}
예제 #9
0
int o_bin_shutdown(orientdb *o, orientdb_con *c, struct timeval *timeout, int nonblocking) {
	UT_string *s = NULL;
	int i;
	
	debug_note(o, ORIENT_INFO, "executing binary method: SHUTDOWN\n");
	i = get_con_fd(o, c);
	if (i == -1) return -1;

	// send the operation we want to execute to the server
	i = o_sendrequest(o, c, O_SHUTDOWN);
	i = o_sendcredentials(o, c, ORIENT_ADMIN);
	i = o_flushsendbuffer(o, c, timeout);
	
	i = o_getresponse(o, c, timeout);
	if (i == 1) { /* EXCEPTION!! */
		debug_note(o, ORIENT_INFO, "received an exception\n");
		utstring_new(s);
		i = o_getexceptions(o, c, timeout, s);
		debug_note(o, ORIENT_INFO, "E: %s\n", utstring_body(s));
		utstring_free(s);
		return -1;
	}
	
	return 0;
}
예제 #10
0
파일: query.c 프로젝트: HardenedBSD/pkg
struct pkg_repo_it *
pkg_repo_binary_query(struct pkg_repo *repo, const char *pattern, match_t match)
{
	sqlite3 *sqlite = PRIV_GET(repo);
	sqlite3_stmt	*stmt = NULL;
	UT_string	*sql = NULL;
	const char	*comp = NULL;
	int		 ret;
	char		 basesql[BUFSIZ] = ""
		"SELECT id, origin, name, name as uniqueid, version, comment, "
		"prefix, desc, arch, maintainer, www, "
		"licenselogic, flatsize, pkgsize, "
		"cksum, manifestdigest, path AS repopath, '%s' AS dbname "
		"FROM packages AS p";

	if (match != MATCH_ALL && (pattern == NULL || pattern[0] == '\0'))
		return (NULL);

	utstring_new(sql);
	comp = pkgdb_get_pattern_query(pattern, match);
	if (comp && comp[0])
		strlcat(basesql, comp, sizeof(basesql));

	utstring_printf(sql, basesql, repo->name);

	utstring_printf(sql, "%s", " ORDER BY name;");

	pkg_debug(4, "Pkgdb: running '%s' query for %s", utstring_body(sql),
	     pattern == NULL ? "all": pattern);
	ret = sqlite3_prepare_v2(sqlite, utstring_body(sql), utstring_len(sql), &stmt,
	    NULL);
	if (ret != SQLITE_OK) {
		ERROR_SQLITE(sqlite, utstring_body(sql));
		utstring_free(sql);
		return (NULL);
	}

	utstring_free(sql);

	if (match != MATCH_ALL && match != MATCH_CONDITION)
		sqlite3_bind_text(stmt, 1, pattern, -1, SQLITE_TRANSIENT);

	return (pkg_repo_binary_it_new(repo, stmt, PKGDB_IT_FLAG_ONCE));
}
예제 #11
0
파일: kvsp-bpub.c 프로젝트: JHUAPL/kvspool
int main(int argc, char *argv[]) {
  void *sp=NULL;
  void *set=NULL;
  int opt,rc=-1;
  char *config_file;
  set = kv_set_new();
  utarray_new(output_keys, &ut_str_icd);
  utarray_new(output_defaults, &ut_str_icd);
  utarray_new(output_types,&ut_int_icd);
  utstring_new(tmp);

  while ( (opt = getopt(argc, argv, "v+d:b:s")) != -1) {
    switch (opt) {
      case 'v': verbose++; break;
      case 's': push_mode++; break;
      case 'd': spool=strdup(optarg); break;
      case 'b': config_file=strdup(optarg); break;
      default: usage(argv[0]); break;
    }
  }
  if (optind < argc) pub_transport = argv[optind++];
  if (!pub_transport) usage(argv[0]);
  if (spool == NULL) usage(argv[0]);
  if (parse_config(config_file) < 0) goto done;

  if ( !(pub_context = zmq_init(1))) goto done;
  if ( !(pub_socket = zmq_socket(pub_context, push_mode?ZMQ_PUSH:ZMQ_PUB))) goto done;
  if (zmq_setsockopt(pub_socket, ZMQ_SNDHWM, &hwm, sizeof(hwm))) goto done;
  if (zmq_bind(pub_socket, pub_transport) == -1) goto done;

  sp = kv_spoolreader_new(spool);
  if (!sp) goto done;

  while (kv_spool_read(sp,set,1) > 0) { /* read til interrupted by signal */
    zmq_msg_t part;
    if (set_to_binary(set,&part) < 0) goto done;
    rc = zmq_sendmsg(pub_socket, &part, 0);
    zmq_msg_close(&part);
    if (rc == -1) goto done;
  }

  rc = 0;

 done:
  if (rc) fprintf(stderr,"zmq: %s %s\n", pub_transport, zmq_strerror(errno));
  if (pub_socket) zmq_close(pub_socket);
  if (pub_context) zmq_term(pub_context);
  if (sp) kv_spoolreader_free(sp);
  kv_set_free(set);
  utarray_free(output_keys);
  utarray_free(output_defaults);
  utarray_free(output_types);
  utstring_free(tmp);

  return 0;
}
예제 #12
0
int main()
{
    UT_string *s;

    utstring_new(s);
    utstring_printf(s, "hello world!" );
    printf("%s\n", utstring_body(s));

    utstring_free(s);
    return 0;
}
예제 #13
0
파일: kvsp-init.c 프로젝트: JHUAPL/kvspool
int main(int argc, char * argv[]) {
  int opt,verbose=0;
  long dirmax=0;
  /* input spool */
  char *dir=NULL,unit,*sz="10GB";
  UT_string *s;
  utstring_new(s);

  while ( (opt = getopt(argc, argv, "v+s:")) != -1) {
    switch (opt) {
      default: usage(argv[0]); break;
      case 'v': verbose++; break;
      case 's': 
         sz = strdup(optarg);
         switch (sscanf(sz, "%ld%c", &dirmax, &unit)) {
           case 2: /* check unit */
            switch (unit) {
              case 't': case 'T': break;
              case 'g': case 'G': break;
              case 'm': case 'M': break;
              case 'k': case 'K': break;
              case '\r': case '\n': case ' ': case '\t': break;
              default: usage(argv[0]); break;
            }
           case 1: /* just a number in bytes */ break;
           default: usage(argv[0]); break;
         }
         break;
    }
  }
  if (optind >= argc) usage(argv[0]);
  if (!dirmax) usage(argv[0]);
  kv_spool_options.dir_max = dirmax;

  while (optind < argc) {
    dir = argv[optind++];

    utstring_clear(s);
    utstring_printf(s,"%s/limits", dir);
    char *p = utstring_body(s);
    FILE *f = fopen(p, "w");
    if (f == NULL) {
      fprintf(stderr,"cannot open %s: %s\n", p, strerror(errno));
      continue;
    }
    fprintf(f, "%s", sz);
    fclose(f);
    sp_attrition(dir);
  }

 done:
  utstring_free(s);
  return 0;
}
예제 #14
0
파일: query.c 프로젝트: HardenedBSD/pkg
struct pkg_repo_it *
pkg_repo_binary_search(struct pkg_repo *repo, const char *pattern, match_t match,
    pkgdb_field field, pkgdb_field sort)
{
	sqlite3 *sqlite = PRIV_GET(repo);
	sqlite3_stmt	*stmt = NULL;
	UT_string	*sql = NULL;
	int		 ret;
	const char	*multireposql = ""
		"SELECT id, origin, name, version, comment, "
		"prefix, desc, arch, maintainer, www, "
		"licenselogic, flatsize, pkgsize, "
		"cksum, path AS repopath, '%1$s' AS dbname, '%2$s' AS repourl "
		"FROM packages ";

	if (pattern == NULL || pattern[0] == '\0')
		return (NULL);

	utstring_new(sql);
	utstring_printf(sql, multireposql, repo->name, repo->url);

	/* close the UNIONs and build the search query */
	utstring_printf(sql, "%s", "WHERE ");

	pkg_repo_binary_build_search_query(sql, match, field, sort);
	utstring_printf(sql, "%s", ";");

	pkg_debug(4, "Pkgdb: running '%s'", utstring_body(sql));
	ret = sqlite3_prepare_v2(sqlite, utstring_body(sql), -1, &stmt, NULL);
	if (ret != SQLITE_OK) {
		ERROR_SQLITE(sqlite, utstring_body(sql));
		utstring_free(sql);
		return (NULL);
	}

	utstring_free(sql);

	sqlite3_bind_text(stmt, 1, pattern, -1, SQLITE_TRANSIENT);

	return (pkg_repo_binary_it_new(repo, stmt, PKGDB_IT_FLAG_ONCE));
}
예제 #15
0
static PyObject* binaryplist_encode(PyObject *self, PyObject *args, PyObject *kwargs)
{
    static char *kwlist[] = {"obj", "unique", "debug", "convert_nulls",
                             "max_recursion", "object_hook", "as_ascii", NULL};
    PyObject *newobj = NULL;
    PyObject *oinput = NULL;
    PyObject *ounique = NULL;
    PyObject *odebug = NULL;
    PyObject *orecursion = NULL;
    binaryplist_encoder encoder;
    
    memset(&encoder, 0, sizeof(binaryplist_encoder));
    encoder.convert_nulls = Py_False;
    encoder.as_ascii = Py_False;
    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|OOOOOO", kwlist, &oinput, &ounique,
        &odebug, &(encoder.convert_nulls), &orecursion, &(encoder.object_hook),
        &(encoder.as_ascii))) {   
        return NULL;
    }
    if (encoder.object_hook && !PyCallable_Check(encoder.object_hook)) {
        PyErr_SetString(PLIST_Error, "object_hook is not callable");
        return NULL;
    }
    encoder.ref_table = PyDict_New();
    encoder.objects = PyList_New(0);
    utstring_new(encoder.output);
    if (!ounique || (ounique && PyObject_IsTrue(ounique))) {
        /* default to True */
        encoder.dounique = 1;
        encoder.uniques = PyDict_New();
    }
    if (odebug && PyObject_IsTrue(odebug)) {
        encoder.debug = 1;
    }
    if (orecursion && PyInt_Check(orecursion)) {
        encoder.max_recursion = PyInt_AsLong(orecursion);
    } else {
        encoder.max_recursion = 1024*16;
    }

    if (encoder_encode_object(&encoder, oinput) == BINARYPLIST_OK
        && encoder_write(&encoder) == BINARYPLIST_OK) {
        newobj = PyString_FromStringAndSize(utstring_body(encoder.output),
            utstring_len(encoder.output));
    }

    Py_DECREF(encoder.ref_table);
    Py_DECREF(encoder.objects);
    Py_XDECREF(encoder.uniques);
    utstring_free(encoder.output);

    return newobj;
}
예제 #16
0
파일: packing.c 프로젝트: Absolight/pkg
int
packing_append_tree(struct packing *pack, const char *treepath,
    const char *newroot)
{
	FTS *fts = NULL;
	FTSENT *fts_e = NULL;
	size_t treelen;
	UT_string *sb;
	char *paths[2] = { __DECONST(char *, treepath), NULL };

	treelen = strlen(treepath);
	fts = fts_open(paths, FTS_PHYSICAL | FTS_XDEV, NULL);
	if (fts == NULL)
		goto cleanup;

	utstring_new(sb);
	while ((fts_e = fts_read(fts)) != NULL) {
		switch(fts_e->fts_info) {
		case FTS_D:
		case FTS_DEFAULT:
		case FTS_F:
		case FTS_SL:
		case FTS_SLNONE:
			 /* Entries not within this tree are irrelevant. */
			 if (fts_e->fts_pathlen <= treelen)
				  break;
			 utstring_clear(sb);
			 /* Strip the prefix to obtain the target path */
			 if (newroot) /* Prepend a root if one is specified */
				  utstring_printf(sb, "%s", newroot);
			 /* +1 = skip trailing slash */
			 utstring_printf(sb, "%s", fts_e->fts_path + treelen + 1);
			 packing_append_file_attr(pack, fts_e->fts_name,
			    utstring_body(sb), NULL, NULL, 0, 0);
			 break;
		case FTS_DC:
		case FTS_DNR:
		case FTS_ERR:
		case FTS_NS:
			 /* XXX error cases, check fts_e->fts_errno and
			  *     bubble up the call chain */
			 break;
		default:
			 break;
		}
	}
	utstring_free(sb);
cleanup:
	fts_close(fts);
	return EPKG_OK;
}
예제 #17
0
void
pkg_free(struct pkg *pkg)
{
	if (pkg == NULL)
		return;

	free(pkg->name);
	free(pkg->origin);
	free(pkg->old_version);
	free(pkg->maintainer);
	free(pkg->www);
	free(pkg->arch);
	free(pkg->abi);
	free(pkg->uid);
	free(pkg->digest);
	free(pkg->old_digest);
	free(pkg->prefix);
	free(pkg->comment);
	free(pkg->desc);
	free(pkg->sum);
	free(pkg->repopath);
	free(pkg->repourl);
	free(pkg->reason);
	free(pkg->dep_formula);

	for (int i = 0; i < PKG_NUM_SCRIPTS; i++)
		if (pkg->scripts[i])
			utstring_free(pkg->scripts[i]);

	pkg_list_free(pkg, PKG_DEPS);
	pkg_list_free(pkg, PKG_RDEPS);
	pkg_list_free(pkg, PKG_FILES);
	pkg_list_free(pkg, PKG_DIRS);
	pkg_list_free(pkg, PKG_OPTIONS);
	pkg_list_free(pkg, PKG_USERS);
	pkg_list_free(pkg, PKG_GROUPS);
	pkg_list_free(pkg, PKG_SHLIBS_REQUIRED);
	pkg_list_free(pkg, PKG_SHLIBS_PROVIDED);
	pkg_list_free(pkg, PKG_PROVIDES);
	pkg_list_free(pkg, PKG_REQUIRES);
	pkg_list_free(pkg, PKG_CATEGORIES);
	pkg_list_free(pkg, PKG_LICENSES);

	LL_FREE(pkg->message, pkg_message_free);
	LL_FREE(pkg->annotations, pkg_kv_free);

	if (pkg->rootfd != -1)
		close(pkg->rootfd);

	free(pkg);
}
예제 #18
0
파일: query.c 프로젝트: HardenedBSD/pkg
int64_t
pkg_repo_binary_stat(struct pkg_repo *repo, pkg_stats_t type)
{
	sqlite3 *sqlite = PRIV_GET(repo);
	sqlite3_stmt	*stmt = NULL;
	int64_t		 stats = 0;
	UT_string	*sql = NULL;
	int		 ret;

	utstring_new(sql);

	switch(type) {
	case PKG_STATS_LOCAL_COUNT:
		goto out;
		break;
	case PKG_STATS_LOCAL_SIZE:
		goto out;
		break;
	case PKG_STATS_REMOTE_UNIQUE:
		utstring_printf(sql, "SELECT COUNT(id) FROM main.packages;");
		break;
	case PKG_STATS_REMOTE_COUNT:
		utstring_printf(sql, "SELECT COUNT(id) FROM main.packages;");
		break;
	case PKG_STATS_REMOTE_SIZE:
		utstring_printf(sql, "SELECT SUM(pkgsize) FROM main.packages;");
		break;
	case PKG_STATS_REMOTE_REPOS:
		goto out;
		break;
	}

	pkg_debug(4, "binary_repo: running '%s'", utstring_body(sql));
	ret = sqlite3_prepare_v2(sqlite, utstring_body(sql), -1, &stmt, NULL);
	if (ret != SQLITE_OK) {
		ERROR_SQLITE(sqlite, utstring_body(sql));
		goto out;
	}

	while (sqlite3_step(stmt) != SQLITE_DONE) {
		stats = sqlite3_column_int64(stmt, 0);
	}

out:
	utstring_free(sql);
	if (stmt != NULL)
		sqlite3_finalize(stmt);

	return (stats);
}
예제 #19
0
파일: prepared.c 프로젝트: dam2k/liborient
void cleanup_oh_prepared(OM_DEFINE_OBJECT(o_handler,oh)) {
	OM_MUTEX_LOCK(o_prepared, oprep);
	if (oh->connection_prepared == TRUE) {
		switch (oh->service_protocol) {
		case ORIENT_PROTO_BINARY:
			freeaddrinfo(oh->result);
			break;
		case ORIENT_PROTO_HTTP:
			break;
		default:
			break;
		}
	}
	oh->connection_prepared = FALSE;
	oh->service_protocol = -1;
	utstring_free(oh->admin->username);
	utstring_free(oh->admin->password);
	OM_FREE(o_ac, oh->admin);
	utstring_free(oh->user->username);
	utstring_free(oh->user->password);
	OM_FREE(o_ac, oh->user);
	OM_MUTEX_UNLOCK(o_prepared, oprep);
}
예제 #20
0
/* return allocated UT_string */
static UT_string *JsonNode_getJSON_UT(JsonNode *node, String last)
{
    asize_t i, nPairs, nChilds;
    UT_string *buff;
    utstring_new(buff);
    if (buff == NULL) return NULL;

    if (!isNullorEmpty(node->m_name)) {
        utstring_printf(buff, "\"%s\":", node->m_name);
    }

    utstring_printf(buff, "%s\n",  JSON_IS_OBJ(node) ? "{" : "[");
    nPairs = JsonNode_getPairCount(node);
    nChilds = JsonNode_getChildCount(node);

    for (i=0; i < nPairs; i++ ) {
        JsonPair *pair = JsonNode_getPair(node, i);

        if (JSON_IS_ARRAY(node)) {
            utstring_printf(buff, "\"%s\"", pair->key);
        } else {
            utstring_printf(buff, "\"%s\":\"%s\"", pair->key, pair->value);
        }

        utstring_printf(buff, "%s\n", (i < nPairs -1 || nChilds > 0) ? "," : "");
    }

    for (i = 0; i < nChilds; i++) {
        JsonNode* child = JsonNode_getChild(node, i);
        UT_string *childJSON = JsonNode_getJSON_UT(child, i == nChilds - 1 ? "\n" : ",\n");
        if (childJSON != NULL) {
            utstring_concat(buff, childJSON);
            utstring_free(childJSON);
        }
    }

    utstring_printf(buff, "%s%s",  JSON_IS_OBJ(node) ? "}" : "]", last);
    return buff;
}
예제 #21
0
파일: sized.c 프로젝트: Tolchi/misc
int get_files(UT_array *files, UT_array *stats) {
  struct dirent *dent;
  char *name, *path;
  file_stat_t fsb;
  int rc=-1,i=0;
  DIR *d;

  UT_string *s;
  utstring_new(s);
  utarray_clear(files);
  utarray_clear(stats);

  if ( (d = opendir(cf.dir)) == NULL) {
    syslog(LOG_ERR,"failed to opendir [%s]: %s\n", cf.dir, strerror(errno));
    goto done;
  }

  while ( (dent = readdir(d)) != NULL) {
    if (dent->d_type != DT_REG) continue;
    if (dent->d_name[0] == '.') continue; /* skip dot files */
    // ok, fully qualify it and push it 
    utstring_clear(s);
    utstring_printf(s, "%s/%s", cf.dir, dent->d_name);
    path = utstring_body(s);
    if (stat(path,&fsb.sb) == -1) {
      syslog(LOG_ERR,"can't stat %s: %s", path, strerror(errno));
      continue;
    }
    fsb.file_idx = i++;
    utarray_push_back(files, &path);
    utarray_push_back(stats, &fsb);
  }
  rc = 0; // success 

 done:
  utstring_free(s);
  if (d) closedir(d);
  return rc;
}
long readSegmentFromFileSystem(
	const char* nid,unsigned long long csn,size_t leftByte, size_t segmentSize,void* buffer)
{
	size_t bytesRead=0; size_t fileNameSize;
	size_t bytesToRead;
	FILE *fp; //file pointer

	//variables to update cache_table
	UT_string* utnid;
	UT_string* filename; utstring_new(filename);

	calculate_filename( (char*)nid,csn,filename);
	if(CONET_DEBUG) 
		fprintf(stderr,"[cacheEngine.c:%d]Opening file \"%s\" \n",__LINE__, utstring_body(filename) );

	fp = fopen( utstring_body(filename), "rb");
	if (fp == NULL)
	{
		fprintf(stderr, "[cacheEngine.c:%d]Error: Failed to open the file %s; maybe the chunk does not exist\n\n", 
			__LINE__, filename);
		if (CONET_SEVERE_DEBUG) exit(-651);
		return ( (size_t) - 1);
	}

	//!!!Questo fa andare in buffer overflow, il primo parametro dovrebbe essere
	//buffer senza +leftByte
	//Go to the leftByte location
	int fseek_return = fseek(fp,leftByte,SEEK_SET);
	if(fseek_return!=0)
	{
		fprintf(stderr,"[cacheEngine.c:%d]Error reading file %s. fseek returned with %d (ferror() tests the error indicator for the stream pointed to by stream,  returning nonzero if it is set.)\n",
			__LINE__,utstring_body(filename),fseek_return );
		//if(CONET_SEVERE_DEBUG) exit(-652);
		
		return ( (size_t) -0);
	}
	
	if(feof(fp))
	{
		fprintf(stderr,"[cacheEngine.c:%d]Error reading file %s. feof(fp)=%d\n",
			__LINE__,utstring_body(filename), feof(fp));
		//if(CONET_SEVERE_DEBUG) exit(-644);
		
		return ( (size_t) -0);
	}

	clearerr(fp);

	bytesRead = fread(buffer, 1, segmentSize, fp);

	if (bytesRead <= 0)
	{
		fprintf(stderr,"%s\n",utstring_body(filename) );
		fprintf(
			stderr, 
			"[cacheEngine.c:%d] Error: in reading file %s. fread returned with %u. ferror=%d, feof(fp)=%d (ferror() tests the error indicator for the stream pointed to by stream,  returning nonzero if it is set.)\n\n",
			__LINE__,utstring_body(filename), bytesRead,ferror(fp), feof(fp)
		);

		if (CONET_SEVERE_DEBUG) exit(-652);
		return bytesRead;
	}

	fclose(fp);

	if(CONET_DEBUG)
		fprintf(stderr,"[cacheEngine.c:%d]Successfully read  %d bytes\n", 
			__LINE__,(int) bytesRead);

	utstring_free(filename);
	return bytesRead;
}
//size_t handleChunk(char* nid,char* csn,long tag,size_t left_edge, size_t segment_size,void* buffer)
size_t handleChunk(char* nid,unsigned long long csn,long tag,unsigned long long chunk_size,void* buffer)
{
	size_t writtenBytes,fileNameSize;
	char* fileName;
	char csnToDelete[64];
	FILE *fp;
	UT_string* folderName;
	utstring_new(folderName);
	UT_string* tempHexCsn;
	utstring_new(tempHexCsn);
	UT_string* filename; utstring_new(filename);


	//variables for cache table
	CacheEntry_t 	*entry, *tmp_entry; 
	CacheEntry_t	*entryToDelete=NULL;
	UT_string 		*utnid, *filenameToDelete;
	//	long 			csnlong;

	//folderName=NULL;
	if(CONET_DEBUG) fprintf(stderr,"\n[cacheEngine.c:%d]New segment nid=%s, csn=%llu received.\n",
		__LINE__,nid,csn);

	calculate_filename(nid,csn,filename);

	utstring_printf(folderName,CONTAINING_FOLDER);
	utstring_printf(folderName,nid);
	mkdir(utstring_body(folderName));
	fp = fopen(utstring_body(filename) , "w+b");

	if (fp == NULL) {
		fprintf(stderr, "Error: Failed to open the file %s for writing;\n\n", fileName);
		if (CONET_SEVERE_DEBUG) exit(-863);
		return ( (size_t) - 1);
	}

	writtenBytes = fwrite(buffer, 1,chunk_size, fp);
	fclose(fp);

	if (writtenBytes!=chunk_size) {
		fprintf(stderr, "Error in writing %s: writtenBytes=%d, while chunk_size=%d\n",
				fileName,writtenBytes,chunk_size);
		if (CONET_SEVERE_DEBUG) exit(-8763);
		return ( (size_t) - 1);
	}

	utstring_new(utnid); utstring_printf(utnid,nid);
	//utstring_printf(tempHexCsn,"0x");utstring_printf(tempHexCsn,csn);
	ADD_TO_CACHE_TABLE(cache_table,entry,tmp_entry,tag,utnid,csn,chunk_size,0,entryToDelete);
	
	//creates json message with nid and of type "stored", and store it to msg_to_controller_buffer;
	fill_message(tag, "stored", nid, csn, msg_to_controller_buffer);
	//BUFFERLEN is defined in c_json.h
	int bytesSentToController=send(socket_to_controller, msg_to_controller_buffer, BUFFERLEN, 0);
	if(CONET_DEBUG )
		fprintf(stderr,"[cacheEngine.c:%d]Message sent to controller = %s\n",__LINE__,msg_to_controller_buffer);
	if 	(bytesSentToController <=strlen(msg_to_controller_buffer) )
	{
        fprintf(stderr,"[cacheEngine.c: %d] bytes sent to controller=%d\n",__LINE__, bytesSentToController);
        if (CONET_SEVERE_DEBUG) exit(-982);
    }



	if (entryToDelete!=NULL)
	{
		sprintf(csnToDelete,"%x",(unsigned int) entryToDelete->csn);
		CALCULATE_UTSTRING_FILE_NAME(entryToDelete->nid,csnToDelete,filenameToDelete);

		if (remove(utstring_body(filenameToDelete) )==0)
			fprintf(stderr,"File %s deleted.\n",utstring_body(filenameToDelete));
		else
			fprintf(stderr,"Error deleting file %s,\n",utstring_body(filenameToDelete));

		//creates json message with nid and of type "stored", and store it to msg_to_controller_buffer;
		fill_message(entryToDelete->tag, "deleted", entryToDelete->nid, entryToDelete->csn, msg_to_controller_buffer);
		//BUFFERLEN is defined in c_json.h
		if 	(send(socket_to_controller, msg_to_controller_buffer, BUFFERLEN, 0) != 
				strlen(msg_to_controller_buffer)
			)
		    fprintf(stderr,"send() sent a different number of bytes than expected\n");
		
		
	}

	fprintf(stderr,"[cacheEngine.c:%d]New chunk added: nid=%s, csn=%llu, tag=%ld\n",__LINE__,nid,csn,tag);

	//    utstring_free(utnid); utstring_free(filenameToDelete);
	utstring_free(filename);

	return writtenBytes;
}
예제 #24
0
int main(void)
{
  UT_array * dictionary;
  UT_array * specialChar;
  
  UT_array * WholeString = NULL;
  CharProfile NewCharProfile;
  CharProbability NewCharProbability;
  
  /*utarray_new(dictionary, &ut_str_icd);
  utarray_new(specialChar, &ut_str_icd);*/
  dictionary = postProcessingInitializeDictionary();
  specialChar = postProcessingInitializeSpecialChar();

  
  /*initialize string profile*/
  UT_icd StringProfile_icd = {sizeof(CharProfile), NULL, NULL, CharProfile_free};
  utarray_new(WholeString, &StringProfile_icd);
  
  /*first char spot*/	
  UT_icd CharProbability_icd = {sizeof(CharProbability), NULL, NULL, NULL};
  utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
  /*first possibility*/
  NewCharProbability.Char = 'f';
  NewCharProbability.Probability = 25;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*second probability*/
  NewCharProbability.Char = 't';
  NewCharProbability.Probability = 75;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability); 
  /*push to the whole string*/
  utarray_push_back(WholeString, &NewCharProfile);
  
  /*second char spot*/	
  utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
  /*first possibility*/
  NewCharProbability.Char = 'o';
  NewCharProbability.Probability = 35;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*second probability*/
  NewCharProbability.Char = '0';
  NewCharProbability.Probability = 85;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*push to the whole string*/
  utarray_push_back(WholeString, &NewCharProfile);
  
  /*third char spot*/	
  utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
  /*first possibility*/
  NewCharProbability.Char = 't';
  NewCharProbability.Probability = 50;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*second probability*/
  NewCharProbability.Char = 'r';
  NewCharProbability.Probability = 100;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*push to the whole string*/
  utarray_push_back(WholeString, &NewCharProfile);
  
  /*third char spot*/	
  utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
  /*first possibility*/
  NewCharProbability.Char = '(';
  NewCharProbability.Probability = 50;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*second probability*/
  NewCharProbability.Char = '=';
  NewCharProbability.Probability = 100;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*push to the whole string*/
  utarray_push_back(WholeString, &NewCharProfile);
  
  /*fifth char spot*/	
  utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
  /*first possibility*/
  NewCharProbability.Char = 'm';
  NewCharProbability.Probability = 50;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*second probability*/
  NewCharProbability.Char = 'n';
  NewCharProbability.Probability = 100;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*push to the whole string*/
  utarray_push_back(WholeString, &NewCharProfile);
  
  /*sixth char spot*/	
  utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
  /*first possibility*/
  NewCharProbability.Char = 'y';
  NewCharProbability.Probability = 50;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*second probability*/
  NewCharProbability.Char = 't';
  NewCharProbability.Probability = 100;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*push to the whole string*/
  utarray_push_back(WholeString, &NewCharProfile);
  
  /*seventh char spot*/	
  utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
  /*first possibility*/
  NewCharProbability.Char = ' ';
  NewCharProbability.Probability = 50;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*second probability*/
  NewCharProbability.Char = 'u';
  NewCharProbability.Probability = 100;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*push to the whole string*/
  utarray_push_back(WholeString, &NewCharProfile);
  
  /*eigth char spot*/	
  utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
  /*first possibility*/
  NewCharProbability.Char = 'h';
  NewCharProbability.Probability = 50;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*second probability*/
  NewCharProbability.Char = 'w';
  NewCharProbability.Probability = 100;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*push to the whole string*/
  utarray_push_back(WholeString, &NewCharProfile);
  
  /*ninth char spot*/	
  utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
  /*first possibility*/
  NewCharProbability.Char = ')';
  NewCharProbability.Probability = 50;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*second probability*/
  NewCharProbability.Char = '}';
  NewCharProbability.Probability = 100;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*push to the whole string*/
  utarray_push_back(WholeString, &NewCharProfile);
  
  /*tenth char spot*/	
  utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
  /*first possibility*/
  NewCharProbability.Char = '\n';
  NewCharProbability.Probability = 50;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*second probability*/
  NewCharProbability.Char = 'u';
  NewCharProbability.Probability = 100;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*push to the whole string*/
  utarray_push_back(WholeString, &NewCharProfile);
  
  /*sixth char spot*/	
  utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
  /*first possibility*/
  NewCharProbability.Char = '{';
  NewCharProbability.Probability = 50;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*second probability*/
  NewCharProbability.Char = '[';
  NewCharProbability.Probability = 100;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*push to the whole string*/
  utarray_push_back(WholeString, &NewCharProfile);
  
  /*sixth char spot*/	
  utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
  /*first possibility*/
  NewCharProbability.Char = 'd';
  NewCharProbability.Probability = 50;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*second probability*/
  NewCharProbability.Char = 'a';
  NewCharProbability.Probability = 100;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*push to the whole string*/
  utarray_push_back(WholeString, &NewCharProfile);
  
  /*sixth char spot*/	
  utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
  /*first possibility*/
  NewCharProbability.Char = 'u';
  NewCharProbability.Probability = 50;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*second probability*/
  NewCharProbability.Char = 'd';
  NewCharProbability.Probability = 100;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*push to the whole string*/
  utarray_push_back(WholeString, &NewCharProfile);
  
  /*sixth char spot*/	
  utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
  /*first possibility*/
  NewCharProbability.Char = 't';
  NewCharProbability.Probability = 50;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*second probability*/
  NewCharProbability.Char = 'y';
  NewCharProbability.Probability = 100;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*push to the whole string*/
  utarray_push_back(WholeString, &NewCharProfile);
  
   /*sixth char spot*/	
  utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
  /*first possibility*/
  NewCharProbability.Char = 'o';
  NewCharProbability.Probability = 50;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*second probability*/
  NewCharProbability.Char = 't';
  NewCharProbability.Probability = 100;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*push to the whole string*/
  utarray_push_back(WholeString, &NewCharProfile);
  
   /*sixth char spot*/	
  utarray_new(NewCharProfile.CharChoices, &CharProbability_icd);
  /*first possibility*/
  NewCharProbability.Char = '}';
  NewCharProbability.Probability = 50;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*second probability*/
  NewCharProbability.Char = ']';
  NewCharProbability.Probability = 100;
  utarray_push_back(NewCharProfile.CharChoices, &NewCharProbability);
  /*push to the whole string*/
  utarray_push_back(WholeString, &NewCharProfile);
  
  
  
  UT_string * advance_string = postProcessingAdvance(WholeString,dictionary,specialChar);
  UT_string * normal_string = postProcessing(WholeString);
  
  printf("Result from highest probability catch is:\n%s\nResult from keyword match is:\n%s\n", utstring_body(normal_string), utstring_body(advance_string));
  
  utstring_free(advance_string);
  utstring_free(normal_string);
  

  postProcessingCleanUP(dictionary, specialChar);
  return 0;
}
예제 #25
0
int main() {
    UT_string *s,*t;
    char V_TestStr[] = "There are two needle\0s in this \0haystack with needle\0s.";
    char V_NeedleStr[] = "needle\0s";
    long *V_KMP_Table;
    long V_FindPos;
    size_t V_StartPos;
    size_t V_FindCnt;


    utstring_new(s);
    utstring_new(t);

    utstring_bincpy(s, V_TestStr, sizeof(V_TestStr)-1);
    printf("\"%s\" len=%u\n", utstring_body(s), utstring_len(s));
    utstring_bincpy(t, V_NeedleStr, sizeof(V_NeedleStr)-1);
    printf("\"%s\" len=%u\n", utstring_body(t), utstring_len(t));

    V_KMP_Table = (long *)malloc(sizeof(long) * (utstring_len(t) + 1));
    if (V_KMP_Table != NULL)
    {
        _utstring_BuildTableR(utstring_body(t), utstring_len(t), V_KMP_Table);

        V_FindCnt = 0;
        V_FindPos = 0;
        V_StartPos = utstring_len(s) - 1;
        do
        {
            V_FindPos = _utstring_findR(utstring_body(s), 
                                        V_StartPos + 1, 
                                        utstring_body(t), 
                                        utstring_len(t), 
                                        V_KMP_Table);
            if (V_FindPos >= 0)
            {
                V_FindCnt++;
                V_StartPos = V_FindPos - 1;
            }
            printf("utstring_find()=%ld\n", V_FindPos);
        } while (V_FindPos >= 0);
        printf("FindCnt=%u\n", V_FindCnt);

        free(V_KMP_Table);
    }
    else
    {
        printf("malloc() failed...\n");
    }

    utstring_free(t);
    utstring_clear(s);
    utstring_printf(s,"ABC ABCDAB ABCDABCDABDE");
    int o;

    o=utstring_find(  s, -9, "ABC", 3 ) ; printf("expect 15 %d\n",o);
    o=utstring_find(  s,  3, "ABC", 3 ) ; printf("expect  4 %d\n",o);
    o=utstring_find(  s, 16, "ABC", 3 ) ; printf("expect -1 %d\n",o);
    o=utstring_findR( s, -9, "ABC", 3 ) ; printf("expect 11 %d\n",o);
    o=utstring_findR( s, 12, "ABC", 3 ) ; printf("expect  4 %d\n",o);
    o=utstring_findR( s, 13, "ABC", 3 ) ; printf("expect 11 %d\n",o);
    o=utstring_findR( s,  2, "ABC", 3 ) ; printf("expect  0 %d\n",o);
    


    utstring_free(s);

    return 0;
}
예제 #26
0
파일: app.c 프로젝트: stoni/luarest
static luarest_status parse_apps(application** apps, char* directory_path)
{
#ifdef WIN32
	WIN32_FIND_DATA ffd;
	WIN32_FIND_DATA fap;
	size_t length_of_arg;
	TCHAR szDir[MAX_PATH];
	TCHAR appFile[MAX_PATH];
	HANDLE hFind = INVALID_HANDLE_VALUE;
	HANDLE hApp = INVALID_HANDLE_VALUE;
	luarest_status ret;
	// Check that the input path plus 2 is not longer than MAX_PATH.
	StringCchLength(directory_path, MAX_PATH, &length_of_arg);
	if (length_of_arg > (MAX_PATH - 2)) {
		_tprintf(TEXT("\nDirectory path is too long.\n"));
		return(LUAREST_ERROR);
	}
	// Prepare string for use with FindFile functions.  First, copy the
	// string to a buffer, then append '\*' to the directory name.
	StringCchCopy(szDir, MAX_PATH, directory_path);
	StringCchCat(szDir, MAX_PATH, TEXT("\\*"));
	// Find the first file in the directory.
	hFind = FindFirstFile(szDir, &ffd);
	if (INVALID_HANDLE_VALUE == hFind) {
      return(LUAREST_ERROR);
	}
	// List all the files in the directory with some info about them.
	do {
		if (ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
			StringCchCopy(appFile, MAX_PATH, directory_path);
			StringCchCat(appFile, MAX_PATH, TEXT("\\"));
			StringCchCat(appFile, MAX_PATH, ffd.cFileName);
			StringCchCat(appFile, MAX_PATH, TEXT("\\"));
			StringCchCat(appFile, MAX_PATH, APP_ENTRY_POINT);
			hApp = FindFirstFile(appFile, &fap);
			if (hApp != INVALID_HANDLE_VALUE) {
				UT_string* app;
				utstring_new(app);
				utstring_printf(app, appFile);
				/* verify application */
				ret = verify_application(apps, ffd.cFileName, app);
				utstring_free(app);
				if (ret != LUAREST_SUCCESS) {
					printf("Application %s couldn't be load due to errors!\n", ffd.cFileName);
				}
			}
			FindClose(hApp);
		}
	}
	while (FindNextFile(hFind, &ffd) != 0);
	FindClose(hFind);
#else
    DIR *dpdf;
    struct dirent *epdf;
    struct stat st;
    int is_dir = 0;
    dpdf = opendir("./");
    if (dpdf != NULL){
        while ( (epdf = readdir(dpdf)) != NULL) {
            if (stat(epdf->d_name, &st) == -1) {
                continue;
            }
            is_dir = (st.st_mode & S_IFDIR) != 0;
            if (is_dir) {
                continue;
            }
            /* re-do for unix */
        }
    }
    closedir(dpdf);
#endif
	return(LUAREST_SUCCESS);
}
예제 #27
0
int main(int argc, char *argv[]) {
    
#ifndef _OPENMP
    fprintf(stderr, "\nERROR: Program built with compiler lacking OpenMP support.\n");
    fprintf(stderr, "See SEAStAR README file for information about suitable compilers.\n");
    exit(EXIT_FAILURE);
#endif
    
    ///////////////////////////
    // Variable declarations
    ///////////////////////////
    
    // Input filenames
    UT_string *in_read1_fq_fn, *in_read2_fq_fn, *in_single1_fq_fn, *in_single2_fq_fn;
    utstring_new(in_read1_fq_fn);
    utstring_new(in_read2_fq_fn);
    utstring_new(in_single1_fq_fn);
    utstring_new(in_single2_fq_fn);
    
    // Output filenames
    UT_string *out_read1_fn, *out_read2_fn, *out_single1_fn, *out_single2_fn, *out_mates_fn, *out_filetype;
    utstring_new(out_filetype);
    utstring_new(out_read1_fn);
    utstring_new(out_read2_fn);
    utstring_new(out_single1_fn);
    utstring_new(out_single2_fn);
    utstring_new(out_mates_fn);
    
    // Read name prefix
    UT_string *out_read_prefix;
    utstring_new(out_read_prefix);
    
    // Flags
    int singles_flag = 0;   // 1 when two output singles files being written
    int num_input_singles_files = 0;
    
    // Read counters
    unsigned long int mp_org = 0, R1_org = 0, R2_org = 0, singlet1_org = 0, singlet2_org = 0;
    unsigned long int mp_cnt = 0, R1_cnt = 0, R2_cnt = 0, singlet1_cnt = 0, singlet2_cnt = 0, s1_cnt = 0, s2_cnt = 0;
    unsigned long int comp_r1 = 0, comp_r2 = 0, comp_s1 = 0, comp_s2 = 0;
    unsigned long int read1_singlet_cnt = 0, read2_singlet_cnt = 0;

    ////////////////////////////////////////////////////////////////////////
    // All done with variable declarations!!
    
    ///////////////////////////////////
    // Command line argtable settings
    ///////////////////////////////////
    
    struct arg_lit *gzip = arg_lit0("z", "gzip", "Output converted files in gzip compressed format. [NULL]");
    struct arg_lit *inv_singles = arg_lit0("v", "invert_singles", "Causes singles output to be the inverse of the input. 2->1 or 1->2 [NULL]");
    struct arg_lit *num_singles = arg_lit0("s", "singles", "Write two singlet files, one for each mate-paired input file. [NULL]");
    struct arg_rem *sing_rem = arg_rem(NULL, "Note! -v is only valid when there are input singlet reads. -s is only valid when there are NO input singlet reads.");    
    struct arg_str *pre_read_id = arg_str0(NULL, "prefix", "<string>", "Prefix to add to read identifiers. [out_prefix]");
    struct arg_lit *no_pre = arg_lit0(NULL, "no_prefix", "Do not change the read names in any way. [NULL]");
    struct arg_lit *pre_read_len = arg_lit0(NULL, "add_len", "Add the final trimmed length value to the read id prefix. [length not added]");
    struct arg_dbl *prob = arg_dbl0("p","correct_prob","<d>","Probability that output reads are correct. 0.0 disables quality trimming. [0.5]");
    struct arg_int *fixed_len = arg_int0("f","fixed_len","<u>","Trim all reads to a fixed length, still filtering on quality [no fixed length]");
    struct arg_int *len = arg_int0("l","min_read_len","<u>","Minimum length of a singlet or longest-mate in nucleotides [24]");
    struct arg_int *mate_len = arg_int0("m","min_mate_len","<u>","Minimum length of the shortest mate in nucleotides [min_read_len]");
    struct arg_dbl *entropy = arg_dbl0("e","entropy_filter","<d>","Remove reads with per position information below given value (in bits per dinucleotide) [No filter]");
    struct arg_lit *entropy_strict = arg_lit0(NULL, "entropy_strict", "Reject reads for low entropy overall, not just the retained part after trimming [NULL]");
    struct arg_lit *mates = arg_lit0(NULL, "mates_file", "Produce a Velvet compatible interleaved paired read output file (e.g. <out_prefix>_mates.fastq). [NULL]");
    struct arg_lit *no_rev = arg_lit0(NULL, "no_rev", "By default, the second read in each pair is reversed for colorspace --mate-file output. --no_rev disables reversing. [rev]");
    struct arg_lit *only_mates = arg_lit0(NULL, "only_mates", "Supress writing .read1 and .read2 outputs. Requires --mates_file. [NULL]");
    struct arg_lit *fasta = arg_lit0(NULL, "fasta", "Write FASTA format files instead of FASTQ for all outputs (e.g. <out_prefix>.<read_type>.fasta). [FASTQ]");
    struct arg_file *input = arg_file1(NULL, NULL, "<in_prefix>", "Input file prefix: (e.g. <in_prefix>_single.fastq [<in_prefix>_read1.fastq <in_prefix>_read2.fastq]) ");
    struct arg_file *output = arg_file1(NULL, NULL, "<out_prefix>", "Output file prefix: (e.g. <out_prefix>_single.fastq [<out_prefix>_read1.fastq <out_prefix>_read2.fastq]) ");
    struct arg_lit *version = arg_lit0(NULL,"version","Print the build version and exit."); 
    struct arg_lit *h = arg_lit0("h", "help", "Request help.");
    struct arg_end *end = arg_end(20);
    
    void *argtable[] = {h,version,gzip,inv_singles,num_singles,sing_rem,prob,len,mate_len,fixed_len,pre_read_id,pre_read_len,no_pre,entropy,entropy_strict,mates,no_rev,only_mates,fasta,input,output,end};
    int arg_errors = 0;
        
    ////////////////////////////////////////////////////////////////////////
    // Handle command line processing (via argtable2 library) 
    ////////////////////////////////////////////////////////////////////////
    
    arg_errors = arg_parse(argc, argv, argtable);

	if (version->count) {
		fprintf(stderr, "%s version: %s\n", argv[0], SS_BUILD_VERSION);
		exit(EXIT_SUCCESS);
    }    
	
    if (h->count) {
        fprintf(stderr,"\ntrim_fastq is a utility for performing quality and information-based\n");
        fprintf(stderr,"trimming on paired or unpaired, nucleotide or SOLiD colorspace reads. \n\n");
        arg_print_syntaxv(stderr, argtable, "\n\n");
        arg_print_glossary(stderr, argtable, "%-25s %s\n");
        fprintf(stderr, "\nInput and output \"prefixes\" are the part of the filename before:\n");
        fprintf(stderr, "_single.fastq [_read1.fastq _read2.fastq]  A singlets (single) file\n");
        fprintf(stderr, "is required.  Mate-paired read files are automatically used if present.\n");
        fprintf(stderr, "Multiple output files only produced for mate-paired inputs.\n");
        fprintf(stderr, "\nNote! Input and output files may be gzipped, and outputs can be written\n");
        fprintf(stderr, "as either FASTQ or FASTA format files.\n");
        
        exit(EXIT_FAILURE);
    }    
    
    if (arg_errors) { 
        arg_print_errors(stderr, end, "trimfastq");
        arg_print_syntaxv(stderr, argtable, "\n");
        exit(EXIT_FAILURE);
    }
    
    // Validate entropy
    if (entropy->count) {
        entropy_cutoff = entropy->dval[0];
        if ((entropy_cutoff < 0.0) || (entropy_cutoff > 4.0)) {
            fprintf(stderr, "entropy_filter must be [0.0 - 4.0] \n");
            exit(EXIT_FAILURE);
        }
        strict_ent = entropy_strict->count;
    } else {
        if (entropy_strict->count) {
            fprintf(stderr, "Error: --entropy_strict requires --entropy_filter.\n");
            exit(EXIT_FAILURE);
        } 
        entropy_cutoff = -1.0;
    }    
    
    // Validate error_prob
    if (prob->count) {
        err_prob = prob->dval[0];
        if ((err_prob < 0.0) || (err_prob > 1.0)) {
            fprintf(stderr, "--correct_prob (-p) must be 0.0 - 1.0 inclusive\n");
            exit(EXIT_FAILURE);
        }
    } else {
        err_prob = 0.5;
    }    

    // Validate min read len
    if (len->count) {
        min_len = len->ival[0];
        if (min_len <= 0) {
            fprintf(stderr, "min_read_len must be > 0\n");
            exit(EXIT_FAILURE);
        }        
    } else {
        min_len = 24;
    }

    // Validate min mate len
    if (mate_len->count) {
        min_mate_len = mate_len->ival[0];
        if (min_mate_len <= 0) {
            fprintf(stderr, "min_mate_len must be > 0\n");
            exit(EXIT_FAILURE);
        }        
        if (min_mate_len > min_len) {
            fprintf(stderr, "min_mate_len must be <= min_len\n");
            exit(EXIT_FAILURE);
        }        
    } else {
        min_mate_len = min_len;
    }
    
    if (fixed_len->count) {
        fix_len = min_mate_len = min_len = fixed_len->ival[0];
        if ((mate_len->count) || (len->count)) {
            fprintf(stderr, "fixed_len cannot be used with min_read_len or min_mate_len\n");
            exit(EXIT_FAILURE);
        }
        if (fix_len <= 0) {
            fprintf(stderr, "fixed_len must be > 0\n");
            exit(EXIT_FAILURE);
        }
    } else {
        fix_len = 0;
    }
    
    if (pre_read_id->count) {
        
        if (no_pre->count) {
            fprintf(stderr, "Error: Both --prefix and --no_prefix were specified.\n");
            exit(EXIT_FAILURE);
        }
        
        if (! strlen(pre_read_id->sval[0])) {
            fprintf(stderr, "Read ID prefix may not be zero length.\n");
            exit(EXIT_FAILURE);
        } 
        
        if (strchr(pre_read_id->sval[0], ':') || strchr(pre_read_id->sval[0], '|') || strchr(pre_read_id->sval[0], '+') || strchr(pre_read_id->sval[0], '/')) {
            fprintf(stderr, "Read ID prefix '%s' may not contain the characters ':', '|', '+' or '/'.\n", pre_read_id->sval[0]);
            exit(EXIT_FAILURE);
        }
        
        // Build default read ID prefix
        ss_strcat_utstring(out_read_prefix, pre_read_id->sval[0]);
        
    } else {

        if (!no_pre->count) {
            
            if (strchr(output->filename[0], ':') || strchr(output->filename[0], '|') || strchr(output->filename[0], '+') || strchr(output->filename[0], '/')) {
                fprintf(stderr, "Read ID prefix '%s' (from output prefix) may not contain the characters ':', '|', '+' or '/'.\n", output->filename[0]);
                fprintf(stderr, "Hint: Use the --prefix parameter if the output file prefix contains path information.\n");
                exit(EXIT_FAILURE);
            }  
            
            // Build default read ID prefix
            ss_strcat_utstring(out_read_prefix, output->filename[0]);
        }
    }
    
    if ((only_mates->count) && (!mates->count)) {
        fprintf(stderr, "--only_mates requires --mates.\n");
        exit(EXIT_FAILURE);
    }

    if ((no_rev->count) && (!mates->count)) {
        fprintf(stderr, "--no_rev requires --mates.\n");
        exit(EXIT_FAILURE);
    }
    
    // Check for null string prefixes
    if (!(strlen(input->filename[0]) && strlen(output->filename[0]))) {
        fprintf(stderr, "Error: NULL prefix strings are not permitted.\n");
        exit(EXIT_FAILURE);
    }
    
    // Construct input filenames
    utstring_printf(in_read1_fq_fn, "%s.read1.fastq", input->filename[0]);
    utstring_printf(in_read2_fq_fn, "%s.read2.fastq", input->filename[0]);
    
    utstring_printf(in_single1_fq_fn, "%s.single.fastq", input->filename[0]);
    
    FILE *in_read_file = NULL;
    
    num_input_singles_files = 1;
    
    // Try to open a singlet fastq file
    // Check singlet output options -s and -v
    // Set input singlet names to
    //   - *.single.fastq or
    //   - *.single1.fastq and *.single2.fastq
    if (!(in_read_file = ss_get_gzFile(utstring_body(in_single1_fq_fn), "r"))) {
        utstring_clear(in_single1_fq_fn);
        utstring_printf(in_single1_fq_fn, "%s.single1.fastq", input->filename[0]);
        utstring_printf(in_single2_fq_fn, "%s.single2.fastq", input->filename[0]);
        num_input_singles_files = 2;
        
        if ((in_read_file = ss_get_gzFile(utstring_body(in_single1_fq_fn), "r")) || (in_read_file = ss_get_gzFile(utstring_body(in_single2_fq_fn), "r"))) {
            singles_flag = 1;   // Two singlet outputs
        } else {
            singles_flag = num_singles->count;  // Number of singlet outputs set by -s parm
            if (inv_singles->count) {
                fprintf(stderr, "Error: Invalid option -v, No input singlet file(s) found. Use -s to select multiple output singlet files.\n");
                exit(EXIT_FAILURE);
            }
        }
    }
    
    if (in_read_file) {
        gzclose(in_read_file);
        if (num_singles->count) {
            fprintf(stderr, "Error: Invalid option -s, Input singlet file(s) found, use -v to change the number of output singlet files.\n");
            exit(EXIT_FAILURE);
        }
    }

    // singles->count inverts the current singles file input scheme
    singles_flag = (singles_flag ^ inv_singles->count);
    
    // Check if input fastq is colorspace
    // If some files are colorspace and some are basespace, throw an error
    int fcount = 0;
    int cscount = 0;
    fcount += ss_is_fastq(utstring_body(in_read1_fq_fn));
    fcount += ss_is_fastq(utstring_body(in_read2_fq_fn));
    fcount += ss_is_fastq(utstring_body(in_single1_fq_fn));
    fcount += ss_is_fastq(utstring_body(in_single2_fq_fn));
    cscount += (ss_is_fastq(utstring_body(in_read1_fq_fn)) && ss_is_colorspace_fastq(utstring_body(in_read1_fq_fn)));
    cscount += (ss_is_fastq(utstring_body(in_read2_fq_fn)) && ss_is_colorspace_fastq(utstring_body(in_read2_fq_fn)));
    cscount += (ss_is_fastq(utstring_body(in_single1_fq_fn)) && ss_is_colorspace_fastq(utstring_body(in_single1_fq_fn)));
    cscount += (ss_is_fastq(utstring_body(in_single2_fq_fn)) && ss_is_colorspace_fastq(utstring_body(in_single2_fq_fn)));
    
    if (cscount && (cscount != fcount)) {        
        printf("Error: Mixed colorspace and basespace FASTQ files detected\n");
        exit(EXIT_FAILURE);
    }
    colorspace_flag = cscount ? 1 : 0;
    
    // Output filenames
    
    if (fasta->count) {
        ss_strcat_utstring(out_filetype, "fasta");
        read_count_divisor = 2;
    } else {
        ss_strcat_utstring(out_filetype, "fastq");
        read_count_divisor = 4;
    }
    
    if (!only_mates->count) {
        utstring_printf(out_read1_fn, "%s.read1.%s", output->filename[0], utstring_body(out_filetype));
        utstring_printf(out_read2_fn, "%s.read2.%s", output->filename[0], utstring_body(out_filetype));
    }
    
    if (singles_flag == 1) {
        utstring_printf(out_single1_fn, "%s.single1.%s", output->filename[0], utstring_body(out_filetype));
        utstring_printf(out_single2_fn, "%s.single2.%s", output->filename[0], utstring_body(out_filetype));
    } else {
        utstring_printf(out_single1_fn, "%s.single.%s", output->filename[0], utstring_body(out_filetype));
    }

    if (mates->count) {
        utstring_printf(out_mates_fn, "%s.mates.%s", output->filename[0], utstring_body(out_filetype));
    }
    ////////////////////////////////////////////////////////////////////////////////////////////////
    // Begin processing!
    
#ifdef _OPENMP    
    omp_set_num_threads(10);
#endif    
    
    // This is the value of a non-valid pipe descriptor    
#define NO_PIPE 0
    
    int r1_pipe[2];
    int r2_pipe[2];
    int s1_pipe[2];
    int s2_pipe[2];
    pipe(r1_pipe);
    pipe(r2_pipe);
    pipe(s1_pipe);
    pipe(s2_pipe);
    
    int r1_out_pipe[2];
    int r2_out_pipe[2];
    int mates_out_pipe[2];
    int s1_out_pipe[2];
    int s2_out_pipe[2];
    pipe(r1_out_pipe);
    pipe(r2_out_pipe);
    pipe(mates_out_pipe);
    pipe(s1_out_pipe);
    pipe(s2_out_pipe);
    
    
#pragma omp parallel sections default(shared)       
    {
        
#pragma omp section 
        {   // Read1 reader
            fq_stream_trimmer(in_read1_fq_fn, r1_pipe[1], out_read_prefix, no_pre->count, pre_read_len->count, &comp_r1, &R1_org, '\0', fasta->count);
        }
        
#pragma omp section 
        {   // Read1 writer
            R1_cnt = ss_stream_writer(out_read1_fn, r1_out_pipe[0], gzip->count) / read_count_divisor;
        }
        
#pragma omp section 
        {   // Read2 reader
            fq_stream_trimmer(in_read2_fq_fn, r2_pipe[1], out_read_prefix, no_pre->count, pre_read_len->count, &comp_r2, &R2_org, '\0', fasta->count);
        }
        
#pragma omp section 
        {   // Read2 writer
            R2_cnt = ss_stream_writer(out_read2_fn, r2_out_pipe[0], gzip->count) / read_count_divisor;
        }
        
#pragma omp section 
        {   // Single1 reader
            
            // When there is only one input singles file, but two output singles files, then supply which mate to use for this stream in the split parameter
            if ((singles_flag) && (num_input_singles_files == 1)) {
                singlet1_cnt = fq_stream_trimmer(in_single1_fq_fn, s1_pipe[1], out_read_prefix, no_pre->count, pre_read_len->count, &comp_s1, &singlet1_org, '1', fasta->count);
            } else {
                singlet1_cnt = fq_stream_trimmer(in_single1_fq_fn, s1_pipe[1], out_read_prefix, no_pre->count, pre_read_len->count, &comp_s1, &singlet1_org, '\0', fasta->count);
            }
        }
        
#pragma omp section 
        {   // Single1 writer
            s1_cnt = ss_stream_writer(out_single1_fn, s1_out_pipe[0], gzip->count) / read_count_divisor;
        }
        
#pragma omp section 
        {   // Single2 reader
            
            // When there is only one input singles file, but two output singles files, then supply which mate to use for this stream in the split parameter
            if ((singles_flag) && (num_input_singles_files == 1)) {
                singlet2_cnt = fq_stream_trimmer(in_single1_fq_fn, s2_pipe[1], out_read_prefix, no_pre->count, pre_read_len->count, &comp_s2, &singlet2_org, '2', fasta->count);
            } else {
                singlet2_cnt = fq_stream_trimmer(in_single2_fq_fn, s2_pipe[1], out_read_prefix, no_pre->count, pre_read_len->count, &comp_s2, &singlet2_org, '\0', fasta->count);
            }
        }
        
#pragma omp section 
        {   // Single2 writer
            s2_cnt = ss_stream_writer(out_single2_fn, s2_out_pipe[0], gzip->count) / read_count_divisor;
        }

#pragma omp section 
        {   // Velvet mates writer
            // Divide count by 2 because both R1 and R2 reads go through this writer
            mp_cnt = ss_stream_writer(out_mates_fn, mates_out_pipe[0], gzip->count)  / 2 / read_count_divisor;
        }
        
#pragma omp section 
        {   // Dispatcher

            
            // Allocate data buffer strings
            
            UT_string *r1_data;
            utstring_new(r1_data);
            UT_string *r2_data;
            utstring_new(r2_data);
            UT_string *s1_data;
            utstring_new(s1_data);
            UT_string *s2_data;
            utstring_new(s2_data);
            
            UT_string *rev_tmp;
            utstring_new(rev_tmp);
            
            UT_string *rev_data;
            utstring_new(rev_data);
            
            // Pipes
            FILE *r1_in = fdopen(r1_pipe[0],"r"); 
            FILE *r2_in = fdopen(r2_pipe[0],"r");                 
            
            FILE *s1_in = fdopen(s1_pipe[0],"r");
            FILE *s2_in = fdopen(s2_pipe[0],"r");             

            FILE *mates_out = fdopen(mates_out_pipe[1],"w"); 
            
            FILE *r1_out = fdopen(r1_out_pipe[1],"w"); 
            FILE *r2_out = fdopen(r2_out_pipe[1],"w");                 
            
            FILE *s1_out = fdopen(s1_out_pipe[1],"w");
            FILE *s2_out = fdopen(s2_out_pipe[1],"w");             
            
            if (!singles_flag) {
                fclose(s2_out);
                s2_out = s1_out;
            }
            
            // Flags for data left in single files
            int single1_hungry = 1;
            int single2_hungry = 1;
            
            // Handle read1 and read2 files
            while (ss_get_utstring(r1_in, r1_data)) {
                if (!ss_get_utstring(r2_in, r2_data)) {
                    fprintf(stderr, "Error: Input read1 and read2 files are not synced\n");
                    exit(EXIT_FAILURE);
                }
                if (keep_read(r1_data)) {
                    if (keep_read(r2_data)) {
                        // Output both read1 and read2
                        if (mates->count) {
                            if (only_mates->count) {
                                // Interleaved velvet output only
                                output_read(r1_data, NULL, NULL, r1_in, NULL, mates_out, fasta->count);
                                if (no_rev->count || !colorspace_flag) {
                                    output_read(r2_data, NULL, NULL, r2_in, NULL, mates_out, fasta->count);
                                } else {
                                    output_read(r2_data, rev_data, rev_tmp, r2_in, NULL, mates_out, fasta->count);
                                }
                            } else {
                                // Interleaved velvet output and normal read file output
                                output_read(r1_data, NULL, NULL, r1_in, r1_out, mates_out, fasta->count);
                                if (no_rev->count || !colorspace_flag) {
                                    output_read(r2_data, NULL, NULL, r2_in, r2_out, mates_out, fasta->count);
                                } else {
                                    output_read(r2_data, rev_data, rev_tmp, r2_in, r2_out, mates_out, fasta->count);
                                }
                            }
                        } else {
                            // No interleaved velvet output
                            output_read(r1_data, NULL, NULL, r1_in, r1_out, NULL, fasta->count);
                            output_read(r2_data, NULL, NULL, r2_in, r2_out, NULL, fasta->count);
                        }
                    } else {
                        // Discard read2, output read1 as singlet
                        output_read(r1_data, NULL, NULL, r1_in, s1_out, NULL, fasta->count);
                        read1_singlet_cnt++;
                    }
                } else {
                    if (keep_read(r2_data)) {
                        // Discard read1, output read2 as singlet
                        output_read(r2_data, NULL, NULL, r2_in, s2_out, NULL, fasta->count);
                        read2_singlet_cnt++;
                    }
                }
                
                // Process reads from singles here to take advantage of
                // parallelism
                if (single1_hungry || single2_hungry) {
                    if (single1_hungry) {
                        if (ss_get_utstring(s1_in, s1_data)) {
                            if (keep_read(s1_data)) {
                                output_read(s1_data, NULL, NULL, s1_in, s1_out, NULL, fasta->count);
                            }
                        } else {
                            single1_hungry = 0;
                        }
                    }
                    if (single2_hungry) {
                        if (ss_get_utstring(s2_in, s2_data)) {
                            if (keep_read(s2_data)) {
                                output_read(s2_data, NULL, NULL, s2_in, s2_out, NULL, fasta->count);
                            }
                        } else {
                            single2_hungry = 0;
                        }
                    }
                }
            }
            
            while (single1_hungry || single2_hungry) {
                if (single1_hungry) {
                    if (ss_get_utstring(s1_in, s1_data)) {
                        if (keep_read(s1_data)) {
                            output_read(s1_data, NULL, NULL, s1_in, s1_out, NULL, fasta->count);
                        }
                    } else {
                        single1_hungry = 0;
                    }
                }
                if (single2_hungry) {
                    if (ss_get_utstring(s2_in, s2_data)) {
                        if (keep_read(s2_data)) {
                            output_read(s2_data, NULL, NULL, s2_in, s2_out, NULL, fasta->count);
                        }
                    } else {
                        single2_hungry = 0;
                    }
                }
            }
            
            fclose(r1_in);
            fclose(r2_in);
            
            fclose(s1_in);
            fclose(s2_in);
            
            fclose(mates_out);
            
            fclose(r1_out);
            fclose(r2_out);
            
            fclose(s1_out);
            
            if (singles_flag) {
                fclose(s2_out);
            }
            
            // Free buffers
            utstring_free(r1_data);
            utstring_free(r2_data);
            utstring_free(s1_data);
            utstring_free(s2_data);
            utstring_free(rev_tmp);
            utstring_free(rev_data);
        }
    }

    if (!(R1_org+singlet1_org+singlet2_org)) {
    
        fprintf(stderr, "ERROR! No reads found in input files, or input(s) not found.\n");
        exit(EXIT_FAILURE);
    
    }
    
    if (R1_org != R2_org) {
        fprintf(stderr, "\nWarning! read1 and read2 fastq files did not contain an equal number of reads. %lu %lu\n", R1_org, R2_org);
    }
    
    if ((R1_org + R2_org) && !(singlet1_cnt + singlet2_cnt)) {
        fprintf(stderr, "\nWarning! read1/read2 files were processed, but no corresponding input singlets were found.\n");
    } 
    
    if (entropy->count) {
        printf("\nLow complexity reads discarded: Read1: %lu, Read2: %lu, Singlets: %lu %lu\n", comp_r1, comp_r2, comp_s1, comp_s2);
    }

    mp_org = R1_org;
    if (!only_mates->count) {
        mp_cnt = R1_cnt;
    }
    
    printf("\nMatepairs: Before: %lu, After: %lu\n", mp_org, mp_cnt);
    printf("Singlets: Before: %lu %lu After: %lu %lu\n", singlet1_org, singlet2_org, s1_cnt, s2_cnt);
    printf("Read1 singlets: %lu, Read2 singlets: %lu, Original singlets: %lu %lu\n", read1_singlet_cnt, read2_singlet_cnt, singlet1_cnt, singlet2_cnt);
    printf("Total Reads Processed: %lu, Reads retained: %lu\n", 2*mp_org+singlet1_org+singlet2_org, 2*mp_cnt+s1_cnt+s2_cnt);
    
    utstring_free(in_read1_fq_fn);
    utstring_free(in_read2_fq_fn);
    utstring_free(in_single1_fq_fn);
    utstring_free(in_single2_fq_fn);
    utstring_free(out_read1_fn);
    utstring_free(out_read2_fn);
    utstring_free(out_single1_fn);
    utstring_free(out_single2_fn);
    utstring_free(out_mates_fn);
    utstring_free(out_filetype);
    
    utstring_free(out_read_prefix);
    
    exit(EXIT_SUCCESS);
}
예제 #28
0
unsigned long int fq_stream_trimmer(UT_string *fq_fn, int pipe_fd, UT_string *out_prefix, int no_pre, int len_pre, unsigned long int *comp_cnt, unsigned long int *org, char split, int fmt_fasta) {
    
    UT_string *new_head_data;
    utstring_new(new_head_data);
    UT_string *head_data;
    utstring_new(head_data);
    UT_string *seq_data;
    utstring_new(seq_data);
    UT_string *extra_data;
    utstring_new(extra_data);
    UT_string *qual_data;
    utstring_new(qual_data);
    
    unsigned long int cnt = 0;
    
    char *start = NULL; 
    char *end = NULL;
    char *suffix = NULL;
    
    FILE *fq_file = NULL;

    FILE *pipe_in = fdopen(pipe_fd, "w");
    
    if (!(utstring_len(fq_fn))) {
        fclose(pipe_in);
        return(0);
    }
    
    // Try to open the fastq file
    if (!(fq_file = gzopen(utstring_body(fq_fn), "r"))) {
        utstring_printf(fq_fn, ".gz");
        if (!(fq_file = gzopen(utstring_body(fq_fn), "r"))) {
            fclose(pipe_in);
            return(0);
        }
    } 
    
    int x = 0;
    
    char head_char = '@';
    
    if (fmt_fasta) {
        head_char = '>';
    }
    
    while (ss_gzget_utstring(fq_file, head_data)) {
        
        ss_gzget_utstring(fq_file, seq_data);
        ss_gzget_utstring(fq_file, extra_data);
        ss_gzget_utstring(fq_file, qual_data);
        
        if (!split || ((suffix = strchr(utstring_body(head_data), '/')) && (suffix[1] == split))) {

            (*org)++;
            
            if ((x = trimmer(utstring_body(qual_data))) >= min_len) {  // Keep at least some of read
                
                // Reject read if complexity is too low
                if ((entropy_cutoff < 0.0) || (entropy_calc(utstring_body(seq_data), x) >= entropy_cutoff)) {
                    
                    // Truncate sequence
                    
                    ss_trim_utstring(seq_data, x);
                    ss_strcat_utstring(seq_data, "\n");
                    
                    if (!fmt_fasta) {
                        ss_trim_utstring(qual_data, x);
                        ss_strcat_utstring(qual_data, "\n");
                    }
                    
                    // Fixup the read name
                    utstring_clear(new_head_data);
                    
                    end = strchr(utstring_body(head_data), ':');
                    
                    if (no_pre) {
                        if ((start = strchr(utstring_body(head_data), '|'))) {
                            start++;
                        } else {
                            if (colorspace_flag) {
                                start = utstring_body(head_data) + 4;
                            } else {
                                start = utstring_body(head_data) + 1;
                            }
                        }
                        *end = '\0';
                    } else {
                        start = utstring_body(out_prefix);
                    }
                    
                    end++;
                    
                    if (colorspace_flag) {
                        if (len_pre) {
                            utstring_printf(new_head_data, "%c%.2s+%u|%s:%s",head_char,utstring_body(head_data)+1,x,start,end);
                        } else {
                            utstring_printf(new_head_data, "%c%.2s+%s:%s",head_char,utstring_body(head_data)+1,start,end);
                        }
                    } else {
                        if (len_pre) {
                            utstring_printf(new_head_data, "%c%u|%s:%s",head_char,x,start,end);
                        } else {
                            utstring_printf(new_head_data, "%c%s:%s",head_char,start,end);
                        }
                    }
                    
                    fputs(utstring_body(new_head_data), pipe_in);
                    fputs(utstring_body(seq_data), pipe_in);
                    
                    if (!fmt_fasta) {
                        fputs(utstring_body(extra_data), pipe_in);
                        fputs(utstring_body(qual_data), pipe_in);
                    }
                
                    cnt++;
                    
                } else {
                    // rejected by entropy filter
                    // Send along placeholder read to be discarded, keeping read1 and read2 in sync
                    // Empty fastq header is a read to be rejected by consumer threads
                    (*comp_cnt)++;
                    fputs("\n", pipe_in);
                }
            } else {
                // rejected by minimum length cutoff
                // Send along placeholder read to be discarded, keeping read1 and read2 in sync
                // Empty fastq header is a read to be rejected by consumer threads
                fputs("\n", pipe_in);
            }
        }
    }
    
    fclose(pipe_in);
    
    gzclose(fq_file);
    
    utstring_free(new_head_data);
    utstring_free(head_data);
    utstring_free(seq_data);
    utstring_free(extra_data);
    utstring_free(qual_data);

    return(cnt);
}
예제 #29
0
파일: rquery.c 프로젝트: Absolight/pkg
int
exec_rquery(int argc, char **argv)
{
	struct pkgdb		*db = NULL;
	struct pkgdb_it		*it = NULL;
	struct pkg		*pkg = NULL;
	char			*pkgname = NULL;
	int			 query_flags = PKG_LOAD_BASIC;
	match_t			 match = MATCH_EXACT;
	int			 ch;
	int			 ret = EPKG_OK;
	int			 retcode = EX_OK;
	int			 i;
	char			 multiline = 0;
	char			*condition = NULL;
	const char		*portsdir;
	UT_string		*sqlcond = NULL;
	const unsigned int	 q_flags_len = NELEM(accepted_rquery_flags);
	const char		*reponame = NULL;
	bool			 onematched = false;
	bool			 old_quiet;
	bool			 index_output = false;

	struct option longopts[] = {
		{ "all",		no_argument,		NULL,	'a' },
		{ "case-sensitive",	no_argument,		NULL,	'C' },
		{ "evaluate",		required_argument,	NULL,	'e' },
		{ "glob",		no_argument,		NULL,	'g' },
		{ "case-insensitive",	no_argument,		NULL,	'i' },
		{ "index-line",		no_argument,		NULL,	'I' },
		{ "repository",		required_argument,	NULL,	'r' },
		{ "no-repo-update",	no_argument,		NULL,	'U' },
		{ "regex",		no_argument,		NULL,	'x' },
		{ NULL,			0,			NULL,	0   },
	};

	portsdir = pkg_object_string(pkg_config_get("PORTSDIR"));

	while ((ch = getopt_long(argc, argv, "+aCgiIxe:r:U", longopts, NULL)) != -1) {
		switch (ch) {
		case 'a':
			match = MATCH_ALL;
			break;
		case 'C':
			pkgdb_set_case_sensitivity(true);
			break;
		case 'e':
			match = MATCH_CONDITION;
			condition = optarg;
			break;
		case 'g':
			match = MATCH_GLOB;
			break;
		case 'i':
			pkgdb_set_case_sensitivity(false);
			break;
		case 'I':
			index_output = true;
			break;
		case 'r':
			reponame = optarg;
			break;
		case 'U':
			auto_update = false;
			break;
		case 'x':
			match = MATCH_REGEX;
			break;
		default:
			usage_rquery();
			return (EX_USAGE);
		}
	}

	argc -= optind;
	argv += optind;

	if (argc == 0 && !index_output) {
		usage_rquery();
		return (EX_USAGE);
	}

	/* Default to all packages if no pkg provided */
	if (!index_output) {
		if (argc == 1 && condition == NULL && match == MATCH_EXACT) {
			match = MATCH_ALL;
		} else if (((argc == 1) ^ (match == MATCH_ALL )) && condition == NULL) {
			usage_rquery();
			return (EX_USAGE);
		}
	} else {
		if (argc == 0)
			match = MATCH_ALL;
	}

	if (!index_output && analyse_query_string(argv[0], accepted_rquery_flags, q_flags_len, &query_flags, &multiline) != EPKG_OK)
		return (EX_USAGE);

	if (condition != NULL) {
		utstring_new(sqlcond);
		if (format_sql_condition(condition, sqlcond, true) != EPKG_OK) {
			utstring_free(sqlcond);
			return (EX_USAGE);
		}
	}

	ret = pkgdb_access(PKGDB_MODE_READ, PKGDB_DB_REPO);
	if (ret == EPKG_ENOACCESS) {
		warnx("Insufficient privileges to query the package database");
		if (sqlcond != NULL)
			utstring_free(sqlcond);
		return (EX_NOPERM);
	} else if (ret != EPKG_OK) {
		if (sqlcond != NULL)
			utstring_free(sqlcond);
		return (EX_IOERR);
	}

	/* first update the remote repositories if needed */
	old_quiet = quiet;
	quiet = true;
	if (auto_update && (ret = pkgcli_update(false, false, reponame)) != EPKG_OK) {
		if (sqlcond != NULL)
			utstring_free(sqlcond);
		return (ret);
	}
	quiet = old_quiet;

	ret = pkgdb_open_all(&db, PKGDB_REMOTE, reponame);
	if (ret != EPKG_OK) {
		if (sqlcond != NULL)
			utstring_free(sqlcond);
		return (EX_IOERR);
	}
	drop_privileges();

	if (index_output)
		query_flags = PKG_LOAD_BASIC|PKG_LOAD_CATEGORIES|PKG_LOAD_DEPS;

	if (match == MATCH_ALL || match == MATCH_CONDITION) {
		const char *condition_sql = NULL;
		if (match == MATCH_CONDITION && sqlcond)
			condition_sql = utstring_body(sqlcond);
		if ((it = pkgdb_repo_query(db, condition_sql, match, reponame)) == NULL) {
			if (sqlcond != NULL)
				utstring_free(sqlcond);
			return (EX_IOERR);
		}

		while ((ret = pkgdb_it_next(it, &pkg, query_flags)) == EPKG_OK) {
			if (index_output)
				print_index(pkg, portsdir);
			else
				print_query(pkg, argv[0],  multiline);
		}

		if (ret != EPKG_END)
			retcode = EX_SOFTWARE;

		pkgdb_it_free(it);
	} else {
		for (i = (index_output ? 0 : 1); i < argc; i++) {
			pkgname = argv[i];

			if ((it = pkgdb_repo_query(db, pkgname, match, reponame)) == NULL) {
				if (sqlcond != NULL)
					utstring_free(sqlcond);
				return (EX_IOERR);
			}

			while ((ret = pkgdb_it_next(it, &pkg, query_flags)) == EPKG_OK) {
				onematched = true;
				if (index_output)
					print_index(pkg, portsdir);
				else
					print_query(pkg, argv[0], multiline);
			}

			if (ret != EPKG_END) {
				retcode = EX_SOFTWARE;
				break;
			}

			pkgdb_it_free(it);
		}
		if (!onematched && retcode == EX_OK)
			retcode = EX_UNAVAILABLE;
	}

	if (sqlcond != NULL)
		utstring_free(sqlcond);
	pkg_free(pkg);
	pkgdb_close(db);

	return (retcode);
}
예제 #30
0
파일: upgrade.c 프로젝트: Absolight/pkg
static void
check_vulnerable(struct pkg_audit *audit, struct pkgdb *db, int sock)
{
	struct pkgdb_it	*it = NULL;
	struct pkg		*pkg = NULL;
	kh_pkgs_t		*check = NULL;
	const char		*uid;
	UT_string		*sb;
	int				ret;
	FILE			*out;

	out = fdopen(sock, "w");
	if (out == NULL) {
		warn("unable to open stream");
		return;
	}

	if ((it = pkgdb_query(db, NULL, MATCH_ALL)) == NULL) {
		warnx("Error accessing the package database");
		pkg_audit_free(audit);
		fclose(out);
		return;
	}
	else {
		check = kh_init_pkgs();

		while ((ret = pkgdb_it_next(it, &pkg, PKG_LOAD_BASIC|PKG_LOAD_RDEPS))
				== EPKG_OK) {
			add_to_check(check, pkg);
			pkg = NULL;
		}

		ret = EX_OK;
	}

	if (db != NULL) {
		pkgdb_it_free(it);
		pkgdb_close(db);
	}

	if (ret != EX_OK) {
		pkg_audit_free(audit);
		kh_destroy_pkgs(check);
		fclose(out);
		return;
	}

	drop_privileges();

	if (pkg_audit_load(audit, NULL) != EPKG_OK) {
		warn("unable to open vulnxml file");
		fclose(out);
		pkg_audit_free(audit);
		return;
	}

#ifdef HAVE_CAPSICUM
	if (cap_enter() < 0 && errno != ENOSYS) {
		warn("cap_enter() failed");
		pkg_audit_free(audit);
		kh_destroy_pkgs(check);
		fclose(out);
		return;
	}
#endif

	if (pkg_audit_process(audit) == EPKG_OK) {
		kh_foreach_value(check, pkg, {
				if (pkg_audit_is_vulnerable(audit, pkg, true, &sb)) {
					pkg_get(pkg, PKG_UNIQUEID, &uid);
					fprintf(out, "%s\n", uid);
					fflush(out);
					utstring_free(sb);
				}
				pkg_free(pkg);
		});