/* Get the value of every header with the given name, separated by commas. If
you only want the first value for header fields that should not be
concatentated in this way, use http_header_get_first. The returned string
must be freed. */
char *http_header_get(const struct http_header *header, const char *name)
{
const struct http_header *p;
char *buf = NULL;
size_t size = 0, offset = 0;
int count;
count = 0;
for (p = header; p != NULL; p = p->next) {
/* RFC 2616, section 4.2: "Multiple message-header fields with the same
field-name MAY be present in a message if and only if the entire
field-value for that header field is defined as a comma-separated
list [i.e., #(values)]. It MUST be possible to combine the multiple
header fields into one "field-name: field-value" pair, without
changing the semantics of the message, by appending each subsequent
field-value to the first, each separated by a comma." */
if (field_name_equal(p->name, name)) {
if (count > 0)
strbuf_append_str(&buf, &size, &offset, ", ");
strbuf_append_str(&buf, &size, &offset, p->value);
count++;
}
}
return buf;
}
/* Send a 407 Proxy Authenticate Required response. */
static int send_proxy_authenticate(struct fdinfo *fdn, int stale)
{
char *buf = NULL;
size_t size = 0, offset = 0;
int n;
strbuf_append_str(&buf, &size, &offset, "HTTP/1.0 407 Proxy Authentication Required\r\n");
strbuf_append_str(&buf, &size, &offset, "Proxy-Authenticate: Basic realm=\"Ncat\"\r\n");
#if HAVE_HTTP_DIGEST
{
char *hdr;
hdr = http_digest_proxy_authenticate("Ncat", stale);
strbuf_sprintf(&buf, &size, &offset, "Proxy-Authenticate: %s\r\n", hdr);
free(hdr);
}
#endif
strbuf_append_str(&buf, &size, &offset, "\r\n");
if (o.debug > 1)
logdebug("RESPONSE:\n%s", buf);
n = send_string(fdn, buf);
free(buf);
return n;
}
void graph_info_make_intersect(const GraphInfo *ginfo, StrBuf *intersect_name)
{
if(intersect_name->end > 0) strbuf_append_char(intersect_name, ',');
strbuf_append_str(intersect_name, ginfo->sample_name.b);
if(ginfo->cleaning.is_graph_intersection) {
strbuf_append_char(intersect_name, ',');
strbuf_append_str(intersect_name, ginfo->cleaning.intersection_name.b);
}
}
static ladders_rec_t *
init_reqrec(lwan_request_t *request, lwan_response_t *response)
{
ladders_rec_t *rec = NULL;
static const char errmsg[] = "{\"error\":\"bad parameter\"}";
const char *start = lwan_request_get_query_param(request, "start");
const char *goal = lwan_request_get_query_param(request, "goal");
if (!start || !goal) {
strbuf_set_static(response->buffer, errmsg, sizeof(errmsg) - 1);
return NULL;
}
strbuf_append_printf(response->buffer,
"{\"start\":\"%s\",\"goal\":\"%s\",",
start, goal);
if (start[5] != 0 || strlen(start) < 5 ||
goal[5] != 0 || strlen(goal) < 5) {
strbuf_append_str(response->buffer,
"\"path\":[],\"error\":\"Paramter start and "
"goal should be five-letter words.\"}", 0);
return NULL;
}
rec = malloc(sizeof(*rec));
strcpy(rec->start, start);
strcpy(rec->goal, goal);
rec->response = response;
strbuf_init(&rec->path);
return rec;
}
static char *http_connect_request_auth(const union sockaddr_u *su, int *n,
struct http_challenge *challenge)
{
char *buf = NULL;
size_t size = 0, offset = 0;
append_connect_request_line(&buf, &size, &offset, su);
strbuf_append_str(&buf, &size, &offset, "Proxy-Authorization:");
if (challenge->scheme == AUTH_BASIC) {
char *auth_str;
auth_str = b64enc((unsigned char *) o.proxy_auth, strlen(o.proxy_auth));
strbuf_sprintf(&buf, &size, &offset, " Basic %s\r\n", auth_str);
free(auth_str);
#if HAVE_HTTP_DIGEST
} else if (challenge->scheme == AUTH_DIGEST) {
char *proxy_auth;
char *username, *password;
char *response_hdr;
/* Split up the proxy auth argument. */
proxy_auth = Strdup(o.proxy_auth);
username = strtok(proxy_auth, ":");
password = strtok(NULL, ":");
if (password == NULL) {
free(proxy_auth);
return NULL;
}
response_hdr = http_digest_proxy_authorization(challenge,
username, password, "CONNECT", sock_to_url(&httpconnect));
if (response_hdr == NULL) {
free(proxy_auth);
return NULL;
}
strbuf_append_str(&buf, &size, &offset, response_hdr);
free(proxy_auth);
free(response_hdr);
#endif
} else {
bye("Unknown authentication type.");
}
strbuf_append_str(&buf, &size, &offset, "\r\n");
*n = offset;
return buf;
}
static void *parse_section(struct parser *parser)
{
struct lexeme *lexeme;
size_t name_len;
if (!lexeme_buffer_consume(&parser->buffer, &lexeme))
return NULL;
strbuf_append_str(&parser->strbuf, lexeme->value.value, lexeme->value.len);
name_len = lexeme->value.len;
strbuf_append_char(&parser->strbuf, '\0');
while (lexeme_buffer_consume(&parser->buffer, &lexeme)) {
strbuf_append_str(&parser->strbuf, lexeme->value.value, lexeme->value.len);
if (parser->buffer.population >= 1)
strbuf_append_char(&parser->strbuf, ' ');
}
struct config_line line = {
.type = CONFIG_LINE_TYPE_SECTION,
.name = strbuf_get_buffer(&parser->strbuf),
.param = line.name + name_len + 1
};
if (!config_buffer_emit(&parser->items, &line))
return NULL;
return parse_config;
}
static void *parse_section_shorthand(struct parser *parser)
{
void *next_state = parse_section(parser);
if (next_state) {
struct config_line line = { .type = CONFIG_LINE_TYPE_SECTION_END };
if (!config_buffer_emit(&parser->items, &line))
return NULL;
return next_state;
}
return NULL;
}
static int append_quoted_string(char **buf, size_t *size, size_t *offset, const char *s)
{
const char *t;
strbuf_append_str(buf, size, offset, "\"");
for (;;) {
t = s;
while (!((*t >= 0 && *t <= 31) || *t == 127 || *t == '\\'))
t++;
strbuf_append(buf, size, offset, s, t - s);
if (*t == '\0')
break;
strbuf_sprintf(buf, size, offset, "\\%c", *t);
s = t + 1;
}
strbuf_append_str(buf, size, offset, "\"");
return *size;
}
/* See section 4.2 of RFC 2616 for header format. */
int http_parse_header(struct http_header **result, const char *header)
{
const char *p, *q;
size_t value_len, value_offset;
struct http_header *node, **prev;
*result = NULL;
prev = result;
p = header;
while (*p != '\0' && !is_crlf(p)) {
/* Get the field name. */
q = p;
while (*q != '\0' && is_token_char(*q))
q++;
if (*q != ':') {
http_header_free(*result);
return 400;
}
node = (struct http_header *) safe_malloc(sizeof(*node));
node->name = mkstr(p, q);
node->value = NULL;
node->next = NULL;
value_len = 0;
value_offset = 0;
/* Copy the header field value until we hit a CRLF. */
p = q + 1;
p = skip_lws(p);
for (;;) {
q = p;
while (*q != '\0' && !is_space_char(*q) && !is_crlf(q)) {
/* Section 2.2 of RFC 2616 disallows control characters. */
if (iscntrl((int) (unsigned char) *q)) {
http_header_node_free(node);
return 400;
}
q++;
}
strbuf_append(&node->value, &value_len, &value_offset, p, q - p);
p = skip_lws(q);
if (is_crlf(p))
break;
/* Replace LWS with a single space. */
strbuf_append_str(&node->value, &value_len, &value_offset, " ");
}
*prev = node;
prev = &node->next;
p = skip_crlf(p);
}
return 0;
}
static char *http_connect_request(const union sockaddr_u *su, int *n)
{
char *buf = NULL;
size_t size = 0, offset = 0;
append_connect_request_line(&buf, &size, &offset, su);
strbuf_append_str(&buf, &size, &offset, "\r\n");
*n = offset;
return buf;
}
static
char *make_description_item_multiline(const char *name, const char *content)
{
char *eol = strchr(content, '\n');
if (!eol)
return NULL;
struct strbuf *buf = strbuf_new();
strbuf_append_str(buf, name);
strbuf_append_str(buf, ":\n");
for (;;)
{
eol = strchrnul(content, '\n');
strbuf_append_strf(buf, ":%.*s\n", (int)(eol - content), content);
if (*eol == '\0' || eol[1] == '\0')
break;
content = eol + 1;
}
return strbuf_free_nobuf(buf);
}
static char *list2lines(GList *list)
{
struct strbuf *s = strbuf_new();
while (list)
{
strbuf_append_str(s, (char*)list->data);
strbuf_append_char(s, '\n');
free(list->data);
list = g_list_delete_link(list, list);
}
return strbuf_free_nobuf(s);
}
void graph_info_append_intersect(ErrorCleaning *cleaning, const char *intersect_name)
{
if(!cleaning->is_graph_intersection)
{
strbuf_set(&cleaning->intersection_name, intersect_name);
}
else
{
strbuf_append_char(&cleaning->intersection_name, ',');
strbuf_append_str(&cleaning->intersection_name, intersect_name);
}
cleaning->is_graph_intersection = true;
}
char *http_digest_proxy_authenticate(const char *realm, int stale)
{
char *buf = NULL;
size_t size = 0, offset = 0;
struct timeval tv;
char *nonce;
if (gettimeofday(&tv, NULL) == -1)
return NULL;
strbuf_append_str(&buf, &size, &offset, "Digest realm=");
append_quoted_string(&buf, &size, &offset, realm);
nonce = make_nonce(&tv);
strbuf_append_str(&buf, &size, &offset, ", nonce=");
append_quoted_string(&buf, &size, &offset, nonce);
free(nonce);
strbuf_append_str(&buf, &size, &offset, ", qop=\"auth\"");
if (stale)
strbuf_append_str(&buf, &size, &offset, ", stale=true");
return buf;
}
char *http_header_to_string(const struct http_header *header, size_t *n)
{
const struct http_header *p;
char *buf = NULL;
size_t size = 0, offset = 0;
strbuf_append_str(&buf, &size, &offset, "");
for (p = header; p != NULL; p = p->next)
strbuf_sprintf(&buf, &size, &offset, "%s: %s\r\n", p->name, p->value);
if (n != NULL)
*n = offset;
return buf;
}
void
btp_thread_append_to_str(struct btp_thread *thread,
struct strbuf *str,
bool verbose)
{
int framecount = btp_thread_get_frame_count(thread);
if (verbose)
{
strbuf_append_strf(str, "Thread no. %d (%d frames)\n",
thread->number, framecount);
}
else
strbuf_append_str(str, "Thread\n");
struct btp_frame *frame = thread->frames;
while (frame)
{
btp_frame_append_to_str(frame, str, verbose);
frame = frame->next;
}
}
char* make_description_comment(problem_data_t *problem_data)
{
char *comment = NULL;
struct problem_item *value;
value = get_problem_data_item_or_NULL(problem_data, FILENAME_COMMENT);
if (value)
{
if (value->content[0])
comment = xasprintf("\n\nComment\n-----\n%s", value->content);
}
if (!comment)
return NULL;
struct strbuf *buf_dsc = strbuf_new();
if (comment)
strbuf_append_str(buf_dsc, comment);
free(comment);
return strbuf_free_nobuf(buf_dsc);
}
void graph_info_merge(GraphInfo *dst, const GraphInfo *src)
{
// Update sample name
if(strcmp(src->sample_name.b,"undefined") != 0) {
if(strcmp(dst->sample_name.b,"undefined") == 0) {
strbuf_set_buff(&dst->sample_name, &src->sample_name);
} else {
strbuf_append_char(&dst->sample_name, ',');
strbuf_append_str(&dst->sample_name, src->sample_name.b);
}
}
uint64_t total_sequence = dst->total_sequence + src->total_sequence;
if(total_sequence > 0)
{
// Average error rates
dst->seq_err
= (dst->seq_err * dst->total_sequence +
src->seq_err * src->total_sequence) /
total_sequence;
// Update mean read length
size_t src_num_contigs = 0;
if(src->total_sequence && src->mean_read_length)
src_num_contigs = ((double)src->total_sequence/src->mean_read_length)+0.5;
graph_info_update_contigs(dst, src->total_sequence, src_num_contigs);
}
// Update error cleaning
error_cleaning_merge(&dst->cleaning, &src->cleaning);
dst->total_sequence = total_sequence;
}
int main(int argc, char **argv)
{
abrt_init(argv);
/* Can't keep these strings/structs static: _() doesn't support that */
const char *program_usage_string = _(
"\b [options] -d DIR\n"
"\n"
"Analyzes C/C++ backtrace, generates duplication hash, backtrace rating, and identifies crash function in dump directory DIR"
);
enum {
OPT_v = 1 << 0,
OPT_d = 1 << 1
};
/* Keep enum above and order of options below in sync! */
struct options program_options[] = {
OPT__VERBOSE(&g_verbose),
OPT_STRING('d', NULL, &dump_dir_name, "DIR", _("Dump directory")),
OPT_END()
};
/*unsigned opts =*/ parse_opts(argc, argv, program_options, program_usage_string);
export_abrt_envvars(0);
struct dump_dir *dd = dd_opendir(dump_dir_name, /*flags:*/ 0);
if (!dd)
return 1;
char *component = dd_load_text(dd, FILENAME_COMPONENT);
/* Read backtrace */
char *backtrace_str = dd_load_text_ext(dd, FILENAME_BACKTRACE,
DD_LOAD_TEXT_RETURN_NULL_ON_FAILURE);
if (!backtrace_str)
{
dd_close(dd);
return 1;
}
/* Compute backtrace hash */
struct btp_location location;
btp_location_init(&location);
char *backtrace_str_ptr = backtrace_str;
struct btp_backtrace *backtrace = btp_backtrace_parse(&backtrace_str_ptr, &location);
free(backtrace_str);
/* Store backtrace hash */
if (!backtrace)
{
/*
* The parser failed. Compute the duphash from the executable
* instead of a backtrace.
* and component only. This is not supposed to happen often.
*/
log(_("Backtrace parsing failed for %s"), dump_dir_name);
log("%d:%d: %s", location.line, location.column, location.message);
struct strbuf *emptybt = strbuf_new();
char *executable = dd_load_text(dd, FILENAME_EXECUTABLE);
strbuf_prepend_str(emptybt, executable);
free(executable);
strbuf_prepend_str(emptybt, component);
VERB3 log("Generating duphash: %s", emptybt->buf);
char hash_str[SHA1_RESULT_LEN*2 + 1];
create_hash(hash_str, emptybt->buf);
dd_save_text(dd, FILENAME_DUPHASH, hash_str);
/*
* Other parts of ABRT assume that if no rating is available,
* it is ok to allow reporting of the bug. To be sure no bad
* backtrace is reported, rate the backtrace with the lowest
* rating.
*/
dd_save_text(dd, FILENAME_RATING, "0");
strbuf_free(emptybt);
free(component);
dd_close(dd);
/* Report success even if the parser failed, as the backtrace
* has been created and rated. The failure is caused by a flaw
* in the parser, not in the backtrace.
*/
return 0;
}
/* Compute duplication hash. */
char *str_hash_core = btp_backtrace_get_duplication_hash(backtrace);
struct strbuf *str_hash = strbuf_new();
strbuf_append_str(str_hash, component);
strbuf_append_str(str_hash, str_hash_core);
VERB3 log("Generating duphash: %s", str_hash->buf);
char hash_str[SHA1_RESULT_LEN*2 + 1];
create_hash(hash_str, str_hash->buf);
dd_save_text(dd, FILENAME_DUPHASH, hash_str);
strbuf_free(str_hash);
free(str_hash_core);
/* Compute the backtrace rating. */
float quality = btp_backtrace_quality_complex(backtrace);
const char *rating;
if (quality < 0.6f)
rating = "0";
else if (quality < 0.7f)
rating = "1";
else if (quality < 0.8f)
rating = "2";
else if (quality < 0.9f)
rating = "3";
else
rating = "4";
dd_save_text(dd, FILENAME_RATING, rating);
/* Get the function name from the crash frame. */
struct btp_frame *crash_frame = btp_backtrace_get_crash_frame(backtrace);
if (crash_frame)
{
if (crash_frame->function_name &&
0 != strcmp(crash_frame->function_name, "??"))
{
dd_save_text(dd, FILENAME_CRASH_FUNCTION, crash_frame->function_name);
}
btp_frame_free(crash_frame);
}
btp_backtrace_free(backtrace);
dd_close(dd);
free(component);
return 0;
}
/**
* Print paths to a string buffer. Paths are sorted before being written.
*
* @param hkey All paths associated with hkey are written to the buffer
* @param sbuf paths are written this string buffer
* @param subset is a temp variable that is reused each time
* @param nbuf temporary buffer, if not NULL, used to add seq=... to output
* @param jposbuf temporary buffer, if not NULL, used to add juncpos=... to output
*/
void gpath_save_sbuf(hkey_t hkey, StrBuf *sbuf, GPathSubset *subset,
dBNodeBuffer *nbuf, SizeBuffer *jposbuf,
const dBGraph *db_graph)
{
ctx_assert(db_graph->num_of_cols == 1 || nbuf == NULL);
ctx_assert(db_graph->num_of_cols == 1 || jposbuf == NULL);
const GPathStore *gpstore = &db_graph->gpstore;
const GPathSet *gpset = &gpstore->gpset;
const size_t ncols = gpstore->gpset.ncols;
GPath *first_gpath = gpath_store_fetch(gpstore, hkey);
const GPath *gpath;
size_t i, j, col;
// Load and sort paths for given kmer
gpath_subset_reset(subset);
gpath_subset_load_llist(subset, first_gpath);
gpath_subset_sort(subset);
if(subset->list.len == 0) return;
// Print "<kmer> <npaths>"
BinaryKmer bkmer = db_graph->ht.table[hkey];
char bkstr[MAX_KMER_SIZE+1];
binary_kmer_to_str(bkmer, db_graph->kmer_size, bkstr);
// strbuf_sprintf(sbuf, "%s %zu\n", bkstr, subset->list.len);
strbuf_append_strn(sbuf, bkstr, db_graph->kmer_size);
strbuf_append_char(sbuf, ' ');
strbuf_append_ulong(sbuf, subset->list.len);
strbuf_append_char(sbuf, '\n');
char orchar[2] = {0};
orchar[FORWARD] = 'F';
orchar[REVERSE] = 'R';
const uint8_t *nseenptr;
for(i = 0; i < subset->list.len; i++)
{
gpath = subset->list.b[i];
nseenptr = gpath_set_get_nseen(gpset, gpath);
// strbuf_sprintf(sbuf, "%c %zu %u %u", orchar[gpath->orient], klen,
// gpath->num_juncs, (uint32_t)nseenptr[0]);
strbuf_append_char(sbuf, orchar[gpath->orient]);
strbuf_append_char(sbuf, ' ');
strbuf_append_ulong(sbuf, gpath->num_juncs);
strbuf_append_char(sbuf, ' ');
strbuf_append_ulong(sbuf, nseenptr[0]);
for(col = 1; col < ncols; col++) {
// strbuf_sprintf(sbuf, ",%u", (uint32_t)nseenptr[col]);
strbuf_append_char(sbuf, ',');
strbuf_append_ulong(sbuf, nseenptr[col]);
}
strbuf_append_char(sbuf, ' ');
strbuf_ensure_capacity(sbuf, sbuf->end + gpath->num_juncs + 2);
binary_seq_to_str(gpath->seq, gpath->num_juncs, sbuf->b+sbuf->end);
sbuf->end += gpath->num_juncs;
if(nbuf)
{
// Trace this path through the graph
// First, find a colour this path is in
for(col = 0; col < ncols && !gpath_has_colour(gpath, ncols, col); col++) {}
if(col == ncols) die("path is not in any colours");
dBNode node = {.key = hkey, .orient = gpath->orient};
db_node_buf_reset(nbuf);
if(jposbuf) size_buf_reset(jposbuf); // indices of junctions in nbuf
gpath_fetch(node, gpath, nbuf, jposbuf, col, db_graph);
strbuf_append_str(sbuf, " seq=");
strbuf_ensure_capacity(sbuf, sbuf->end + db_graph->kmer_size + nbuf->len);
sbuf->end += db_nodes_to_str(nbuf->b, nbuf->len, db_graph,
sbuf->b+sbuf->end);
if(jposbuf) {
strbuf_append_str(sbuf, " juncpos=");
strbuf_append_ulong(sbuf, jposbuf->b[0]);
for(j = 1; j < jposbuf->len; j++) {
strbuf_append_char(sbuf, ',');
strbuf_append_ulong(sbuf, jposbuf->b[j]);
}
}
}
strbuf_append_char(sbuf, '\n');
}
}
// @subset is a temp variable that is reused each time
// @sbuf is a temp variable that is reused each time
static inline int _gpath_gzsave_node(hkey_t hkey,
StrBuf *sbuf, GPathSubset *subset,
dBNodeBuffer *nbuf, SizeBuffer *jposbuf,
gzFile gzout, pthread_mutex_t *outlock,
const dBGraph *db_graph)
{
gpath_save_sbuf(hkey, sbuf, subset, nbuf, jposbuf, db_graph);
if(sbuf->end > DEFAULT_IO_BUFSIZE)
_gpath_save_flush(gzout, sbuf, outlock);
return 0; // => keep iterating
}
char *make_description(problem_data_t *problem_data, char **names_to_skip,
unsigned max_text_size, unsigned desc_flags)
{
struct strbuf *buf_dsc = strbuf_new();
const char *analyzer = problem_data_get_content_or_NULL(problem_data,
FILENAME_ANALYZER);
GList *list = g_hash_table_get_keys(problem_data);
list = g_list_sort(list, (GCompareFunc)strcmp);
GList *l;
/* Print one-liners. Format:
* NAME1: <maybe more spaces>VALUE1
* NAME2: <maybe more spaces>VALUE2
*/
bool empty = true;
l = list;
while (l)
{
const char *key = l->data;
l = l->next;
/* Skip items we are not interested in */
//TODO: optimize by doing this once, not 3 times:
if (names_to_skip
&& rejected_name(key, names_to_skip, desc_flags))
continue;
struct problem_item *item = g_hash_table_lookup(problem_data, key);
if (!item)
continue;
if ((desc_flags & MAKEDESC_SHOW_ONLY_LIST) && !(item->flags & CD_FLAG_LIST))
continue;
if ((item->flags & CD_FLAG_TXT)
&& strlen(item->content) <= max_text_size
) {
char *formatted = problem_item_format(item);
char *output = formatted ? formatted : item->content;
char *eol = strchr(output, '\n');
if (!eol)
{
int pad = 16 - (strlen(key) + 2);
if (pad < 0) pad = 0;
strbuf_append_strf(buf_dsc, "%s: %*s%s\n", key, pad, "", output);
empty = false;
}
free(formatted);
}
}
bool append_empty_line = !empty;
if (desc_flags & MAKEDESC_SHOW_FILES)
{
/* Print file info. Format:
* <empty line if needed>
* NAME1: <maybe more spaces>Binary file, NNN bytes
* NAME2: <maybe more spaces>Text file, NNN bytes
*
* In many cases, it is useful to know how big binary files are
* (for example, helps with diagnosing bug upload problems)
*/
l = list;
while (l)
{
const char *key = l->data;
l = l->next;
/* Skip items we are not interested in */
if (names_to_skip
&& rejected_name(key, names_to_skip, desc_flags))
continue;
struct problem_item *item = g_hash_table_lookup(problem_data, key);
if (!item)
continue;
if ((desc_flags & MAKEDESC_SHOW_ONLY_LIST) && !(item->flags & CD_FLAG_LIST))
continue;
if ((item->flags & CD_FLAG_BIN)
|| ((item->flags & CD_FLAG_TXT) && strlen(item->content) > max_text_size)
) {
if (append_empty_line)
strbuf_append_char(buf_dsc, '\n');
append_empty_line = false;
struct stat statbuf;
int stat_err = 0;
if (item->flags & CD_FLAG_BIN)
stat_err = stat(item->content, &statbuf);
else
statbuf.st_size = strlen(item->content);
/* We don't print item->content for CD_FLAG_BIN, as it is
* always "/path/to/dump/dir/KEY" - not informative.
*/
int pad = 16 - (strlen(key) + 2);
if (pad < 0) pad = 0;
strbuf_append_strf(buf_dsc,
(!stat_err ? "%s: %*s%s file, %llu bytes\n" : "%s: %*s%s file\n"),
key,
pad, "",
((item->flags & CD_FLAG_BIN) ? "Binary" : "Text"),
(long long)statbuf.st_size
);
empty = false;
}
}
}
if (desc_flags & MAKEDESC_SHOW_MULTILINE)
{
/* Print multi-liners. Format:
* <empty line if needed>
* NAME:
* :LINE1
* :LINE2
* :LINE3
*/
l = list;
while (l)
{
const char *key = l->data;
l = l->next;
/* Skip items we are not interested in */
if (names_to_skip
&& rejected_name(key, names_to_skip, desc_flags))
continue;
struct problem_item *item = g_hash_table_lookup(problem_data, key);
if (!item)
continue;
if ((desc_flags & MAKEDESC_SHOW_ONLY_LIST) && !(item->flags & CD_FLAG_LIST))
continue;
if ((item->flags & CD_FLAG_TXT)
&& (strlen(item->content) <= max_text_size
|| (!strcmp(analyzer, "Kerneloops") && !strcmp(key, FILENAME_BACKTRACE))))
{
char *formatted = problem_item_format(item);
char *output = make_description_item_multiline(key, formatted ? formatted : item->content);
if (output)
{
if (!empty)
strbuf_append_str(buf_dsc, "\n");
strbuf_append_str(buf_dsc, output);
empty = false;
free(output);
}
free(formatted);
}
}
}
g_list_free(list);
return strbuf_free_nobuf(buf_dsc);
}
static
char *format_percented_string(const char *str, problem_data_t *pd)
{
size_t old_pos[MAX_OPT_DEPTH] = { 0 };
int okay[MAX_OPT_DEPTH] = { 1 };
int opt_depth = 1;
struct strbuf *result = strbuf_new();
while (*str) {
switch (*str) {
default:
strbuf_append_char(result, *str);
str++;
break;
case '\\':
if (str[1])
str++;
strbuf_append_char(result, *str);
str++;
break;
case '[':
if (str[1] == '[' && opt_depth < MAX_OPT_DEPTH)
{
old_pos[opt_depth] = result->len;
okay[opt_depth] = 1;
opt_depth++;
str += 2;
} else {
strbuf_append_char(result, *str);
str++;
}
break;
case ']':
if (str[1] == ']' && opt_depth > 1)
{
opt_depth--;
if (!okay[opt_depth])
{
result->len = old_pos[opt_depth];
result->buf[result->len] = '\0';
}
str += 2;
} else {
strbuf_append_char(result, *str);
str++;
}
break;
case '%': ;
char *nextpercent = strchr(++str, '%');
if (!nextpercent)
{
error_msg_and_die("Unterminated %%element%%: '%s'", str - 1);
}
*nextpercent = '\0';
const problem_item *item = problem_data_get_item_or_NULL(pd, str);
*nextpercent = '%';
if (item && (item->flags & CD_FLAG_TXT))
strbuf_append_str(result, item->content);
else
okay[opt_depth - 1] = 0;
str = nextpercent + 1;
break;
}
}
if (opt_depth > 1)
{
error_msg_and_die("Unbalanced [[ ]] bracket");
}
if (!okay[0])
{
error_msg("Undefined variable outside of [[ ]] bracket");
}
return strbuf_free_nobuf(result);
}
lwan_tpl_chunk_t *
lwan_tpl_apply_until(lwan_tpl_chunk_t *chunks, strbuf_t *buf,
char *(*var_get)(const char *name, void *data), void *var_get_data,
bool (*until)(lwan_tpl_chunk_t *chunk, void *data), void *until_data)
{
lwan_tpl_chunk_t *chunk = chunks;
for (; chunk; chunk = chunk->next) {
if (until(chunk, until_data))
break;
switch (chunk->action) {
case TPL_ACTION_APPEND:
strbuf_append_str(buf, chunk->data, 0);
break;
case TPL_ACTION_APPEND_CHAR:
strbuf_append_char(buf, (char)(uintptr_t)chunk->data);
break;
case TPL_ACTION_VARIABLE:
{
char *tmp = var_get((const char*)chunk->data, var_get_data);
strbuf_append_str(buf, tmp, 0);
free(tmp);
}
break;
case TPL_ACTION_LIST_START_ITER:
strbuf_append_str(buf, "[begin_iter:", 0);
strbuf_append_str(buf, chunk->data, 0);
strbuf_append_str(buf, "]", 0);
break;
case TPL_ACTION_LIST_END_ITER:
strbuf_append_str(buf, "[end_iter:", 0);
strbuf_append_str(buf, chunk->data, 0);
strbuf_append_str(buf, "]", 0);
break;
case TPL_ACTION_IF_VARIABLE_NOT_EMPTY:
{
const char *var_name = (const char*)chunk->data;
char *tmp = var_get(var_name, var_get_data);
if (tmp && *tmp) {
chunk = lwan_tpl_apply_until(chunk->next, buf, var_get, var_get_data,
until_not_empty, chunk->data);
} else {
for (chunk = chunk->next; chunk; chunk = chunk->next) {
if (chunk->action == TPL_ACTION_END_IF_VARIABLE_NOT_EMPTY && !strcmp(chunk->data, var_name))
break;
}
}
free(tmp);
}
break;
case TPL_ACTION_END_IF_VARIABLE_NOT_EMPTY:
/* Shouldn't happen */
break;
case TPL_ACTION_APPLY_TPL:
{
strbuf_t *tmp = lwan_tpl_apply(chunk->data, var_get, var_get_data);
strbuf_append_str(buf, strbuf_get_buffer(tmp), strbuf_get_length(tmp));
strbuf_free(tmp);
}
}
}
return chunk;
}
char *run_unstrip_n(const char *dump_dir_name, unsigned timeout_sec)
{
int flags = EXECFLG_INPUT_NUL | EXECFLG_OUTPUT | EXECFLG_SETSID | EXECFLG_QUIET;
VERB1 flags &= ~EXECFLG_QUIET;
int pipeout[2];
char* args[4];
args[0] = (char*)"eu-unstrip";
args[1] = xasprintf("--core=%s/"FILENAME_COREDUMP, dump_dir_name);
args[2] = (char*)"-n";
args[3] = NULL;
pid_t child = fork_execv_on_steroids(flags, args, pipeout, /*env_vec:*/ NULL, /*dir:*/ NULL, /*uid(unused):*/ 0);
free(args[1]);
/* Bugs in unstrip or corrupted coredumps can cause it to enter infinite loop.
* Therefore we have a (largish) timeout, after which we kill the child.
*/
ndelay_on(pipeout[0]);
int t = time(NULL); /* int is enough, no need to use time_t */
int endtime = t + timeout_sec;
struct strbuf *buf_out = strbuf_new();
while (1)
{
int timeout = endtime - t;
if (timeout < 0)
{
kill(child, SIGKILL);
strbuf_free(buf_out);
buf_out = NULL;
break;
}
/* We don't check poll result - checking read result is enough */
struct pollfd pfd;
pfd.fd = pipeout[0];
pfd.events = POLLIN;
poll(&pfd, 1, timeout * 1000);
char buff[1024];
int r = read(pipeout[0], buff, sizeof(buff) - 1);
if (r <= 0)
{
/* I did see EAGAIN happening here */
if (r < 0 && errno == EAGAIN)
goto next;
break;
}
buff[r] = '\0';
strbuf_append_str(buf_out, buff);
next:
t = time(NULL);
}
close(pipeout[0]);
/* Prevent having zombie child process */
int status;
safe_waitpid(child, &status, 0);
if (status != 0 || buf_out == NULL)
{
/* unstrip didnt exit with exit code 0, or we timed out */
strbuf_free(buf_out);
return NULL;
}
return strbuf_free_nobuf(buf_out);
}
int ctx_calls2vcf(int argc, char **argv)
{
const char *in_path = NULL, *out_path = NULL, *out_type = NULL;
// Filtering parameters
int32_t min_mapq = -1, max_align_len = -1, max_allele_len = -1;
// Alignment parameters
int nwmatch = 1, nwmismatch = -2, nwgapopen = -4, nwgapextend = -1;
// ref paths
char const*const* ref_paths = NULL;
size_t nref_paths = 0;
// flank file
const char *sam_path = NULL;
//
// Things we figure out by looking at the input
//
bool isbubble = false;
// samples in VCF, (0 for bubble, does not include ref in breakpoint calls)
size_t i, kmer_size, num_samples;
//
// Reference genome
//
// Hash map of chromosome name -> sequence
ChromHash *genome;
ReadBuffer chroms;
// Arg parsing
char cmd[100];
char shortopts[300];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
// silence error messages from getopt_long
// opterr = 0;
while((c = getopt_long_only(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 'o': cmd_check(!out_path, cmd); out_path = optarg; break;
case 'O': cmd_check(!out_type, cmd); out_type = optarg; break;
case 'f': cmd_check(!futil_get_force(), cmd); futil_set_force(true); break;
case 'F': cmd_check(!sam_path,cmd); sam_path = optarg; break;
case 'Q': cmd_check(min_mapq < 0,cmd); min_mapq = cmd_uint32(cmd, optarg); break;
case 'A': cmd_check(max_align_len < 0,cmd); max_align_len = cmd_uint32(cmd, optarg); break;
case 'L': cmd_check(max_allele_len < 0,cmd); max_allele_len = cmd_uint32(cmd, optarg); break;
case 'm': nwmatch = cmd_int32(cmd, optarg); break;
case 'M': nwmismatch = cmd_int32(cmd, optarg); break;
case 'g': nwgapopen = cmd_int32(cmd, optarg); break;
case 'G': nwgapextend = cmd_int32(cmd, optarg); break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
die("`"CMD" "SUBCMD" -h` for help. Bad option: %s", argv[optind-1]);
default: ctx_assert2(0, "shouldn't reach here: %c", c);
}
}
// Defaults for unset values
if(out_path == NULL) out_path = "-";
if(max_align_len < 0) max_align_len = DEFAULT_MAX_ALIGN;
if(max_allele_len < 0) max_allele_len = DEFAULT_MAX_ALLELE;
if(optind+2 > argc)
cmd_print_usage("Require <in.txt.gz> and at least one reference");
in_path = argv[optind++];
ref_paths = (char const*const*)argv + optind;
nref_paths = argc - optind;
// These functions call die() on error
gzFile gzin = futil_gzopen(in_path, "r");
// Read call file header
cJSON *json = json_hdr_load(gzin, in_path);
// Check we can handle the kmer size
kmer_size = json_hdr_get_kmer_size(json, in_path);
db_graph_check_kmer_size(kmer_size, in_path);
// Get format (bubble or breakpoint file)
cJSON *json_fmt = json_hdr_get(json, "file_format", cJSON_String, in_path);
if(strcmp(json_fmt->valuestring,"CtxBreakpoints") == 0) isbubble = false;
else if(strcmp(json_fmt->valuestring,"CtxBubbles") == 0) isbubble = true;
else die("Unknown format: '%s'", json_fmt->valuestring);
status("Reading %s in %s format", futil_inpath_str(in_path),
isbubble ? "bubble" : "breakpoint");
if(isbubble) {
// bubble specific
if(sam_path == NULL)
cmd_print_usage("Require -F <flanks.sam> with bubble file");
if(min_mapq < 0) min_mapq = DEFAULT_MIN_MAPQ;
}
else {
// breakpoint specific
if(min_mapq >= 0)
cmd_print_usage("-Q,--min-mapq <Q> only valid with bubble calls");
}
// Open flank file if it exists
htsFile *samfh = NULL;
bam_hdr_t *bam_hdr = NULL;
bam1_t *mflank = NULL;
if(sam_path)
{
if((samfh = hts_open(sam_path, "r")) == NULL)
die("Cannot open SAM/BAM %s", sam_path);
// Load BAM header
bam_hdr = sam_hdr_read(samfh);
if(bam_hdr == NULL) die("Cannot load BAM header: %s", sam_path);
mflank = bam_init1();
}
// Output VCF has 0 samples if bubbles file, otherwise has N where N is
// number of samples/colours in the breakpoint graph
size_t num_graph_samples = json_hdr_get_ncols(json, in_path);
size_t num_graph_nonref = json_hdr_get_nonref_ncols(json, in_path);
num_samples = 0;
if(!isbubble) {
// If last colour has "is_ref", drop number of samples by one
num_samples = num_graph_nonref < num_graph_samples ? num_graph_samples-1
: num_graph_samples;
}
//
// Open output file
//
if(!out_path) out_path = "-";
int mode = vcf_misc_get_outtype(out_type, out_path);
futil_create_output(out_path);
htsFile *vcffh = hts_open(out_path, modes_htslib[mode]);
status("[calls2vcf] Reading %s call file with %zu samples",
isbubble ? "Bubble" : "Breakpoint", num_graph_samples);
status("[calls2vcf] %zu sample output to: %s format: %s",
num_samples, futil_outpath_str(out_path), hsmodes_htslib[mode]);
if(isbubble) status("[calls2vcf] min. MAPQ: %i", min_mapq);
status("[calls2vcf] max alignment length: %i", max_align_len);
status("[calls2vcf] max VCF allele length: %i", max_allele_len);
status("[calls2vcf] alignment match:%i mismatch:%i gap open:%i extend:%i",
nwmatch, nwmismatch, nwgapopen, nwgapextend);
// Load reference genome
read_buf_alloc(&chroms, 1024);
genome = chrom_hash_init();
chrom_hash_load(ref_paths, nref_paths, &chroms, genome);
// convert to upper case
char *s;
for(i = 0; i < chroms.len; i++)
for(s = chroms.b[i].seq.b; *s; s++) *s = toupper(*s);
if(!isbubble) brkpnt_check_refs_match(json, genome, in_path);
bcf_hdr_t *vcfhdr = make_vcf_hdr(json, in_path, !isbubble, kmer_size,
ref_paths, nref_paths,
chroms.b, chroms.len);
if(bcf_hdr_write(vcffh, vcfhdr) != 0) die("Cannot write VCF header");
AlignedCall *call = acall_init();
CallDecomp *aligner = call_decomp_init(vcffh, vcfhdr);
scoring_t *scoring = call_decomp_get_scoring(aligner);
scoring_init(scoring, nwmatch, nwmismatch, nwgapopen, nwgapextend,
false, false, 0, 0, 0, 0);
CallFileEntry centry;
call_file_entry_alloc(¢ry);
char kmer_str[50];
sprintf(kmer_str, ";K%zu", kmer_size);
if(isbubble)
{
// Bubble calls
DecompBubble *bubbles = decomp_bubble_init();
// Set scoring for aligning 3' flank
scoring = decomp_bubble_get_scoring(bubbles);
scoring_init(scoring, nwmatch, nwmismatch, nwgapopen, nwgapextend,
true, true, 0, 0, 0, 0);
while(call_file_read(gzin, in_path, ¢ry)) {
do {
if(sam_read1(samfh, bam_hdr, mflank) < 0)
die("We've run out of SAM entries!");
} while(mflank->core.flag & (BAM_FSECONDARY | BAM_FSUPPLEMENTARY));
// Align call
strbuf_reset(&call->info);
decomp_bubble_call(bubbles, genome, kmer_size, min_mapq,
¢ry, mflank, bam_hdr, call);
strbuf_append_str(&call->info, kmer_str);
acall_decompose(aligner, call, max_align_len, max_allele_len);
}
// print bubble stats
DecompBubbleStats *bub_stats = ctx_calloc(1, sizeof(*bub_stats));
decomp_bubble_cpy_stats(bub_stats, bubbles);
print_bubble_stats(bub_stats);
ctx_free(bub_stats);
decomp_bubble_destroy(bubbles);
}
else
{
// Breakpoint calls
DecompBreakpoint *breakpoints = decomp_brkpt_init();
while(call_file_read(gzin, in_path, ¢ry)) {
strbuf_reset(&call->info);
decomp_brkpt_call(breakpoints, genome, num_samples, ¢ry, call);
strbuf_append_str(&call->info, kmer_str);
acall_decompose(aligner, call, max_align_len, max_allele_len);
}
// print bubble stats
DecompBreakpointStats *brk_stats = ctx_calloc(1, sizeof(*brk_stats));
decomp_brkpt_cpy_stats(brk_stats, breakpoints);
print_breakpoint_stats(brk_stats);
ctx_free(brk_stats);
decomp_brkpt_destroy(breakpoints);
}
// Print stats
DecomposeStats *astats = ctx_calloc(1, sizeof(*astats));
call_decomp_cpy_stats(astats, aligner);
print_acall_stats(astats);
ctx_free(astats);
call_file_entry_dealloc(¢ry);
call_decomp_destroy(aligner);
acall_destroy(call);
// Finished - clean up
cJSON_Delete(json);
gzclose(gzin);
bcf_hdr_destroy(vcfhdr);
hts_close(vcffh);
for(i = 0; i < chroms.len; i++) seq_read_dealloc(&chroms.b[i]);
read_buf_dealloc(&chroms);
chrom_hash_destroy(genome);
if(sam_path) {
hts_close(samfh);
bam_hdr_destroy(bam_hdr);
bam_destroy1(mflank);
}
return EXIT_SUCCESS;
}
char *get_backtrace(const char *dump_dir_name, unsigned timeout_sec, const char *debuginfo_dirs)
{
INITIALIZE_LIBABRT();
struct dump_dir *dd = dd_opendir(dump_dir_name, /*flags:*/ 0);
if (!dd)
return NULL;
char *executable = dd_load_text(dd, FILENAME_EXECUTABLE);
dd_close(dd);
/* Let user know what's going on */
log(_("Generating backtrace"));
unsigned i = 0;
char *args[25];
args[i++] = (char*)"gdb";
args[i++] = (char*)"-batch";
struct strbuf *set_debug_file_directory = strbuf_new();
unsigned auto_load_base_index = 0;
if(debuginfo_dirs == NULL)
{
// set non-existent debug file directory to prevent resolving
// function names - we need offsets for core backtrace.
strbuf_append_str(set_debug_file_directory, "set debug-file-directory /");
}
else
{
strbuf_append_str(set_debug_file_directory, "set debug-file-directory /usr/lib/debug");
struct strbuf *debug_directories = strbuf_new();
const char *p = debuginfo_dirs;
while (1)
{
while (*p == ':')
p++;
if (*p == '\0')
break;
const char *colon_or_nul = strchrnul(p, ':');
strbuf_append_strf(debug_directories, "%s%.*s/usr/lib/debug", (debug_directories->len == 0 ? "" : ":"),
(int)(colon_or_nul - p), p);
p = colon_or_nul;
}
strbuf_append_strf(set_debug_file_directory, ":%s", debug_directories->buf);
args[i++] = (char*)"-iex";
auto_load_base_index = i;
args[i++] = xasprintf("add-auto-load-safe-path %s", debug_directories->buf);
args[i++] = (char*)"-iex";
args[i++] = xasprintf("add-auto-load-scripts-directory %s", debug_directories->buf);
strbuf_free(debug_directories);
}
args[i++] = (char*)"-ex";
const unsigned debug_dir_cmd_index = i++;
args[debug_dir_cmd_index] = strbuf_free_nobuf(set_debug_file_directory);
/* "file BINARY_FILE" is needed, without it gdb cannot properly
* unwind the stack. Currently the unwind information is located
* in .eh_frame which is stored only in binary, not in coredump
* or debuginfo.
*
* Fedora GDB does not strictly need it, it will find the binary
* by its build-id. But for binaries either without build-id
* (= built on non-Fedora GCC) or which do not have
* their debuginfo rpm installed gdb would not find BINARY_FILE
* so it is still makes sense to supply "file BINARY_FILE".
*
* Unfortunately, "file BINARY_FILE" doesn't work well if BINARY_FILE
* was deleted (as often happens during system updates):
* gdb uses specified BINARY_FILE
* even if it is completely unrelated to the coredump.
* See https://bugzilla.redhat.com/show_bug.cgi?id=525721
*
* TODO: check mtimes on COREFILE and BINARY_FILE and not supply
* BINARY_FILE if it is newer (to at least avoid gdb complaining).
*/
args[i++] = (char*)"-ex";
const unsigned file_cmd_index = i++;
args[file_cmd_index] = xasprintf("file %s", executable);
free(executable);
args[i++] = (char*)"-ex";
const unsigned core_cmd_index = i++;
args[core_cmd_index] = xasprintf("core-file %s/"FILENAME_COREDUMP, dump_dir_name);
args[i++] = (char*)"-ex";
const unsigned bt_cmd_index = i++;
/*args[9] = ... see below */
args[i++] = (char*)"-ex";
args[i++] = (char*)"info sharedlib";
/* glibc's abort() stores its message in __abort_msg variable */
args[i++] = (char*)"-ex";
args[i++] = (char*)"print (char*)__abort_msg";
args[i++] = (char*)"-ex";
args[i++] = (char*)"print (char*)__glib_assert_msg";
args[i++] = (char*)"-ex";
args[i++] = (char*)"info all-registers";
args[i++] = (char*)"-ex";
const unsigned dis_cmd_index = i++;
args[dis_cmd_index] = (char*)"disassemble";
args[i++] = NULL;
/* Get the backtrace, but try to cap its size */
/* Limit bt depth. With no limit, gdb sometimes OOMs the machine */
unsigned bt_depth = 1024;
const char *thread_apply_all = "thread apply all -ascending";
const char *full = " full";
char *bt = NULL;
while (1)
{
args[bt_cmd_index] = xasprintf("%s backtrace %u%s", thread_apply_all, bt_depth, full);
bt = exec_vp(args, /*redirect_stderr:*/ 1, timeout_sec, NULL);
free(args[bt_cmd_index]);
if ((bt && strnlen(bt, 256*1024) < 256*1024) || bt_depth <= 32)
{
break;
}
bt_depth /= 2;
if (bt)
log("Backtrace is too big (%u bytes), reducing depth to %u",
(unsigned)strlen(bt), bt_depth);
else
/* (NB: in fact, current impl. of exec_vp() never returns NULL) */
log("Failed to generate backtrace, reducing depth to %u",
bt_depth);
free(bt);
/* Replace -ex disassemble (which disasms entire function $pc points to)
* to a version which analyzes limited, small patch of code around $pc.
* (Users reported a case where bare "disassemble" attempted to process
* entire .bss).
* TODO: what if "$pc-N" underflows? in my test, this happens:
* Dump of assembler code from 0xfffffffffffffff0 to 0x30:
* End of assembler dump.
* (IOW: "empty" dump)
*/
args[dis_cmd_index] = (char*)"disassemble $pc-20, $pc+64";
if (bt_depth <= 64 && thread_apply_all[0] != '\0')
{
/* This program likely has gazillion threads, dont try to bt them all */
bt_depth = 128;
thread_apply_all = "";
}
if (bt_depth <= 64 && full[0] != '\0')
{
/* Looks like there are gigantic local structures or arrays, disable "full" bt */
bt_depth = 128;
full = "";
}
}
if (auto_load_base_index > 0)
{
free(args[auto_load_base_index]);
free(args[auto_load_base_index + 2]);
}
free(args[debug_dir_cmd_index]);
free(args[file_cmd_index]);
free(args[core_cmd_index]);
return bt;
}
/**
*
* @param[out] status See `man 2 wait` for status information.
* @return Malloc'ed string
*/
static char* exec_vp(char **args, int redirect_stderr, int exec_timeout_sec, int *status)
{
/* Nuke everything which may make setlocale() switch to non-POSIX locale:
* we need to avoid having gdb output in some obscure language.
*/
static const char *const env_vec[] = {
"LANG",
"LC_ALL",
"LC_COLLATE",
"LC_CTYPE",
"LC_MESSAGES",
"LC_MONETARY",
"LC_NUMERIC",
"LC_TIME",
/* Workaround for
* http://sourceware.org/bugzilla/show_bug.cgi?id=9622
* (gdb emitting ESC sequences even with -batch)
*/
"TERM",
NULL
};
int flags = EXECFLG_INPUT_NUL | EXECFLG_OUTPUT | EXECFLG_SETSID | EXECFLG_QUIET;
if (redirect_stderr)
flags |= EXECFLG_ERR2OUT;
VERB1 flags &= ~EXECFLG_QUIET;
int pipeout[2];
pid_t child = fork_execv_on_steroids(flags, args, pipeout, (char**)env_vec, /*dir:*/ NULL, /*uid(unused):*/ 0);
/* We use this function to run gdb and unstrip. Bugs in gdb or corrupted
* coredumps were observed to cause gdb to enter infinite loop.
* Therefore we have a (largish) timeout, after which we kill the child.
*/
ndelay_on(pipeout[0]);
int t = time(NULL); /* int is enough, no need to use time_t */
int endtime = t + exec_timeout_sec;
struct strbuf *buf_out = strbuf_new();
while (1)
{
int timeout = endtime - t;
if (timeout < 0)
{
kill(child, SIGKILL);
strbuf_append_strf(buf_out, "\n"
"Timeout exceeded: %u seconds, killing %s.\n"
"Looks like gdb hung while generating backtrace.\n"
"This may be a bug in gdb. Consider submitting a bug report to gdb developers.\n"
"Please attach coredump from this crash to the bug report if you do.\n",
exec_timeout_sec, args[0]
);
break;
}
/* We don't check poll result - checking read result is enough */
struct pollfd pfd;
pfd.fd = pipeout[0];
pfd.events = POLLIN;
poll(&pfd, 1, timeout * 1000);
char buff[1024];
int r = read(pipeout[0], buff, sizeof(buff) - 1);
if (r <= 0)
{
/* I did see EAGAIN happening here */
if (r < 0 && errno == EAGAIN)
goto next;
break;
}
buff[r] = '\0';
strbuf_append_str(buf_out, buff);
next:
t = time(NULL);
}
close(pipeout[0]);
/* Prevent having zombie child process, and maybe collect status
* (note that status == NULL is ok too) */
safe_waitpid(child, status, 0);
return strbuf_free_nobuf(buf_out);
}
static bcf_hdr_t* make_vcf_hdr(cJSON *json, const char *in_path,
bool is_breakpoint, size_t kmer_size,
char const*const* ref_paths, size_t nref_paths,
read_t *chroms, size_t nchroms)
{
ctx_assert(json != NULL);
StrBuf hdrbuf;
strbuf_alloc(&hdrbuf, 1024);
char datestr[9];
time_t date = time(NULL);
strftime(datestr, 9, "%Y%m%d", localtime(&date));
strbuf_append_str(&hdrbuf, "##fileformat=VCFv4.2\n##fileDate=");
strbuf_append_str(&hdrbuf, datestr);
strbuf_append_str(&hdrbuf, "\n");
// Print commands used to generate header
cJSON *commands = json_hdr_get(json, "commands", cJSON_Array, in_path);
cJSON *command = commands->child;
// Print this command
char keystr[8];
char *prevstr = NULL;
size_t i;
if(command) {
cJSON *key = json_hdr_get(command, "key", cJSON_String, in_path);
prevstr = key->valuestring;
}
// Print command entry for this command
strbuf_append_str(&hdrbuf, "##mccortex_");
strbuf_append_str(&hdrbuf, hex_rand_str(keystr, sizeof(keystr)));
strbuf_append_str(&hdrbuf, "=<prev=\"");
strbuf_append_str(&hdrbuf, prevstr ? prevstr : "NULL");
strbuf_append_str(&hdrbuf, "\",cmd=\"");
strbuf_append_str(&hdrbuf, cmd_get_cmdline());
strbuf_append_str(&hdrbuf, "\",cwd=\"");
strbuf_append_str(&hdrbuf, cmd_get_cwd());
strbuf_append_str(&hdrbuf, "\",version="CTX_VERSION">\n");
// Print previous commands
vcf_hdrtxt_append_commands(command, &hdrbuf, in_path);
// Print field definitions
if(is_breakpoint)
strbuf_append_str(&hdrbuf, "##INFO=<ID=BRKPNT,Number=1,Type=String,Description=\"Breakpoint call\">\n");
else
strbuf_append_str(&hdrbuf, "##INFO=<ID=BUBBLE,Number=1,Type=String,Description=\"Bubble call\">\n");
strbuf_sprintf(&hdrbuf, "##INFO=<ID=K%zu,Number=0,Type=Flag,Description=\"Found at k=%zu\">\n", kmer_size, kmer_size);
strbuf_append_str(&hdrbuf, "##FORMAT=<ID=GT,Number=1,Type=String,Description=\"Genotype\">\n");
strbuf_append_str(&hdrbuf, "##FILTER=<ID=PASS,Description=\"All filters passed\">\n");
// Print reference paths
strbuf_append_str(&hdrbuf, "##reference=");
strbuf_append_str(&hdrbuf, ref_paths[0]);
for(i = 1; i < nref_paths; i++) {
strbuf_append_char(&hdrbuf, ',');
strbuf_append_str(&hdrbuf, ref_paths[i]);
}
strbuf_append_str(&hdrbuf, "\n");
// Print contigs lengths
for(i = 0; i < nchroms; i++) {
strbuf_sprintf(&hdrbuf, "##contig=<ID=%s,length=%zu>\n",
chroms[i].name.b, chroms[i].seq.end);
}
// Print VCF column header
strbuf_append_str(&hdrbuf, "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT");
if(is_breakpoint)
{
// Print a column for each sample
cJSON *graph_json = json_hdr_get(json, "graph", cJSON_Object, in_path);
cJSON *colours_json = json_hdr_get(graph_json, "colours", cJSON_Array, in_path);
cJSON *colour_json = colours_json->child;
if(colour_json == NULL) die("Missing colours");
for(; colour_json; colour_json = colour_json->next)
{
if(!json_hdr_colour_is_ref(colour_json)) {
cJSON *sample_json = json_hdr_get(colour_json, "sample", cJSON_String, in_path);
strbuf_append_str(&hdrbuf, "\t");
strbuf_append_str(&hdrbuf, sample_json->valuestring);
}
}
}
strbuf_append_char(&hdrbuf, '\n');
bcf_hdr_t *hdr = bcf_hdr_init("w");
if(bcf_hdr_parse(hdr, hdrbuf.b) != 0) die("Cannot construct VCF header");
strbuf_dealloc(&hdrbuf);
return hdr;
}
static int run_post_create(const char *dirname)
{
/* If doesn't start with "g_settings_dump_location/"... */
if (!dir_is_in_dump_location(dirname))
{
/* Then refuse to operate on it (someone is attacking us??) */
error_msg("Bad problem directory name '%s', should start with: '%s'", dirname, g_settings_dump_location);
return 400; /* Bad Request */
}
if (!dir_has_correct_permissions(dirname, DD_PERM_EVENTS))
{
error_msg("Problem directory '%s' has wrong owner or group", dirname);
return 400; /* */
}
/* Check completness */
{
struct dump_dir *dd = dd_opendir(dirname, DD_OPEN_READONLY);
char *provoker = NULL;
const bool event_dir = dd && problem_dump_dir_was_provoked_by_abrt_event(dd, &provoker);
if (event_dir)
{
if (g_settings_debug_level == 0)
{
error_msg("Removing problem provoked by ABRT(pid:%s): '%s'", provoker, dirname);
dd_delete(dd);
}
else
{
char *dumpdir = NULL;
char *event = NULL;
char *reason = NULL;
char *cmdline = NULL;
/* Ignore errors */
dd_get_env_variable(dd, "DUMP_DIR", &dumpdir);
dd_get_env_variable(dd, "EVENT", &event);
reason = dd_load_text(dd, FILENAME_REASON);
cmdline = dd_load_text(dd, FILENAME_CMDLINE);
error_msg("ABRT_SERVER_PID=%s;DUMP_DIR='%s';EVENT='%s';REASON='%s';CMDLINE='%s'",
provoker, dumpdir, event, reason, cmdline);
}
free(provoker);
return 400;
}
const bool complete = dd && problem_dump_dir_is_complete(dd);
dd_close(dd);
if (complete)
{
error_msg("Problem directory '%s' has already been processed", dirname);
return 403;
}
}
int child_stdout_fd;
int child_pid = spawn_event_handler_child(dirname, "post-create", &child_stdout_fd);
char *dup_of_dir = NULL;
struct strbuf *cmd_output = strbuf_new();
bool child_is_post_create = 1; /* else it is a notify child */
read_child_output:
//log("Reading from event fd %d", child_stdout_fd);
/* Read streamed data and split lines */
for (;;)
{
char buf[250]; /* usually we get one line, no need to have big buf */
errno = 0;
int r = safe_read(child_stdout_fd, buf, sizeof(buf) - 1);
if (r <= 0)
break;
buf[r] = '\0';
/* split lines in the current buffer */
char *raw = buf;
char *newline;
while ((newline = strchr(raw, '\n')) != NULL)
{
*newline = '\0';
strbuf_append_str(cmd_output, raw);
char *msg = cmd_output->buf;
if (child_is_post_create
&& prefixcmp(msg, "DUP_OF_DIR: ") == 0
) {
free(dup_of_dir);
dup_of_dir = xstrdup(msg + strlen("DUP_OF_DIR: "));
}
else
log("%s", msg);
strbuf_clear(cmd_output);
/* jump to next line */
raw = newline + 1;
}
/* beginning of next line. the line continues by next read */
strbuf_append_str(cmd_output, raw);
}
/* EOF/error */
/* Wait for child to actually exit, collect status */
int status = 0;
if (safe_waitpid(child_pid, &status, 0) <= 0)
/* should not happen */
perror_msg("waitpid(%d)", child_pid);
/* If it was a "notify[-dup]" event, then we're done */
if (!child_is_post_create)
goto ret;
/* exit 0 means "this is a good, non-dup dir" */
/* exit with 1 + "DUP_OF_DIR: dir" string => dup */
if (status != 0)
{
if (WIFSIGNALED(status))
{
log("'post-create' on '%s' killed by signal %d",
dirname, WTERMSIG(status));
goto delete_bad_dir;
}
/* else: it is WIFEXITED(status) */
if (!dup_of_dir)
{
log("'post-create' on '%s' exited with %d",
dirname, WEXITSTATUS(status));
goto delete_bad_dir;
}
}
const char *work_dir = (dup_of_dir ? dup_of_dir : dirname);
/* Load problem_data (from the *first dir* if this one is a dup) */
struct dump_dir *dd = dd_opendir(work_dir, /*flags:*/ 0);
if (!dd)
/* dd_opendir already emitted error msg */
goto delete_bad_dir;
/* Update count */
char *count_str = dd_load_text_ext(dd, FILENAME_COUNT, DD_FAIL_QUIETLY_ENOENT);
unsigned long count = strtoul(count_str, NULL, 10);
/* Don't increase crash count if we are working with newly uploaded
* directory (remote crash) which already has its crash count set.
*/
if ((status != 0 && dup_of_dir) || count == 0)
{
count++;
char new_count_str[sizeof(long)*3 + 2];
sprintf(new_count_str, "%lu", count);
dd_save_text(dd, FILENAME_COUNT, new_count_str);
/* This condition can be simplified to either
* (status * != 0 && * dup_of_dir) or (count == 1). But the
* chosen form is much more reliable and safe. We must not call
* dd_opendir() to locked dd otherwise we go into a deadlock.
*/
if (strcmp(dd->dd_dirname, dirname) != 0)
{
/* Update the last occurrence file by the time file of the new problem */
struct dump_dir *new_dd = dd_opendir(dirname, DD_OPEN_READONLY);
char *last_ocr = NULL;
if (new_dd)
{
/* TIME must exists in a valid dump directory but we don't want to die
* due to broken duplicated dump directory */
last_ocr = dd_load_text_ext(new_dd, FILENAME_TIME,
DD_LOAD_TEXT_RETURN_NULL_ON_FAILURE | DD_FAIL_QUIETLY_ENOENT);
dd_close(new_dd);
}
else
{ /* dd_opendir() already produced a message with good information about failure */
error_msg("Can't read the last occurrence file from the new dump directory.");
}
if (!last_ocr)
{ /* the new dump directory may lie in the dump location for some time */
log("Using current time for the last occurrence file which may be incorrect.");
time_t t = time(NULL);
last_ocr = xasprintf("%lu", (long)t);
}
dd_save_text(dd, FILENAME_LAST_OCCURRENCE, last_ocr);
free(last_ocr);
}
}
/* Reset mode/uig/gid to correct values for all files created by event run */
dd_sanitize_mode_and_owner(dd);
dd_close(dd);
if (!dup_of_dir)
log_notice("New problem directory %s, processing", work_dir);
else
{
log_warning("Deleting problem directory %s (dup of %s)",
strrchr(dirname, '/') + 1,
strrchr(dup_of_dir, '/') + 1);
delete_dump_dir(dirname);
}
/* Run "notify[-dup]" event */
int fd;
child_pid = spawn_event_handler_child(
work_dir,
(dup_of_dir ? "notify-dup" : "notify"),
&fd
);
//log("Started notify, fd %d -> %d", fd, child_stdout_fd);
xmove_fd(fd, child_stdout_fd);
child_is_post_create = 0;
strbuf_clear(cmd_output);
free(dup_of_dir);
dup_of_dir = NULL;
goto read_child_output;
delete_bad_dir:
log_warning("Deleting problem directory '%s'", dirname);
delete_dump_dir(dirname);
ret:
strbuf_free(cmd_output);
free(dup_of_dir);
close(child_stdout_fd);
return 0;
}
// caller is reposible for freeing *product* and *version*
static void parse_release(const char *release, char** product, char** version, int flags)
{
/* Fedora has a single non-numeric release - Rawhide */
if (strstr(release, "Rawhide"))
{
*product = xstrdup("Fedora");
*version = xstrdup("rawhide");
log_debug("%s: version:'%s' product:'%s'", __func__, *version, *product);
return;
}
/* openSUSE has two non-numeric releases - Factory and Tumbleweed
None of them is unfortunately identified in any of /etc/SuSE-brand,
/etc/SuSE-release or /etc/os-release. Keep this piece of code commented
just not to forget about that. */
/*
if (strstr(release, "Factory"))
{
*product = xstrdup("openSUSE");
*version = xstrdup("Factory");
log_debug("%s: version:'%s' product:'%s'", __func__, *version, *product);
return;
}
if (strstr(release, "Tumbleweed"))
{
*product = xstrdup("openSUSE");
*version = xstrdup("Tumbleweed");
log_debug("%s: version:'%s' product:'%s'", __func__, *version, *product);
return;
}
*/
bool it_is_rhel = false;
struct strbuf *buf_product = strbuf_new();
if (strstr(release, "Fedora"))
{
strbuf_append_str(buf_product, "Fedora");
}
else if (strstr(release, "Red Hat Enterprise Linux"))
{
strbuf_append_str(buf_product, "Red Hat Enterprise Linux");
it_is_rhel = true;
}
else if (strstr(release, "openSUSE"))
{
strbuf_append_str(buf_product, "openSUSE");
}
else
{
/* TODO: add logic for parsing other distros' names here */
strbuf_append_str(buf_product, release);
}
/* Examples of release strings:
* installed system: "Red Hat Enterprise Linux Server release 6.2 Beta (Santiago)"
* anaconda: "Red Hat Enterprise Linux 6.2"
*/
struct strbuf *buf_version = strbuf_new();
const char *r = strstr(release, "release");
const char *space = r ? strchr(r, ' ') : NULL;
if (!space)
{
/* Try to find "<space><digit>" sequence */
space = release;
while ((space = strchr(space, ' ')) != NULL)
{
if (space[1] >= '0' && space[1] <= '9')
break;
space++;
}
}
if (space)
{
space++;
/* Observed also: "Fedora 16-Alpha" rhbz#730887 */
while ((*space >= '0' && *space <= '9') || *space == '.')
{
/* Eat string like "5.2" */
strbuf_append_char(buf_version, *space);
space++;
}
if (flags & RETAIN_ALPHA_BETA_TAIL_IN_VER)
{
/* Example: "... 6.2 [Beta ](Santiago)".
* 'space' variable points to non-digit char after "2".
* We assume that non-parenthesized text is "Alpha"/"Beta"/etc.
* If this text is only whitespace, we won't append it.
*/
const char *to_append = space;
while (*space && *space != '(') /* go to '(' */
space++;
while (space > to_append && space[-1] == ' ') /* back to 1st non-space */
space--;
strbuf_append_strf(buf_version, "%.*s", (int)(space - to_append), to_append);
}
}
if ((flags & APPEND_MAJOR_VER_TO_RHEL_PRODUCT) && it_is_rhel)
{
char *v = buf_version->buf;
/* Append "integer part" of version to product:
* "10.2<anything>" -> append " 10"
* "10 <anything>" -> append " 10"
* "10" -> append " 10"
* "10abcde" -> append ?????
*/
unsigned idx_dot = strchrnul(v, '.') - v;
unsigned idx_space = strchrnul(v, ' ') - v;
strbuf_append_strf(buf_product, " %.*s",
(idx_dot < idx_space ? idx_dot : idx_space), v
);
}
*version = strbuf_free_nobuf(buf_version);
*product = strbuf_free_nobuf(buf_product);
log_debug("%s: version:'%s' product:'%s'", __func__, *version, *product);
}
void vcf_hdrtxt_append_commands(cJSON *command, StrBuf *hdr, const char *path)
{
bool first;
for(; command != NULL; command = command->next)
{
cJSON *key = json_hdr_get(command, "key", cJSON_String, path);
cJSON *cmd = json_hdr_get(command, "cmd", cJSON_Array, path);
cJSON *cwd = json_hdr_get(command, "cwd", cJSON_String, path);
cJSON *prev = json_hdr_get(command, "prev", cJSON_Array, path);
cJSON *ver = json_hdr_try(command, "mccortex",cJSON_String, path);
prev = prev->child; // result could be NULL
if(prev && prev->type != cJSON_String) die("Invalid 'prev' field");
strbuf_append_str(hdr, "##mccortex_");
strbuf_append_str(hdr, key->valuestring);
strbuf_append_str(hdr, "=<prev=\"");
strbuf_append_str(hdr, prev ? prev->valuestring : "NULL");
if(prev) {
while((prev = prev->next) != NULL) {
strbuf_append_str(hdr, ";");
strbuf_append_str(hdr, prev->valuestring);
}
}
strbuf_append_str(hdr, "\",cmd=\"");
for(first = true, cmd = cmd->child; cmd; cmd = cmd->next, first = false) {
if(!first) strbuf_append_char(hdr, ' ');
strbuf_append_str(hdr, cmd->valuestring);
}
strbuf_append_str(hdr, "\",cwd=\"");
strbuf_append_str(hdr, cwd->valuestring);
strbuf_append_str(hdr, "\"");
if(ver) {
strbuf_append_str(hdr, ",version=\"");
strbuf_append_str(hdr, ver->valuestring);
strbuf_append_str(hdr, "\"");
}
strbuf_append_str(hdr, ">\n");
}
}
