static struct http_buf *http_serialize_response(struct http_channel *c,
struct http_response *r)
{
struct http_header *h;
wrbuf_rewind(c->wrbuf);
wrbuf_printf(c->wrbuf, "HTTP/%s %s %s\r\n", c->version, r->code, r->msg);
for (h = r->headers; h; h = h->next)
wrbuf_printf(c->wrbuf, "%s: %s\r\n", h->name, h->value);
if (r->payload)
{
wrbuf_printf(c->wrbuf, "Content-Length: %d\r\n", r->payload ?
(int) strlen(r->payload) : 0);
wrbuf_printf(c->wrbuf, "Content-Type: %s\r\n", r->content_type);
if (!strcmp(r->content_type, "text/xml"))
{
xmlDoc *doc = xmlParseMemory(r->payload, strlen(r->payload));
if (doc)
{
xmlFreeDoc(doc);
}
else
{
yaz_log(YLOG_WARN, "Sending non-wellformed "
"response (bug #1162");
yaz_log(YLOG_WARN, "payload: %s", r->payload);
}
}
}
wrbuf_puts(c->wrbuf, "\r\n");
if (r->payload)
wrbuf_puts(c->wrbuf, r->payload);
return http_buf_bywrbuf(c->http_server, c->wrbuf);
}
static int yaz_query_charset_convert_buf(char *buf_in, int len_in,
char **buf_out, int *len_out,
ODR o, WRBUF wrbuf, yaz_iconv_t cd)
{
int r = 0;
wrbuf_rewind(wrbuf);
wrbuf_iconv_write(wrbuf, cd, buf_in, len_in);
wrbuf_iconv_reset(wrbuf, cd);
*len_out = wrbuf_len(wrbuf);
if (*len_out == 0)
{ /* we assume conversion failed */
*buf_out = buf_in;
*len_out = len_in;
r = -1;
}
else
{
/* conversion OK */
*buf_out = (char*) odr_malloc(o, *len_out);
memcpy(*buf_out, wrbuf_buf(wrbuf), *len_out);
}
return r;
}
int yaz_marc_write_iso2709(yaz_marc_t mt, WRBUF wr)
{
struct yaz_marc_node *n;
int indicator_length;
int identifier_length;
int length_data_entry;
int length_starting;
int length_implementation;
int data_offset = 0;
const char *leader = 0;
WRBUF wr_dir, wr_head, wr_data_tmp;
int base_address;
for (n = mt->nodes; n; n = n->next)
if (n->which == YAZ_MARC_LEADER)
leader = n->u.leader;
if (!leader)
return -1;
if (!atoi_n_check(leader+10, 1, &indicator_length))
return -1;
if (!atoi_n_check(leader+11, 1, &identifier_length))
return -1;
if (!atoi_n_check(leader+20, 1, &length_data_entry))
return -1;
if (!atoi_n_check(leader+21, 1, &length_starting))
return -1;
if (!atoi_n_check(leader+22, 1, &length_implementation))
return -1;
wr_data_tmp = wrbuf_alloc();
wr_dir = wrbuf_alloc();
for (n = mt->nodes; n; n = n->next)
{
int data_length = 0;
struct yaz_marc_subfield *s;
switch(n->which)
{
case YAZ_MARC_DATAFIELD:
wrbuf_printf(wr_dir, "%.3s", n->u.datafield.tag);
data_length += indicator_length;
wrbuf_rewind(wr_data_tmp);
for (s = n->u.datafield.subfields; s; s = s->next)
{
/* write dummy IDFS + content */
wrbuf_iconv_putchar(wr_data_tmp, mt->iconv_cd, ' ');
wrbuf_iconv_puts(wr_data_tmp, mt->iconv_cd, s->code_data);
marc_iconv_reset(mt, wr_data_tmp);
}
/* write dummy FS (makes MARC-8 to become ASCII) */
wrbuf_iconv_putchar(wr_data_tmp, mt->iconv_cd, ' ');
marc_iconv_reset(mt, wr_data_tmp);
data_length += wrbuf_len(wr_data_tmp);
break;
case YAZ_MARC_CONTROLFIELD:
wrbuf_printf(wr_dir, "%.3s", n->u.controlfield.tag);
wrbuf_rewind(wr_data_tmp);
wrbuf_iconv_puts(wr_data_tmp, mt->iconv_cd,
n->u.controlfield.data);
marc_iconv_reset(mt, wr_data_tmp);
wrbuf_iconv_putchar(wr_data_tmp, mt->iconv_cd, ' ');/* field sep */
marc_iconv_reset(mt, wr_data_tmp);
data_length += wrbuf_len(wr_data_tmp);
break;
case YAZ_MARC_COMMENT:
break;
case YAZ_MARC_LEADER:
break;
}
if (data_length)
{
wrbuf_printf(wr_dir, "%0*d", length_data_entry, data_length);
wrbuf_printf(wr_dir, "%0*d", length_starting, data_offset);
data_offset += data_length;
}
}
/* mark end of directory */
wrbuf_putc(wr_dir, ISO2709_FS);
/* base address of data (comes after leader+directory) */
base_address = 24 + wrbuf_len(wr_dir);
wr_head = wrbuf_alloc();
/* write record length */
wrbuf_printf(wr_head, "%05d", base_address + data_offset + 1);
/* from "original" leader */
wrbuf_write(wr_head, leader+5, 7);
/* base address of data */
wrbuf_printf(wr_head, "%05d", base_address);
/* from "original" leader */
wrbuf_write(wr_head, leader+17, 7);
wrbuf_write(wr, wrbuf_buf(wr_head), 24);
wrbuf_write(wr, wrbuf_buf(wr_dir), wrbuf_len(wr_dir));
wrbuf_destroy(wr_head);
wrbuf_destroy(wr_dir);
wrbuf_destroy(wr_data_tmp);
for (n = mt->nodes; n; n = n->next)
{
struct yaz_marc_subfield *s;
switch(n->which)
{
case YAZ_MARC_DATAFIELD:
wrbuf_write(wr, n->u.datafield.indicator, indicator_length);
for (s = n->u.datafield.subfields; s; s = s->next)
{
wrbuf_putc(wr, ISO2709_IDFS);
wrbuf_iconv_puts(wr, mt->iconv_cd, s->code_data);
marc_iconv_reset(mt, wr);
}
wrbuf_putc(wr, ISO2709_FS);
break;
case YAZ_MARC_CONTROLFIELD:
wrbuf_iconv_puts(wr, mt->iconv_cd, n->u.controlfield.data);
marc_iconv_reset(mt, wr);
wrbuf_putc(wr, ISO2709_FS);
break;
case YAZ_MARC_COMMENT:
break;
case YAZ_MARC_LEADER:
break;
}
}
wrbuf_printf(wr, "%c", ISO2709_RS);
return 0;
}
int yaz_marc_write_xml(yaz_marc_t mt, xmlNode **root_ptr,
const char *ns,
const char *format,
const char *type)
{
struct yaz_marc_node *n;
int identifier_length;
const char *leader = 0;
xmlNode *record_ptr;
xmlNsPtr ns_record;
WRBUF wr_cdata = 0;
for (n = mt->nodes; n; n = n->next)
if (n->which == YAZ_MARC_LEADER)
{
leader = n->u.leader;
break;
}
if (!leader)
return -1;
if (!atoi_n_check(leader+11, 1, &identifier_length))
return -1;
wr_cdata = wrbuf_alloc();
record_ptr = xmlNewNode(0, BAD_CAST "record");
*root_ptr = record_ptr;
ns_record = xmlNewNs(record_ptr, BAD_CAST ns, 0);
xmlSetNs(record_ptr, ns_record);
if (format)
xmlNewProp(record_ptr, BAD_CAST "format", BAD_CAST format);
if (type)
xmlNewProp(record_ptr, BAD_CAST "type", BAD_CAST type);
for (n = mt->nodes; n; n = n->next)
{
struct yaz_marc_subfield *s;
xmlNode *ptr;
switch(n->which)
{
case YAZ_MARC_DATAFIELD:
ptr = xmlNewChild(record_ptr, ns_record, BAD_CAST "datafield", 0);
xmlNewProp(ptr, BAD_CAST "tag", BAD_CAST n->u.datafield.tag);
if (n->u.datafield.indicator)
{
int i;
for (i = 0; n->u.datafield.indicator[i]; i++)
{
char ind_str[6];
char ind_val[2];
sprintf(ind_str, "ind%d", i+1);
ind_val[0] = n->u.datafield.indicator[i];
ind_val[1] = '\0';
xmlNewProp(ptr, BAD_CAST ind_str, BAD_CAST ind_val);
}
}
for (s = n->u.datafield.subfields; s; s = s->next)
{
xmlNode *ptr_subfield;
size_t using_code_len = get_subfield_len(mt, s->code_data,
identifier_length);
wrbuf_rewind(wr_cdata);
wrbuf_iconv_puts(wr_cdata, mt->iconv_cd,
s->code_data + using_code_len);
marc_iconv_reset(mt, wr_cdata);
ptr_subfield = xmlNewTextChild(
ptr, ns_record,
BAD_CAST "subfield", BAD_CAST wrbuf_cstr(wr_cdata));
wrbuf_rewind(wr_cdata);
wrbuf_iconv_write(wr_cdata, mt->iconv_cd,
s->code_data, using_code_len);
xmlNewProp(ptr_subfield, BAD_CAST "code",
BAD_CAST wrbuf_cstr(wr_cdata));
}
break;
case YAZ_MARC_CONTROLFIELD:
wrbuf_rewind(wr_cdata);
wrbuf_iconv_puts(wr_cdata, mt->iconv_cd, n->u.controlfield.data);
marc_iconv_reset(mt, wr_cdata);
ptr = xmlNewTextChild(record_ptr, ns_record,
BAD_CAST "controlfield",
BAD_CAST wrbuf_cstr(wr_cdata));
xmlNewProp(ptr, BAD_CAST "tag", BAD_CAST n->u.controlfield.tag);
break;
case YAZ_MARC_COMMENT:
ptr = xmlNewComment(BAD_CAST n->u.comment);
xmlAddChild(record_ptr, ptr);
break;
case YAZ_MARC_LEADER:
xmlNewTextChild(record_ptr, ns_record, BAD_CAST "leader",
BAD_CAST n->u.leader);
break;
}
}
wrbuf_destroy(wr_cdata);
return 0;
}
int yaz_record_conv_configure_t(yaz_record_conv_t p, const xmlNode *ptr,
struct yaz_record_conv_type *types)
{
struct yaz_record_conv_type bt[4];
size_t i = 0;
/* register marc */
bt[i].construct = construct_marc;
bt[i].convert = convert_marc;
bt[i++].destroy = destroy_marc;
bt[i-1].next = &bt[i];
bt[i].construct = construct_solrmarc;
bt[i].convert = convert_solrmarc;
bt[i++].destroy = destroy_solrmarc;
bt[i-1].next = &bt[i];
bt[i].construct = construct_select;
bt[i].convert = convert_select;
bt[i++].destroy = destroy_select;
#if YAZ_HAVE_XSLT
/* register xslt */
bt[i-1].next = &bt[i];
bt[i].construct = construct_xslt;
bt[i].convert = convert_xslt;
bt[i++].destroy = destroy_xslt;
#endif
bt[i-1].next = types;
yaz_record_conv_reset(p);
/* parsing element children */
for (ptr = ptr->children; ptr; ptr = ptr->next)
{
struct yaz_record_conv_type *t;
struct yaz_record_conv_rule *r;
void *info = 0;
if (ptr->type != XML_ELEMENT_NODE)
continue;
for (t = &bt[0]; t; t = t->next)
{
wrbuf_rewind(p->wr_error);
info = t->construct(ptr, p->path, p->wr_error);
if (info || wrbuf_len(p->wr_error))
break;
/* info== 0 and no error reported , ie not handled by it */
}
if (!info)
{
if (wrbuf_len(p->wr_error) == 0)
wrbuf_printf(p->wr_error, "Element <backend>: "
"unsupported element <%s>", ptr->name);
return -1;
}
r = (struct yaz_record_conv_rule *) nmem_malloc(p->nmem, sizeof(*r));
r->next = 0;
r->info = info;
r->type = nmem_malloc(p->nmem, sizeof(*t));
memcpy(r->type, t, sizeof(*t));
*p->rules_p = r;
p->rules_p = &r->next;
}
return 0;
}
static int convert_marc(void *info, WRBUF record, WRBUF wr_error)
{
struct marc_info *mi = info;
const char *input_charset = mi->input_charset;
int ret = 0;
yaz_marc_t mt = yaz_marc_create();
yaz_marc_xml(mt, mi->output_format_mode);
if (mi->leader_spec)
yaz_marc_leader_spec(mt, mi->leader_spec);
if (mi->input_format_mode == YAZ_MARC_ISO2709)
{
int sz = yaz_marc_read_iso2709(mt, wrbuf_buf(record),
wrbuf_len(record));
if (sz > 0)
{
if (yaz_marc_check_marc21_coding(input_charset, wrbuf_buf(record),
wrbuf_len(record)))
input_charset = "utf-8";
ret = 0;
}
else
ret = -1;
}
else if (mi->input_format_mode == YAZ_MARC_MARCXML ||
mi->input_format_mode == YAZ_MARC_TURBOMARC)
{
xmlDocPtr doc = xmlParseMemory(wrbuf_buf(record),
wrbuf_len(record));
if (!doc)
{
wrbuf_printf(wr_error, "xmlParseMemory failed");
ret = -1;
}
else
{
ret = yaz_marc_read_xml(mt, xmlDocGetRootElement(doc));
if (ret)
wrbuf_printf(wr_error, "yaz_marc_read_xml failed");
}
xmlFreeDoc(doc);
}
else
{
wrbuf_printf(wr_error, "unsupported input format");
ret = -1;
}
if (ret == 0)
{
yaz_iconv_t cd = yaz_iconv_open(mi->output_charset, input_charset);
if (cd)
yaz_marc_iconv(mt, cd);
wrbuf_rewind(record);
ret = yaz_marc_write_mode(mt, record);
if (ret)
wrbuf_printf(wr_error, "yaz_marc_write_mode failed");
if (cd)
yaz_iconv_close(cd);
}
yaz_marc_destroy(mt);
return ret;
}
static int rpn2solr_simple(solr_transform_t ct,
void (*pr)(const char *buf, void *client_data),
void *client_data,
Z_AttributesPlusTerm *apt, WRBUF w,
Z_AttributesPlusTerm *apt2)
{
int ret = 0;
Z_Term *term = apt->term;
Odr_int trunc = get_truncation(apt);
const char *relation2 = 0;
const char *relation1 = solr_lookup_reverse(ct, "relation.",
apt->attributes);
/* Attempt to fix bug #2978: Look for a relation attribute */
if (!relation1)
relation1 = lookup_relation_index_from_attr(apt->attributes);
if (!relation1)
{
return YAZ_BIB1_UNSUPP_RELATION_ATTRIBUTE;
}
if (apt2)
{
relation2 = solr_lookup_reverse(ct, "relation.",
apt2->attributes);
if (!relation2)
relation2 = lookup_relation_index_from_attr(apt2->attributes);
}
wrbuf_rewind(w);
ret = rpn2solr_attr(ct, apt->attributes, w);
if (ret)
return ret;
if ((trunc >= 0 && trunc <= 3) || trunc == 100 || trunc == 104)
;
else
{
return YAZ_BIB1_UNSUPP_TRUNCATION_ATTRIBUTE;
}
if (!relation1)
ret = emit_term(ct, w, term, trunc);
else if (relation1[0] == '<' || relation1[0] == 'l')
{
wrbuf_puts(w, "[* TO ");
ret = emit_term(ct, w, term, trunc);
if (!strcmp(relation1, "le") || !strcmp(relation1, "<="))
wrbuf_puts(w, "]");
else
wrbuf_puts(w, "}");
}
else if (relation1[0] == '>' || relation1[0] == 'g')
{
if (!strcmp(relation1, ">=") || !strcmp(relation1, "ge"))
wrbuf_puts(w, "[");
else
wrbuf_puts(w, "{");
ret = emit_term(ct, w, term, trunc);
wrbuf_puts(w, " TO ");
if (apt2)
{
emit_term(ct, w, apt2->term, 0);
if (!relation2 || !strcmp(relation2, "<=") ||
!strcmp(relation2, "le"))
wrbuf_puts(w, "]");
else
wrbuf_puts(w, "}");
}
else
wrbuf_puts(w, "*]");
}
else
ret = emit_term(ct, w, term, trunc);
if (ret == 0)
pr(wrbuf_cstr(w), client_data);
return ret;
}
// Prepare for a relevance-sorted read
void relevance_prepare_read(struct relevance *rel, struct reclist *reclist)
{
int i;
float *idfvec = xmalloc(rel->vec_len * sizeof(float));
reclist_enter(reclist);
// Calculate document frequency vector for each term.
for (i = 1; i < rel->vec_len; i++)
{
if (!rel->doc_frequency_vec[i])
idfvec[i] = 0;
else
{
/* add one to nominator idf(t,D) to ensure a value > 0 */
idfvec[i] = log((float) (1 + rel->doc_frequency_vec[0]) /
rel->doc_frequency_vec[i]);
}
}
// Calculate relevance for each document
while (1)
{
int relevance = 0;
WRBUF w;
struct word_entry *e = rel->entries;
struct record_cluster *rec = reclist_read_record(reclist);
if (!rec)
break;
w = rec->relevance_explain2;
wrbuf_rewind(w);
wrbuf_puts(w, "relevance = 0;\n");
for (i = 1; i < rel->vec_len; i++)
{
float termfreq = (float) rec->term_frequency_vecf[i];
int add = 100000 * termfreq * idfvec[i];
wrbuf_printf(w, "idf[%d] = log(((1 + total(%d))/termoccur(%d));\n",
i, rel->doc_frequency_vec[0],
rel->doc_frequency_vec[i]);
wrbuf_printf(w, "%s: relevance += 100000 * tf[%d](%f) * "
"idf[%d](%f) (%d);\n",
e->display_str, i, termfreq, i, idfvec[i], add);
relevance += add;
e = e->next;
}
if (!rel->rank_cluster)
{
struct record *record;
int cluster_size = 0;
for (record = rec->records; record; record = record->next)
cluster_size++;
wrbuf_printf(w, "score = relevance(%d)/cluster_size(%d);\n",
relevance, cluster_size);
relevance /= cluster_size;
}
else
{
wrbuf_printf(w, "score = relevance(%d);\n", relevance);
}
rec->relevance_score = relevance;
}
reclist_leave(reclist);
xfree(idfvec);
}
static int tst_convert_x(yaz_iconv_t cd, const char *buf, const char *cmpbuf,
int expect_error)
{
int ret = 1;
WRBUF b = wrbuf_alloc();
char outbuf[16];
size_t inbytesleft = strlen(buf);
const char *inp = buf;
int rounds = 0;
for (rounds = 0; inbytesleft && rounds < (int) sizeof(outbuf); rounds++)
{
size_t outbytesleft = sizeof(outbuf);
char *outp = outbuf;
size_t r = yaz_iconv(cd, (char**) &inp, &inbytesleft,
&outp, &outbytesleft);
wrbuf_write(b, outbuf, outp - outbuf);
if (r == (size_t) (-1))
{
int e = yaz_iconv_error(cd);
if (e != YAZ_ICONV_E2BIG)
{
if (expect_error != -1)
if (e != expect_error)
ret = 0;
break;
}
}
else
{
size_t outbytesleft = sizeof(outbuf);
char *outp = outbuf;
r = yaz_iconv(cd, 0, 0, &outp, &outbytesleft);
wrbuf_write(b, outbuf, outp - outbuf);
if (expect_error != -1)
if (expect_error)
ret = 0;
break;
}
}
if (wrbuf_len(b) == strlen(cmpbuf)
&& !memcmp(cmpbuf, wrbuf_buf(b), wrbuf_len(b)))
;
else
{
WRBUF w = wrbuf_alloc();
ret = 0;
wrbuf_rewind(w);
wrbuf_puts_escaped(w, buf);
yaz_log(YLOG_LOG, "input %s", wrbuf_cstr(w));
wrbuf_rewind(w);
wrbuf_write_escaped(w, wrbuf_buf(b), wrbuf_len(b));
yaz_log(YLOG_LOG, "got %s", wrbuf_cstr(w));
wrbuf_rewind(w);
wrbuf_puts_escaped(w, cmpbuf);
yaz_log(YLOG_LOG, "exp %s", wrbuf_cstr(w));
wrbuf_destroy(w);
}
wrbuf_destroy(b);
return ret;
}
static void tstwrbuf(void)
{
int step;
WRBUF wr;
wr = 0;
wrbuf_destroy(wr);
wr = wrbuf_alloc();
YAZ_CHECK(wr);
wrbuf_destroy(wr);
wr = wrbuf_alloc();
YAZ_CHECK(wr);
for (step = 1; step < 65; step++)
{
int i, j, k;
int len;
char buf[64];
char *cp;
for (j = 1; j<step; j++)
{
for (i = 0; i<j; i++)
buf[i] = i+1;
buf[i] = '\0';
wrbuf_puts(wr, buf);
}
cp = wrbuf_buf(wr);
len = wrbuf_len(wr);
YAZ_CHECK(len == step * (step-1) / 2);
k = 0;
for (j = 1; j<step; j++)
for (i = 0; i<j; i++)
{
YAZ_CHECK(cp[k] == i+1);
k++;
}
wrbuf_rewind(wr);
}
wrbuf_rewind(wr);
wrbuf_puts(wr, "1234");
wrbuf_insert(wr, 0, "abc", 3);
YAZ_CHECK(!strcmp(wrbuf_cstr(wr), "abc1234"));
wrbuf_rewind(wr);
wrbuf_puts(wr, "1234");
wrbuf_insert(wr, 1, "abc", 3);
YAZ_CHECK(!strcmp(wrbuf_cstr(wr), "1abc234"));
wrbuf_rewind(wr);
wrbuf_puts(wr, "1234");
wrbuf_insert(wr, 4, "abc", 3);
YAZ_CHECK(!strcmp(wrbuf_cstr(wr), "1234abc"));
wrbuf_rewind(wr);
wrbuf_puts(wr, "1234");
wrbuf_insert(wr, 5, "abc", 3);
YAZ_CHECK(!strcmp(wrbuf_cstr(wr), "1234"));
wrbuf_destroy(wr);
}
static void marcdump_read_xml(yaz_marc_t mt, const char *fname)
{
WRBUF wrbuf = wrbuf_alloc();
#if USE_XMLREADER
xmlTextReaderPtr reader = xmlReaderForFile(fname, 0 /* encoding */,
0 /* options */);
if (reader)
{
int ret;
while ((ret = xmlTextReaderRead(reader)) == 1)
{
int type = xmlTextReaderNodeType(reader);
if (type == XML_READER_TYPE_ELEMENT)
{
const char *name = (const char *)
xmlTextReaderLocalName(reader);
if (!strcmp(name, "record") || !strcmp(name, "r"))
{
xmlNodePtr ptr = xmlTextReaderExpand(reader);
int r = yaz_marc_read_xml(mt, ptr);
if (r)
{
no_errors++;
fprintf(stderr, "yaz_marc_read_xml failed\n");
}
else
{
int write_rc = yaz_marc_write_mode(mt, wrbuf);
if (write_rc)
{
yaz_log(YLOG_WARN, "yaz_marc_write_mode: "
"write error: %d", write_rc);
no_errors++;
}
fputs(wrbuf_cstr(wrbuf), stdout);
wrbuf_rewind(wrbuf);
}
}
}
}
}
#else
xmlDocPtr doc = xmlParseFile(fname);
if (doc)
{
xmlNodePtr ptr = xmlDocGetRootElement(doc);
for (; ptr; ptr = ptr->next)
{
if (ptr->type == XML_ELEMENT_NODE)
{
if (!strcmp((const char *) ptr->name, "collection"))
{
ptr = ptr->children;
continue;
}
if (!strcmp((const char *) ptr->name, "record") ||
!strcmp((const char *) ptr->name, "r"))
{
int r = yaz_marc_read_xml(mt, ptr);
if (r)
{
no_errors++;
fprintf(stderr, "yaz_marc_read_xml failed\n");
}
else
{
yaz_marc_write_mode(mt, wrbuf);
fputs(wrbuf_cstr(wrbuf), stdout);
wrbuf_rewind(wrbuf);
}
}
}
}
xmlFreeDoc(doc);
}
#endif
fputs(wrbuf_cstr(wrbuf), stdout);
wrbuf_destroy(wrbuf);
}
Z_HTTP_Response *yaz_url_exec(yaz_url_t p, const char *uri,
const char *method,
Z_HTTP_Header *user_headers,
const char *buf, size_t len)
{
Z_HTTP_Response *res = 0;
int number_of_redirects = 0;
yaz_cookies_reset(p->cookies);
wrbuf_rewind(p->w_error);
while (1)
{
void *add;
COMSTACK conn = 0;
int code;
const char *location = 0;
char *http_user = 0;
char *http_pass = 0;
char *uri_lean = 0;
Z_GDU *gdu;
extract_user_pass(p->odr_out->mem, uri, &uri_lean,
&http_user, &http_pass);
gdu = z_get_HTTP_Request_uri(p->odr_out, uri_lean, 0, p->proxy ? 1 : 0);
gdu->u.HTTP_Request->method = odr_strdup(p->odr_out, method);
yaz_cookies_request(p->cookies, p->odr_out, gdu->u.HTTP_Request);
for ( ; user_headers; user_headers = user_headers->next)
{
/* prefer new Host over user-supplied Host */
if (!strcmp(user_headers->name, "Host"))
;
/* prefer user-supplied User-Agent over YAZ' own */
else if (!strcmp(user_headers->name, "User-Agent"))
z_HTTP_header_set(p->odr_out, &gdu->u.HTTP_Request->headers,
user_headers->name, user_headers->value);
else
z_HTTP_header_add(p->odr_out, &gdu->u.HTTP_Request->headers,
user_headers->name, user_headers->value);
}
if (http_user && http_pass)
z_HTTP_header_add_basic_auth(p->odr_out,
&gdu->u.HTTP_Request->headers,
http_user, http_pass);
res = 0;
if (buf && len)
{
gdu->u.HTTP_Request->content_buf = (char *) buf;
gdu->u.HTTP_Request->content_len = len;
}
if (!z_GDU(p->odr_out, &gdu, 0, 0))
{
wrbuf_printf(p->w_error, "Can not encode HTTP request for URL %s",
uri);
log_warn(p);
return 0;
}
conn = cs_create_host_proxy(uri_lean, 1, &add, p->proxy);
if (!conn)
{
wrbuf_printf(p->w_error, "Can not resolve URL %s", uri);
log_warn(p);
}
else if (cs_connect(conn, add) < 0)
{
wrbuf_printf(p->w_error, "Can not connect to URL %s", uri);
log_warn(p);
}
else
{
int len;
char *buf = odr_getbuf(p->odr_out, &len, 0);
if (p->verbose)
fwrite(buf, 1, len, stdout);
if (cs_put(conn, buf, len) < 0)
{
wrbuf_printf(p->w_error, "cs_put fail for URL %s", uri);
log_warn(p);
}
else
{
char *netbuffer = 0;
int netlen = 0;
int cs_res = cs_get(conn, &netbuffer, &netlen);
if (cs_res <= 0)
{
wrbuf_printf(p->w_error, "cs_get failed for URL %s", uri);
log_warn(p);
}
else
{
Z_GDU *gdu;
if (p->verbose)
fwrite(netbuffer, 1, cs_res, stdout);
odr_setbuf(p->odr_in, netbuffer, cs_res, 0);
if (!z_GDU(p->odr_in, &gdu, 0, 0)
|| gdu->which != Z_GDU_HTTP_Response)
{
wrbuf_printf(p->w_error, "HTTP decoding fail for "
"URL %s", uri);
log_warn(p);
}
else
{
res = gdu->u.HTTP_Response;
}
}
xfree(netbuffer);
}
}
if (conn)
cs_close(conn);
if (!res)
break;
code = res->code;
location = z_HTTP_header_lookup(res->headers, "Location");
if (++number_of_redirects <= p->max_redirects &&
location && (code == 301 || code == 302 || code == 307))
{
int host_change = 0;
const char *nlocation = yaz_check_location(p->odr_in, uri,
location, &host_change);
odr_reset(p->odr_out);
uri = odr_strdup(p->odr_out, nlocation);
}
else
break;
yaz_cookies_response(p->cookies, res);
odr_reset(p->odr_in);
}
return res;
}
static int rpn2solr_simple(solr_transform_t ct,
void (*pr)(const char *buf, void *client_data),
void *client_data,
Z_Operand *q, WRBUF w)
{
int ret = 0;
if (q->which != Z_Operand_APT)
{
ret = -1;
solr_transform_set_error(ct, YAZ_BIB1_RESULT_SET_UNSUPP_AS_A_SEARCH_TERM, 0);
}
else
{
Z_AttributesPlusTerm *apt = q->u.attributesPlusTerm;
Z_Term *term = apt->term;
const char *sterm = 0;
size_t lterm = 0;
Odr_int trunc = get_truncation(apt);
wrbuf_rewind(w);
ret = rpn2solr_attr(ct, apt->attributes, w);
if (trunc == 0 || trunc == 1 || trunc == 100 || trunc == 104)
;
else
{
solr_transform_set_error(ct, YAZ_BIB1_UNSUPP_TRUNCATION_ATTRIBUTE, 0);
return -1;
}
switch (term->which)
{
case Z_Term_general:
lterm = term->u.general->len;
sterm = (const char *) term->u.general->buf;
break;
case Z_Term_numeric:
wrbuf_printf(w, ODR_INT_PRINTF, *term->u.numeric);
break;
case Z_Term_characterString:
sterm = term->u.characterString;
lterm = strlen(sterm);
break;
default:
ret = -1;
solr_transform_set_error(ct, YAZ_BIB1_TERM_TYPE_UNSUPP, 0);
}
if (sterm)
{
size_t i;
int must_quote = 0;
for (i = 0 ; i < lterm; i++)
if (sterm[i] == ' ')
must_quote = 1;
if (must_quote)
wrbuf_puts(w, "\"");
for (i = 0 ; i < lterm; i++)
{
if (sterm[i] == '\\' && i < lterm - 1)
{
i++;
if (strchr(SOLR_SPECIAL, sterm[i]))
wrbuf_putc(w, '\\');
wrbuf_putc(w, sterm[i]);
}
else if (sterm[i] == '?' && trunc == 104)
{
wrbuf_putc(w, '*');
}
else if (sterm[i] == '#' && trunc == 104)
{
wrbuf_putc(w, '?');
}
else if (strchr(SOLR_SPECIAL, sterm[i]))
{
wrbuf_putc(w, '\\');
wrbuf_putc(w, sterm[i]);
}
else
wrbuf_putc(w, sterm[i]);
}
if (trunc == 1)
wrbuf_puts(w, "*");
if (must_quote)
wrbuf_puts(w, "\"");
}
if (ret == 0)
pr(wrbuf_cstr(w), client_data);
}
return ret;
}
static void show_record(struct http_channel *c, struct http_session *s)
{
struct http_response *rs = c->response;
struct http_request *rq = c->request;
struct record_cluster *rec, *prev_r, *next_r;
struct conf_service *service;
const char *idstr = http_argbyname(rq, "id");
const char *offsetstr = http_argbyname(rq, "offset");
const char *binarystr = http_argbyname(rq, "binary");
const char *checksumstr = http_argbyname(rq, "checksum");
if (!s)
return;
service = s->psession->service;
if (!idstr)
{
error(rs, PAZPAR2_MISSING_PARAMETER, "id");
return;
}
wrbuf_rewind(c->wrbuf);
if (!(rec = show_single_start(s->psession, idstr, &prev_r, &next_r)))
{
if (session_active_clients(s->psession) == 0)
{
error(rs, PAZPAR2_RECORD_MISSING, idstr);
}
else if (session_set_watch(s->psession, SESSION_WATCH_RECORD,
cmd_record_ready, c, c) != 0)
{
error(rs, PAZPAR2_RECORD_MISSING, idstr);
}
return;
}
if (offsetstr || checksumstr)
{
const char *syntax = http_argbyname(rq, "syntax");
const char *esn = http_argbyname(rq, "esn");
int i;
struct record*r = rec->records;
int binary = 0;
const char *nativesyntax = http_argbyname(rq, "nativesyntax");
if (binarystr && *binarystr != '0')
binary = 1;
if (checksumstr)
{
long v = atol(checksumstr);
for (i = 0; r; r = r->next)
if (v == r->checksum)
break;
if (!r)
error(rs, PAZPAR2_RECORD_FAIL, "no record");
}
else
{
int offset = atoi(offsetstr);
for (i = 0; i < offset && r; r = r->next, i++)
;
if (!r)
error(rs, PAZPAR2_RECORD_FAIL, "no record at offset given");
}
if (r)
{
http_channel_observer_t obs =
http_add_observer(c, r->client, show_raw_reset);
int ret = client_show_raw_begin(r->client, r->position,
syntax, esn,
obs /* data */,
show_raw_record_error,
(binary ?
show_raw_record_ok_binary :
show_raw_record_ok),
(binary ? 1 : 0),
nativesyntax);
if (ret == -1)
{
http_remove_observer(obs);
error(rs, PAZPAR2_NO_SESSION, 0);
}
}
}
else
{
struct record *r;
response_open_no_status(c, "record");
wrbuf_puts(c->wrbuf, "\n <recid>");
wrbuf_xmlputs(c->wrbuf, rec->recid);
wrbuf_puts(c->wrbuf, "</recid>\n");
if (prev_r)
{
wrbuf_puts(c->wrbuf, " <prevrecid>");
wrbuf_xmlputs(c->wrbuf, prev_r->recid);
wrbuf_puts(c->wrbuf, "</prevrecid>\n");
}
if (next_r)
{
wrbuf_puts(c->wrbuf, " <nextrecid>");
wrbuf_xmlputs(c->wrbuf, next_r->recid);
wrbuf_puts(c->wrbuf, "</nextrecid>\n");
}
wrbuf_printf(c->wrbuf, " <activeclients>%d</activeclients>\n",
session_active_clients(s->psession));
write_metadata(c->wrbuf, service, rec->metadata, 1, 1);
for (r = rec->records; r; r = r->next)
write_subrecord(r, c->wrbuf, service, 2);
response_close(c, "record");
}
show_single_stop(s->psession, rec);
}
static void response_open_no_status(struct http_channel *c, const char *command)
{
wrbuf_rewind(c->wrbuf);
wrbuf_printf(c->wrbuf, "%s<%s>",
HTTP_COMMAND_RESPONSE_PREFIX, command);
}
static int yaz_marc_write_xml_turbo_xml(yaz_marc_t mt, xmlNode **root_ptr,
const char *ns,
const char *format,
const char *type)
{
struct yaz_marc_node *n;
int identifier_length;
const char *leader = 0;
xmlNode *record_ptr;
xmlNsPtr ns_record;
WRBUF wr_cdata = 0;
for (n = mt->nodes; n; n = n->next)
if (n->which == YAZ_MARC_LEADER)
{
leader = n->u.leader;
break;
}
if (!leader)
return -1;
if (!atoi_n_check(leader+11, 1, &identifier_length))
return -1;
wr_cdata = wrbuf_alloc();
record_ptr = xmlNewNode(0, BAD_CAST "r");
*root_ptr = record_ptr;
ns_record = xmlNewNs(record_ptr, BAD_CAST ns, 0);
xmlSetNs(record_ptr, ns_record);
if (format)
xmlNewProp(record_ptr, BAD_CAST "format", BAD_CAST format);
if (type)
xmlNewProp(record_ptr, BAD_CAST "type", BAD_CAST type);
for (n = mt->nodes; n; n = n->next)
{
xmlNode *ptr;
char field[10];
field[0] = 'c';
field[4] = '\0';
switch(n->which)
{
case YAZ_MARC_DATAFIELD:
add_marc_datafield_turbo_xml(mt, n, record_ptr, ns_record, wr_cdata, identifier_length);
break;
case YAZ_MARC_CONTROLFIELD:
wrbuf_rewind(wr_cdata);
wrbuf_iconv_puts(wr_cdata, mt->iconv_cd, n->u.controlfield.data);
marc_iconv_reset(mt, wr_cdata);
strncpy(field + 1, n->u.controlfield.tag, 3);
ptr = xmlNewTextChild(record_ptr, ns_record,
BAD_CAST field,
BAD_CAST wrbuf_cstr(wr_cdata));
break;
case YAZ_MARC_COMMENT:
ptr = xmlNewComment(BAD_CAST n->u.comment);
xmlAddChild(record_ptr, ptr);
break;
case YAZ_MARC_LEADER:
xmlNewTextChild(record_ptr, ns_record, BAD_CAST "l",
BAD_CAST n->u.leader);
break;
}
}
wrbuf_destroy(wr_cdata);
return 0;
}
static void process_text_file(struct config_t *p_config)
{
char *line = 0;
char linebuf[1024];
xmlDoc *doc = xmlParseFile(p_config->conffile);
xmlNode *xml_node = xmlDocGetRootElement(doc);
long unsigned int token_count = 0;
long unsigned int line_count = 0;
UErrorCode status = U_ZERO_ERROR;
if (!xml_node)
{
printf("Could not parse XML config file '%s' \n",
p_config->conffile);
exit(1);
}
p_config->chain = icu_chain_xml_config(xml_node, 1, &status);
if (!p_config->chain || !U_SUCCESS(status))
{
printf("Could not set up ICU chain from config file '%s' \n",
p_config->conffile);
exit(1);
}
if (p_config->xmloutput)
fprintf(p_config->outfile,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
"<icu>\n"
"<tokens>\n");
/* read input lines for processing */
while ((line=fgets(linebuf, sizeof(linebuf)-1, p_config->infile)))
{
WRBUF sw = wrbuf_alloc();
WRBUF cdata = wrbuf_alloc();
int success = icu_chain_assign_cstr(p_config->chain, line, &status);
line_count++;
while (success && icu_chain_next_token(p_config->chain, &status))
{
if (U_FAILURE(status))
success = 0;
else
{
size_t start, len;
const char *org_string = 0;
const char *sortkey = icu_chain_token_sortkey(p_config->chain);
icu_chain_get_org_info2(p_config->chain, &start, &len,
&org_string);
wrbuf_rewind(sw);
wrbuf_puts_escaped(sw, sortkey);
token_count++;
if (p_config->xmloutput)
{
fprintf(p_config->outfile,
"<token id=\"%lu\" line=\"%lu\"",
token_count, line_count);
wrbuf_rewind(cdata);
wrbuf_xmlputs(cdata, icu_chain_token_norm(p_config->chain));
fprintf(p_config->outfile, " norm=\"%s\"",
wrbuf_cstr(cdata));
wrbuf_rewind(cdata);
wrbuf_xmlputs(cdata, icu_chain_token_display(p_config->chain));
fprintf(p_config->outfile, " display=\"%s\"",
wrbuf_cstr(cdata));
if (p_config->sortoutput)
{
wrbuf_rewind(cdata);
wrbuf_xmlputs(cdata, wrbuf_cstr(sw));
fprintf(p_config->outfile, " sortkey=\"%s\"",
wrbuf_cstr(cdata));
}
fprintf(p_config->outfile, "/>\n");
}
else
{
fprintf(p_config->outfile, "%lu %lu '%s' '%s'",
token_count,
line_count,
icu_chain_token_norm(p_config->chain),
icu_chain_token_display(p_config->chain));
if (p_config->sortoutput)
{
fprintf(p_config->outfile, " '%s'", wrbuf_cstr(sw));
}
if (p_config->org_output)
{
fprintf(p_config->outfile, " %ld+%ld",
(long) start, (long) len);
fputc(' ', p_config->outfile);
fwrite(org_string, 1, start, p_config->outfile);
fputc('*', p_config->outfile);
fwrite(org_string + start, 1, len, p_config->outfile);
fputc('*', p_config->outfile);
fputs(org_string + start + len, p_config->outfile);
}
fprintf(p_config->outfile, "\n");
}
}
}
wrbuf_destroy(sw);
wrbuf_destroy(cdata);
}
if (p_config->xmloutput)
fprintf(p_config->outfile,
"</tokens>\n"
"</icu>\n");
icu_chain_destroy(p_config->chain);
xmlFreeDoc(doc);
if (line)
free(line);
}
static int rpn2cql_simple(cql_transform_t ct,
void (*pr)(const char *buf, void *client_data),
void *client_data,
Z_Operand *q, WRBUF w)
{
int ret = 0;
if (q->which != Z_Operand_APT)
{
ret = -1;
cql_transform_set_error(ct, YAZ_BIB1_RESULT_SET_UNSUPP_AS_A_SEARCH_TERM, 0);
}
else
{
Z_AttributesPlusTerm *apt = q->u.attributesPlusTerm;
Z_Term *term = apt->term;
const char *sterm = 0;
size_t lterm = 0;
Odr_int trunc = lookup_truncation(apt->attributes);
size_t i;
wrbuf_rewind(w);
ret = rpn2cql_attr(ct, apt->attributes, w);
switch (term->which)
{
case Z_Term_general:
lterm = term->u.general->len;
sterm = (const char *) term->u.general->buf;
break;
case Z_Term_numeric:
wrbuf_printf(w, ODR_INT_PRINTF, *term->u.numeric);
break;
case Z_Term_characterString:
sterm = term->u.characterString;
lterm = strlen(sterm);
break;
default:
cql_transform_set_error(ct, YAZ_BIB1_TERM_TYPE_UNSUPP, 0);
return -1;
}
if (trunc <= 3 || trunc == 100 || trunc == 102 || trunc == 104)
{
int quote_it = 0;
for (i = 0 ; i < lterm; i++)
if (strchr(" ()=></", sterm[i]))
{
quote_it = 1;
break;
}
if (lterm == 0)
quote_it = 1;
if (quote_it)
wrbuf_puts(w, "\"");
if (trunc == 2 || trunc == 3)
wrbuf_puts(w, "*");
for (i = 0; i < lterm; i++)
{
if (sterm[i] == '\\' && i < lterm - 1)
{
i++;
if (strchr("*?\"\\", sterm[i]))
wrbuf_putc(w, '\\');
wrbuf_putc(w, sterm[i]);
}
else if (trunc == 102 && sterm[i] == '.' && sterm[i+1] == '*')
{
wrbuf_putc(w, '*');
i++;
}
else if (trunc == 102 && sterm[i] == '.')
wrbuf_putc(w, '?');
else if (trunc == 104 && sterm[i] == '?')
wrbuf_putc(w, '*');
else if (trunc == 104 && sterm[i] == '#')
wrbuf_putc(w, '?');
else if (strchr("*?\"", sterm[i]))
{
wrbuf_putc(w, '\\');
wrbuf_putc(w, sterm[i]);
}
else
wrbuf_putc(w, sterm[i]);
}
if (trunc == 1 || trunc == 3)
wrbuf_puts(w, "*");
if (quote_it)
wrbuf_puts(w, "\"");
}
else
{
cql_transform_set_error(
ct, YAZ_BIB1_UNSUPP_TRUNCATION_ATTRIBUTE, 0);
ret = -1;
}
if (ret == 0)
pr(wrbuf_cstr(w), client_data);
}
return ret;
}
