void pp2_charset_token_destroy(pp2_charset_token_t prt)
{
assert(prt);
#if YAZ_HAVE_ICU
if (prt->iter)
icu_iter_destroy(prt->iter);
#endif
if(prt->norm_str)
wrbuf_destroy(prt->norm_str);
if(prt->sort_str)
wrbuf_destroy(prt->sort_str);
xfree(prt);
}
void yaz_query_charset_convert_rpnquery(Z_RPNQuery *q,
ODR o, yaz_iconv_t cd)
{
WRBUF wrbuf = wrbuf_alloc();
yaz_query_charset_convert_structure(q->RPNStructure, o, wrbuf, cd);
wrbuf_destroy(wrbuf);
}
static int compare(cql_transform_t ct, const char *pqf, const char *cql)
{
int ret = 0;
ODR odr = odr_createmem(ODR_ENCODE);
WRBUF w = wrbuf_alloc();
Z_RPNQuery *q = p_query_rpn(odr, pqf);
if (q)
{
int r = cql_transform_rpn2cql_wrbuf(ct, w, q);
if (r != 0)
{
/* transform error */
yaz_log(YLOG_LOG, "%s -> Error %d", pqf, r);
if (!cql) /* also expected error? */
ret = 1;
}
else if (r == 0)
{
yaz_log(YLOG_LOG, "%s -> %s", pqf, wrbuf_cstr(w));
if (cql && !strcmp(wrbuf_cstr(w), cql))
{
ret = 1;
}
}
}
wrbuf_destroy(w);
odr_destroy(odr);
return ret;
}
static void tst2(void)
{
WRBUF w = wrbuf_alloc();
cql_transform_t ct = 0;
const char *srcdir = getenv("srcdir");
if (srcdir)
{
wrbuf_puts(w, srcdir);
wrbuf_puts(w, "/");
}
wrbuf_puts(w, "../etc/pqf.properties");
ct = cql_transform_open_fname(wrbuf_cstr(w));
YAZ_CHECK(compare(ct, "@attr 1=4 abc", "dc.title=abc"));
YAZ_CHECK(compare(ct, "@attr 1=4 @attr 4=108 abc", "dc.title=/exact abc"));
YAZ_CHECK(compare(ct, "@attr 1=4 @attr 3=1 @attr 6=1 abc", "dc.title=abc"));
YAZ_CHECK(compare(ct, "@attr 1=4 @attr 4=1 @attr 6=1 abc",
"dc.title=abc"));
YAZ_CHECK(compare(ct, "@attr 1=1016 abc", "abc"));
YAZ_CHECK(compare(ct, "@attr 2=1 @attr 1=30 1980", "dc.date<1980"));
YAZ_CHECK(compare(ct, "@attr 1=30 @attr 2=3 1980", "dc.date=1980"));
YAZ_CHECK(compare(ct, "@attr 1=30 @attr 2=5 1980", "dc.date>1980"));
YAZ_CHECK(compare(ct, "@attr 1=30 @attr 2=2 1980", "dc.date<=1980"));
YAZ_CHECK(compare(ct, "@attr 1=30 @attr 2=4 1980", "dc.date>=1980"));
YAZ_CHECK(compare(ct, "@attr 2=103 @attr 1=_ALLRECORDS 1", "cql.allRecords=1"));
YAZ_CHECK(compare(ct, "@attr 1=500 abc", 0));
cql_transform_close(ct);
wrbuf_destroy(w);
}
void yaz_query_charset_convert_apt(Z_AttributesPlusTerm *apt,
ODR o, yaz_iconv_t cd)
{
WRBUF wrbuf = wrbuf_alloc();
yaz_query_charset_convert_term(apt->term, o, wrbuf, cd);
wrbuf_destroy(wrbuf);
}
int conv_configure_test(const char *xmlstring, const char *expect_error,
yaz_record_conv_t *pt)
{
WRBUF w = wrbuf_alloc();
int ret;
yaz_record_conv_t p = conv_configure(xmlstring, w);
if (!p)
{
if (expect_error && !strcmp(wrbuf_cstr(w), expect_error))
ret = 1;
else
{
ret = 0;
printf("%s\n", wrbuf_cstr(w));
}
}
else
{
if (expect_error)
ret = 0;
else
ret = 1;
}
if (pt)
*pt = p;
else
if (p)
yaz_record_conv_destroy(p);
wrbuf_destroy(w);
return ret;
}
int main(int argc, char **argv)
{
struct json_node *n;
int print = 0;
int ret;
char *arg;
while ((ret = options("p", argv, argc, &arg)) != YAZ_OPTIONS_EOF)
{
switch (ret)
{
case 'p':
print++;
break;
default:
usage(argv[0]);
}
}
n = do_parse_from_stdin();
if (!n)
exit(1);
if (print)
{
WRBUF result = wrbuf_alloc();
if (print > 1)
json_write_wrbuf_pretty(n, result);
else
json_write_wrbuf(n, result);
puts(wrbuf_cstr(result));
wrbuf_destroy(result);
}
json_remove_node(n);
return 0;
}
void http_set_proxyaddr(const char *host, struct conf_server *server)
{
const char *p;
short port;
struct hostent *he;
WRBUF w = wrbuf_alloc();
yaz_log(YLOG_LOG, "HTTP backend %s", host);
p = strchr(host, ':');
if (p)
{
port = atoi(p + 1);
wrbuf_write(w, host, p - host);
wrbuf_puts(w, "");
}
else
{
port = 80;
wrbuf_puts(w, host);
}
if (!(he = gethostbyname(wrbuf_cstr(w))))
{
fprintf(stderr, "Failed to lookup '%s'\n", wrbuf_cstr(w));
exit(1);
}
wrbuf_destroy(w);
server->http_server->proxy_addr = xmalloc(sizeof(struct sockaddr_in));
server->http_server->proxy_addr->sin_family = he->h_addrtype;
memcpy(&server->http_server->proxy_addr->sin_addr.s_addr,
he->h_addr_list[0], he->h_length);
server->http_server->proxy_addr->sin_port = htons(port);
}
void yaz_marc_destroy(yaz_marc_t mt)
{
if (!mt)
return ;
nmem_destroy(mt->nmem);
wrbuf_destroy(mt->m_wr);
xfree(mt->leader_spec);
xfree(mt);
}
int yaz_query_charset_convert_rpnquery_check(Z_RPNQuery *q,
ODR o, yaz_iconv_t cd)
{
int r = 0;
WRBUF wrbuf = wrbuf_alloc();
r = yaz_query_charset_convert_structure(q->RPNStructure, o, wrbuf, cd);
wrbuf_destroy(wrbuf);
return r;
}
void add_marc_datafield_turbo_xml(yaz_marc_t mt, struct yaz_marc_node *n,
xmlNode *record_ptr,
xmlNsPtr ns_record, WRBUF wr_cdata,
int identifier_length)
{
xmlNode *ptr;
struct yaz_marc_subfield *s;
WRBUF subfield_name = wrbuf_alloc();
/* TODO consider if safe */
char field[10];
field[0] = 'd';
strncpy(field + 1, n->u.datafield.tag, 3);
field[4] = '\0';
ptr = xmlNewChild(record_ptr, ns_record, BAD_CAST field, 0);
if (n->u.datafield.indicator)
{
int i;
for (i = 0; n->u.datafield.indicator[i]; i++)
{
char ind_str[6];
char ind_val[2];
ind_val[0] = n->u.datafield.indicator[i];
ind_val[1] = '\0';
sprintf(ind_str, "%s%d", indicator_name[1], i+1);
xmlNewProp(ptr, BAD_CAST ind_str, BAD_CAST ind_val);
}
}
for (s = n->u.datafield.subfields; s; s = s->next)
{
int not_written;
xmlNode *ptr_subfield;
size_t using_code_len = get_subfield_len(mt, s->code_data,
identifier_length);
wrbuf_rewind(wr_cdata);
wrbuf_iconv_puts(wr_cdata, mt->iconv_cd, s->code_data + using_code_len);
marc_iconv_reset(mt, wr_cdata);
wrbuf_rewind(subfield_name);
wrbuf_puts(subfield_name, "s");
not_written = element_name_append_attribute_value(mt, subfield_name, 0, s->code_data, using_code_len) != 0;
ptr_subfield = xmlNewTextChild(ptr, ns_record,
BAD_CAST wrbuf_cstr(subfield_name),
BAD_CAST wrbuf_cstr(wr_cdata));
if (not_written)
{
/* Generate code attribute value and add */
wrbuf_rewind(wr_cdata);
wrbuf_iconv_write(wr_cdata, mt->iconv_cd,s->code_data, using_code_len);
xmlNewProp(ptr_subfield, BAD_CAST "code", BAD_CAST wrbuf_cstr(wr_cdata));
}
}
wrbuf_destroy(subfield_name);
}
static void tstwrbuf(void)
{
int step;
WRBUF wr = wrbuf_alloc();
YAZ_CHECK(wr);
wrbuf_destroy(wr);
wr = wrbuf_alloc();
YAZ_CHECK(wr);
for (step = 1; step < 65; step++)
{
int i, j, k;
int len;
char buf[64];
char *cp;
for (j = 1; j<step; j++)
{
for (i = 0; i<j; i++)
buf[i] = i+1;
buf[i] = '\0';
wrbuf_puts(wr, buf);
}
cp = wrbuf_buf(wr);
len = wrbuf_len(wr);
YAZ_CHECK(len == step * (step-1) / 2);
k = 0;
for (j = 1; j<step; j++)
for (i = 0; i<j; i++)
{
YAZ_CHECK(cp[k] == i+1);
k++;
}
wrbuf_rewind(wr);
}
wrbuf_destroy(wr);
}
void yaz_record_conv_destroy(yaz_record_conv_t p)
{
if (p)
{
yaz_record_conv_reset(p);
nmem_destroy(p->nmem);
wrbuf_destroy(p->wr_error);
xfree(p->path);
xfree(p);
}
}
int solr_transform_rpn2solr_stream(solr_transform_t ct,
void (*pr)(const char *buf, void *client_data),
void *client_data,
Z_RPNQuery *q)
{
WRBUF w = wrbuf_alloc();
int r = solr_transform_rpn2solr_stream_r(ct, w, pr, client_data, q);
if (r)
solr_transform_set_error(ct, r, wrbuf_len(w) ? wrbuf_cstr(w) : 0);
wrbuf_destroy(w);
return r;
}
int solr_transform_rpn2solr_stream(solr_transform_t ct,
void (*pr)(const char *buf, void *client_data),
void *client_data,
Z_RPNQuery *q)
{
int r;
WRBUF w = wrbuf_alloc();
solr_transform_set_error(ct, 0, 0);
r = rpn2solr_structure(ct, pr, client_data, q->RPNStructure, 0, w);
wrbuf_destroy(w);
return r;
}
void yaz_url_destroy(yaz_url_t p)
{
if (p)
{
odr_destroy(p->odr_in);
odr_destroy(p->odr_out);
xfree(p->proxy);
wrbuf_destroy(p->w_error);
yaz_cookies_destroy(p->cookies);
xfree(p);
}
}
static int conv_convert_test(yaz_record_conv_t p,
const char *input_record,
const char *output_expect_record)
{
int ret = 0;
if (!p)
{
YAZ_CHECK(ret);
}
else
{
WRBUF output_record = wrbuf_alloc();
int r = yaz_record_conv_record(p, input_record, strlen(input_record),
output_record);
if (r)
{
if (output_expect_record)
{
printf("yaz_record_conv error=%s\n",
yaz_record_conv_get_error(p));
ret = 0;
}
else
ret = 1;
}
else
{
if (!output_expect_record)
{
ret = 0;
}
else if (strcmp(output_expect_record, wrbuf_cstr(output_record)))
{
ret = 0;
printf("got-output_record len=%ld: %s\n",
(long) wrbuf_len(output_record),
wrbuf_cstr(output_record));
printf("output_expect_record len=%ld %s\n",
(long) strlen(output_expect_record),
output_expect_record);
}
else
{
ret = 1;
}
}
wrbuf_destroy(output_record);
}
return ret;
}
Z_RPNQuery *ccl_rpn_query (ODR o, struct ccl_rpn_node *p)
{
YAZ_PQF_Parser parser = yaz_pqf_create();
WRBUF wr = wrbuf_alloc();
Z_RPNQuery *q;
ccl_pquery(wr, p);
q = yaz_pqf_parse(parser, o, wrbuf_cstr(wr));
wrbuf_destroy(wr);
yaz_pqf_destroy(parser);
return q;
}
Z_AttributesPlusTerm *ccl_scan_query (ODR o, struct ccl_rpn_node *p)
{
YAZ_PQF_Parser parser = yaz_pqf_create();
WRBUF wr = wrbuf_alloc();
Z_AttributesPlusTerm *q;
Odr_oid *setp;
ccl_pquery(wr, p);
q = yaz_pqf_scan(parser, o, &setp, wrbuf_cstr(wr));
wrbuf_destroy(wr);
yaz_pqf_destroy(parser);
return q;
}
static int compare2(cql_transform_t ct, const char *pqf, const char *cql,
int expected_error)
{
int ret = 0;
ODR odr = odr_createmem(ODR_ENCODE);
WRBUF w = wrbuf_alloc();
Z_RPNQuery *q = p_query_rpn(odr, pqf);
if (q)
{
int r = cql_transform_rpn2cql_wrbuf(ct, w, q);
if (r != 0)
{
const char *addinfo = 0;
int err = cql_transform_error(ct, &addinfo);
/* transform error */
yaz_log(YLOG_LOG, "%s -> Error %d", pqf, r);
if (err == 0)
;
else if (err == expected_error)
{
if (addinfo && cql && !strcmp(addinfo, cql))
ret = 1;
else if (!addinfo && !cql)
ret = 1;
}
}
else if (r == 0)
{
yaz_log(YLOG_LOG, "%s -> %s", pqf, wrbuf_cstr(w));
if (!expected_error)
ret = 1;
else if (cql && !strcmp(wrbuf_cstr(w), cql))
{
ret = 1;
}
else
{
yaz_log(YLOG_WARN, " expected: %s", cql ? cql : "null");
yaz_log(YLOG_WARN, " got: %s", wrbuf_cstr(w));
}
}
}
wrbuf_destroy(w);
odr_destroy(odr);
return ret;
}
int yaz_record_conv_opac_record(yaz_record_conv_t p,
Z_OPACRecord *input_record,
WRBUF output_record)
{
int ret = 0;
struct yaz_record_conv_rule *r = p->rules;
if (!r || r->type->construct != construct_marc)
{
wrbuf_puts(p->wr_error, "Expecting MARC rule as first rule for OPAC");
ret = -1; /* no marc rule so we can't do OPAC */
}
else
{
struct marc_info *mi = r->info;
const char *input_charset = mi->input_charset;
yaz_iconv_t cd;
WRBUF res = wrbuf_alloc();
yaz_marc_t mt = yaz_marc_create();
if (yaz_opac_check_marc21_coding(input_charset, input_record))
input_charset = "utf-8";
cd = yaz_iconv_open(mi->output_charset, input_charset);
wrbuf_rewind(p->wr_error);
yaz_marc_xml(mt, mi->output_format_mode);
yaz_marc_iconv(mt, cd);
yaz_opac_decode_wrbuf(mt, input_record, res);
if (ret != -1)
{
ret = yaz_record_conv_record_rule(p,
r->next,
wrbuf_buf(res), wrbuf_len(res),
output_record);
}
yaz_marc_destroy(mt);
if (cd)
yaz_iconv_close(cd);
wrbuf_destroy(res);
}
return ret;
}
void xmlquerytopqf(const char *xmlstr)
{
xmlDocPtr doc;
doc = xmlParseMemory(xmlstr, strlen(xmlstr));
if (!doc)
{
fprintf(stderr, "%s: xml parse error for XML:\n%s\n", prog, xmlstr);
exit(1);
}
else
{
int error_code = 0;
const char *addinfo = 0;
Z_Query *query = 0;
ODR odr = odr_createmem(ODR_ENCODE);
const xmlNode *root_element = xmlDocGetRootElement(doc);
yaz_xml2query(root_element, &query, odr, &error_code, &addinfo);
if (error_code)
{
fprintf(stderr, "%s: yaz_xml2query failed code=%d addinfo=%s\n",
prog, error_code, addinfo);
exit(1);
}
else if (!query)
{
fprintf(stderr, "%s: yaz_xml2query no query result\n",
prog);
exit(1);
}
else
{
WRBUF w = wrbuf_alloc();
yaz_query_to_wrbuf(w, query);
printf("%s\n", wrbuf_cstr(w));
wrbuf_destroy(w);
}
odr_destroy(odr);
xmlFreeDoc(doc);
}
}
static int convert_solrmarc(void *info, WRBUF record, WRBUF wr_error)
{
WRBUF w = wrbuf_alloc();
const char *buf = wrbuf_buf(record);
size_t i, sz = wrbuf_len(record);
for (i = 0; i < sz; i++)
{
int ch;
if (buf[i] == '#' && i < sz - 3 && buf[i+3] == ';'
&& atoi_n_check(buf+i+1, 2, &ch))
i += 3;
else
ch = buf[i];
wrbuf_putc(w, ch);
}
wrbuf_rewind(record);
wrbuf_write(record, wrbuf_buf(w), wrbuf_len(w));
wrbuf_destroy(w);
return 0;
}
static void marcdump_read_line(yaz_marc_t mt, const char *fname)
{
FILE *inf = fopen(fname, "rb");
if (!inf)
{
fprintf(stderr, "%s: cannot open %s:%s\n",
prog, fname, strerror(errno));
exit(1);
}
while (yaz_marc_read_line(mt, getbyte_stream,
ungetbyte_stream, inf) == 0)
{
WRBUF wrbuf = wrbuf_alloc();
yaz_marc_write_mode(mt, wrbuf);
fputs(wrbuf_cstr(wrbuf), stdout);
wrbuf_destroy(wrbuf);
}
fclose(inf);
}
void cql_transform_close(cql_transform_t ct)
{
struct cql_prop_entry *pe;
if (!ct)
return;
pe = ct->entry;
while (pe)
{
struct cql_prop_entry *pe_next = pe->next;
xfree(pe->pattern);
xfree(pe->value);
xfree(pe);
pe = pe_next;
}
xfree(ct->addinfo);
yaz_tok_cfg_destroy(ct->tok_cfg);
wrbuf_destroy(ct->w);
nmem_destroy(ct->nmem);
xfree(ct);
}
int expect_pqf(const char *pqf, const char *expect_pqf, int expect_error)
{
YAZ_PQF_Parser parser = yaz_pqf_create();
int res = 0;
ODR odr = odr_createmem(ODR_ENCODE);
Z_RPNQuery *rpn;
if (!parser)
return 0;
if (!odr)
return 0;
rpn = yaz_pqf_parse(parser, odr, pqf);
if (!rpn)
{
const char *msg;
size_t offset;
int got_error = yaz_pqf_error (parser, &msg, &offset);
if (expect_error == got_error)
res = 1;
}
else if (expect_error == YAZ_PQF_ERROR_NONE)
{
WRBUF wrbuf = wrbuf_alloc();
if (wrbuf)
{
yaz_rpnquery_to_wrbuf(wrbuf, rpn);
if (!strcmp(wrbuf_cstr(wrbuf), expect_pqf))
res = 1;
wrbuf_destroy(wrbuf);
}
}
yaz_pqf_destroy(parser);
odr_destroy(odr);
return res;
}
// Cleanup channel
static void http_channel_destroy(IOCHAN i)
{
struct http_channel *s = iochan_getdata(i);
http_server_t http_server;
if (s->proxy)
{
if (s->proxy->iochan)
{
#ifdef WIN32
closesocket(iochan_getfd(s->proxy->iochan));
#else
close(iochan_getfd(s->proxy->iochan));
#endif
iochan_destroy(s->proxy->iochan);
}
http_buf_destroy_queue(s->http_server, s->proxy->oqueue);
xfree(s->proxy);
}
http_buf_destroy_queue(s->http_server, s->iqueue);
http_buf_destroy_queue(s->http_server, s->oqueue);
http_fire_observers(s);
http_destroy_observers(s);
http_server = s->http_server; /* save it for destroy (decref) */
http_server_destroy(http_server);
#ifdef WIN32
closesocket(iochan_getfd(i));
#else
close(iochan_getfd(i));
#endif
iochan_destroy(i);
nmem_destroy(s->nmem);
wrbuf_destroy(s->wrbuf);
xfree(s);
}
static struct json_node *do_parse_from_stdin(void)
{
FILE *f = stdin;
WRBUF w = wrbuf_alloc();
struct json_node *n;
size_t pos;
const char *json_str;
const char *err_msg;
int c;
while ((c = getc(f)) != EOF)
wrbuf_putc(w, c);
json_str = wrbuf_cstr(w);
n = json_parse2(json_str, &err_msg, &pos);
if (!n)
{
fprintf(stderr, "JSON parse error: %s\nLeading text was:\n", err_msg);
fwrite(json_str, 1, pos, stderr);
fprintf(stderr, "^\n");
}
wrbuf_destroy(w);
return n;
}
static void error(struct http_response *rs,
enum pazpar2_error_code code,
const char *addinfo)
{
struct http_channel *c = rs->channel;
WRBUF text = wrbuf_alloc();
const char *http_status = "417";
const char *msg = get_msg(code);
rs->msg = nmem_strdup(c->nmem, msg);
strcpy(rs->code, http_status);
wrbuf_printf(text, HTTP_COMMAND_RESPONSE_PREFIX "<error code=\"%d\" msg=\"%s\">", (int) code,
msg);
if (addinfo)
wrbuf_xmlputs(text, addinfo);
wrbuf_puts(text, "</error>");
yaz_log(YLOG_WARN, "HTTP %s %s%s%s", http_status,
msg, addinfo ? ": " : "" , addinfo ? addinfo : "");
rs->payload = nmem_strdup(c->nmem, wrbuf_cstr(text));
wrbuf_destroy(text);
http_send_response(c);
}
static int yaz_marc_write_xml_turbo_xml(yaz_marc_t mt, xmlNode **root_ptr,
const char *ns,
const char *format,
const char *type)
{
struct yaz_marc_node *n;
int identifier_length;
const char *leader = 0;
xmlNode *record_ptr;
xmlNsPtr ns_record;
WRBUF wr_cdata = 0;
for (n = mt->nodes; n; n = n->next)
if (n->which == YAZ_MARC_LEADER)
{
leader = n->u.leader;
break;
}
if (!leader)
return -1;
if (!atoi_n_check(leader+11, 1, &identifier_length))
return -1;
wr_cdata = wrbuf_alloc();
record_ptr = xmlNewNode(0, BAD_CAST "r");
*root_ptr = record_ptr;
ns_record = xmlNewNs(record_ptr, BAD_CAST ns, 0);
xmlSetNs(record_ptr, ns_record);
if (format)
xmlNewProp(record_ptr, BAD_CAST "format", BAD_CAST format);
if (type)
xmlNewProp(record_ptr, BAD_CAST "type", BAD_CAST type);
for (n = mt->nodes; n; n = n->next)
{
xmlNode *ptr;
char field[10];
field[0] = 'c';
field[4] = '\0';
switch(n->which)
{
case YAZ_MARC_DATAFIELD:
add_marc_datafield_turbo_xml(mt, n, record_ptr, ns_record, wr_cdata, identifier_length);
break;
case YAZ_MARC_CONTROLFIELD:
wrbuf_rewind(wr_cdata);
wrbuf_iconv_puts(wr_cdata, mt->iconv_cd, n->u.controlfield.data);
marc_iconv_reset(mt, wr_cdata);
strncpy(field + 1, n->u.controlfield.tag, 3);
ptr = xmlNewTextChild(record_ptr, ns_record,
BAD_CAST field,
BAD_CAST wrbuf_cstr(wr_cdata));
break;
case YAZ_MARC_COMMENT:
ptr = xmlNewComment(BAD_CAST n->u.comment);
xmlAddChild(record_ptr, ptr);
break;
case YAZ_MARC_LEADER:
xmlNewTextChild(record_ptr, ns_record, BAD_CAST "l",
BAD_CAST n->u.leader);
break;
}
}
wrbuf_destroy(wr_cdata);
return 0;
}