/* Match the already-compiled PCRE pattern against the data in SUBJECT,
of size SEARCH_BYTES and starting with offset SEARCH_OFFSET, with
options OPTIONS, and storing resulting matches into SUB. Return
the (nonnegative) match location or a (negative) error number. */
static int
jit_exec (char const *subject, int search_bytes, int search_offset,
int options, int *sub)
{
while (true)
{
int e = pcre_exec (cre, extra, subject, search_bytes, search_offset,
options, sub, NSUB);
# if PCRE_STUDY_JIT_COMPILE
if (e == PCRE_ERROR_JIT_STACKLIMIT
&& 0 < jit_stack_size && jit_stack_size <= INT_MAX / 2)
{
int old_size = jit_stack_size;
int new_size = jit_stack_size = old_size * 2;
static pcre_jit_stack *jit_stack;
if (jit_stack)
pcre_jit_stack_free (jit_stack);
jit_stack = pcre_jit_stack_alloc (old_size, new_size);
if (!jit_stack)
error (EXIT_TROUBLE, 0,
_("failed to allocate memory for the PCRE JIT stack"));
pcre_assign_jit_stack (extra, NULL, jit_stack);
continue;
}
# endif
return e;
}
}
void ymd_final(struct ymd_mach *vm) {
gc_final(vm);
kpool_final(vm);
vm_final_context(vm);
if (vm->pcre_js)
pcre_jit_stack_free(vm->pcre_js);
assert (vm->gc.used == 0); // Must free all memory!
assert (vm->gc.n_alloced == 0); // Allocated object must be zero.
free(vm);
}
static void free_pcre1_regexp(struct grep_pat *p)
{
pcre_free(p->pcre1_regexp);
#ifdef GIT_PCRE1_USE_JIT
if (p->pcre1_jit_on) {
pcre_free_study(p->pcre1_extra_info);
pcre_jit_stack_free(p->pcre1_jit_stack);
} else
#endif
{
pcre_free(p->pcre1_extra_info);
}
pcre_free((void *)p->pcre1_tables);
}
static void
rspamd_regexp_dtor (rspamd_regexp_t *re)
{
if (re) {
if (re->raw_re && re->raw_re != re->re) {
pcre_free (re->raw_re);
#ifdef HAVE_PCRE_JIT
if (re->raw_extra) {
pcre_free_study (re->raw_extra);
}
if (re->raw_jstack) {
pcre_jit_stack_free (re->raw_jstack);
}
#else
pcre_free (re->raw_extra);
#endif
}
if (re->re) {
pcre_free (re->re);
#ifdef HAVE_PCRE_JIT
if (re->extra) {
pcre_free_study (re->extra);
}
if (re->jstack) {
pcre_jit_stack_free (re->jstack);
}
#else
pcre_free (re->extra);
#endif
}
if (re->pattern) {
g_free (re->pattern);
}
}
}
static void
pcre_RegexObject_dealloc(pcre_RegexObject* self)
{
free(self->pattern);
Py_XDECREF(self->groupindex);
if (self->re != NULL)
pcre_free(self->re);
if (self->study != NULL)
pcre_free_study(self->study);
if (self->jit_stack != NULL)
pcre_jit_stack_free(self->jit_stack);
self->ob_type->tp_free((PyObject*)self);
}
static void setstack8(pcre_extra *extra)
{
static pcre_jit_stack *stack;
if (!extra) {
if (stack)
pcre_jit_stack_free(stack);
stack = NULL;
return;
}
if (!stack)
stack = pcre_jit_stack_alloc(1, 1024 * 1024);
/* Extra can be NULL. */
pcre_assign_jit_stack(extra, callback8, stack);
}
/*
* Regex stuff
*/
void regex_free(tvh_regex_t *regex)
{
#if ENABLE_PCRE || ENABLE_PCRE2
if (regex->is_posix) {
#endif
regfree(®ex->re_posix_code);
regex->re_posix_text = NULL;
#if ENABLE_PCRE || ENABLE_PCRE2
} else {
#if ENABLE_PCRE
#ifdef PCRE_CONFIG_JIT
#if PCRE_STUDY_JIT_COMPILE
if (regex->re_jit_stack) {
pcre_jit_stack_free(regex->re_jit_stack);
regex->re_jit_stack = NULL;
}
#endif
pcre_free_study(regex->re_extra);
#else
pcre_free(regex->re_extra);
#endif
pcre_free(regex->re_code);
regex->re_extra = NULL;
regex->re_code = NULL;
regex->re_text = NULL;
#elif ENABLE_PCRE2
pcre2_jit_stack_free(regex->re_jit_stack);
pcre2_match_data_free(regex->re_match);
pcre2_code_free(regex->re_code);
pcre2_match_context_free(regex->re_mcontext);
regex->re_match = NULL;
regex->re_code = NULL;
regex->re_mcontext = NULL;
regex->re_jit_stack = NULL;
#endif
}
#endif
}
/**
* @brief Execute the rule.
*
* @param[in] ib Ironbee engine
* @param[in] tx The transaction.
* @param[in,out] User data. A @c pcre_rule_data_t.
* @param[in] flags Operator instance flags
* @param[in] field The field content.
* @param[out] result The result.
* @returns IB_OK most times. IB_EALLOC when a memory allocation error handles.
*/
static ib_status_t pcre_operator_execute(ib_engine_t *ib,
ib_tx_t *tx,
const ib_rule_t *rule,
void *data,
ib_flags_t flags,
ib_field_t *field,
ib_num_t *result)
{
IB_FTRACE_INIT();
assert(ib!=NULL);
assert(tx!=NULL);
assert(tx->dpi!=NULL);
assert(data!=NULL);
int matches;
ib_status_t ib_rc;
const int ovecsize = 3 * MATCH_MAX;
int *ovector = (int *)malloc(ovecsize*sizeof(*ovector));
const char* subject = NULL;
size_t subject_len = 0;
const ib_bytestr_t* bytestr;
pcre_rule_data_t *rule_data = (pcre_rule_data_t *)data;
pcre_extra *edata = NULL;
#ifdef PCRE_JIT_STACK
pcre_jit_stack *jit_stack = pcre_jit_stack_alloc(PCRE_JIT_MIN_STACK_SZ,
PCRE_JIT_MAX_STACK_SZ);
#endif
if (ovector==NULL) {
IB_FTRACE_RET_STATUS(IB_EALLOC);
}
if (field->type == IB_FTYPE_NULSTR) {
ib_rc = ib_field_value(field, ib_ftype_nulstr_out(&subject));
if (ib_rc != IB_OK) {
free(ovector);
IB_FTRACE_RET_STATUS(ib_rc);
}
if (subject != NULL) {
subject_len = strlen(subject);
}
}
else if (field->type == IB_FTYPE_BYTESTR) {
ib_rc = ib_field_value(field, ib_ftype_bytestr_out(&bytestr));
if (ib_rc != IB_OK) {
free(ovector);
IB_FTRACE_RET_STATUS(ib_rc);
}
if (bytestr != NULL) {
subject_len = ib_bytestr_length(bytestr);
subject = (const char *) ib_bytestr_const_ptr(bytestr);
}
}
else {
free(ovector);
IB_FTRACE_RET_STATUS(IB_EINVAL);
}
if (subject == NULL) {
subject = "";
}
/* Debug block. Escapes a string and prints it to the log.
* Memory is freed. */
if (ib_log_get_level(ib) >= 9) {
/* Worst case, we can have a string that is 4x larger.
* Consider if a string of 0xF7 is passed. That single character
* will expand to a string of 4 printed characters +1 for the \0
* character. */
char *debug_str = ib_util_hex_escape(subject, subject_len);
if ( debug_str != NULL ) {
ib_log_debug3_tx(tx, "Matching against: %s", debug_str);
free( debug_str );
}
}
#ifdef PCRE_JIT_STACK
/* Log if we expected jit, but did not get it. */
if (rule_data->is_jit && jit_stack == NULL) {
ib_log_debug(ib,
"Failed to allocate a jit stack for a jit-compiled rule. "
"Not using jit for this call.");
edata = NULL;
}
/* If the study data is NULL or size zero, don't use it. */
else if (rule_data->edata == NULL || rule_data->study_data_sz <= 0) {
edata = NULL;
}
/* Only if we get here do we use the study data (edata) in the rule_data. */
else {
edata = rule_data->edata;
pcre_assign_jit_stack(rule_data->edata, NULL, jit_stack);
}
#endif
matches = pcre_exec(rule_data->cpatt,
edata,
subject,
subject_len,
0, /* Starting offset. */
0, /* Options. */
ovector,
ovecsize);
#ifdef PCRE_JIT_STACK
if (jit_stack != NULL) {
pcre_jit_stack_free(jit_stack);
}
#endif
if (matches > 0) {
if (ib_flags_all(rule->flags, IB_RULE_FLAG_CAPTURE) == true) {
pcre_set_matches(ib, tx, ovector, matches, subject);
}
ib_rc = IB_OK;
*result = 1;
}
else if (matches == PCRE_ERROR_NOMATCH) {
if (ib_log_get_level(ib) >= 7) {
char* tmp_c = malloc(subject_len+1);
memcpy(tmp_c, subject, subject_len);
tmp_c[subject_len] = '\0';
/* No match. Return false to the caller (*result = 0). */
ib_log_debug2_tx(tx, "No match for [%s] using pattern [%s].",
tmp_c,
rule_data->patt);
free(tmp_c);
}
ib_rc = IB_OK;
*result = 0;
}
else {
/* Some other error occurred. Set the status to false and
report the error. */
ib_rc = IB_EUNKNOWN;
*result = 0;
}
free(ovector);
IB_FTRACE_RET_STATUS(ib_rc);
}
/**
* @brief Execute the PCRE operator
*
* @param[in] tx Current transaction.
* @param[in] instance_data Instance data needed for execution.
* @param[in] field The field to operate on.
* @param[in] capture If non-NULL, the collection to capture to.
* @param[out] result The result of the operator 1=true 0=false.
* @param[in] cbdata Callback data.
*
* @returns IB_OK most times. IB_EALLOC when a memory allocation error handles.
*/
static
ib_status_t pcre_operator_execute(
ib_tx_t *tx,
void *instance_data,
const ib_field_t *field,
ib_field_t *capture,
ib_num_t *result,
void *cbdata
)
{
assert(instance_data != NULL);
assert(tx != NULL);
int matches;
ib_status_t ib_rc;
const int ovecsize = 3 * MATCH_MAX;
int *ovector = (int *)malloc(ovecsize*sizeof(*ovector));
const char *subject = NULL;
size_t subject_len = 0;
const ib_bytestr_t *bytestr;
modpcre_operator_data_t *operator_data =
(modpcre_operator_data_t *)instance_data;
pcre_extra *edata = NULL;
#ifdef PCRE_JIT_STACK
pcre_jit_stack *jit_stack = NULL;
#endif
assert(operator_data->cpdata->is_dfa == false);
if (ovector==NULL) {
return IB_EALLOC;
}
if (field->type == IB_FTYPE_NULSTR) {
ib_rc = ib_field_value(field, ib_ftype_nulstr_out(&subject));
if (ib_rc != IB_OK) {
free(ovector);
return ib_rc;
}
if (subject != NULL) {
subject_len = strlen(subject);
}
}
else if (field->type == IB_FTYPE_BYTESTR) {
ib_rc = ib_field_value(field, ib_ftype_bytestr_out(&bytestr));
if (ib_rc != IB_OK) {
free(ovector);
return ib_rc;
}
if (bytestr != NULL) {
subject_len = ib_bytestr_length(bytestr);
subject = (const char *) ib_bytestr_const_ptr(bytestr);
}
}
else {
free(ovector);
return IB_EINVAL;
}
if (subject == NULL) {
subject = "";
}
if (operator_data->cpdata->is_jit) {
#ifdef PCRE_JIT_STACK
jit_stack = pcre_jit_stack_alloc(operator_data->cpdata->jit_stack_start,
operator_data->cpdata->jit_stack_max);
if (jit_stack == NULL) {
ib_log_warn(ib,
"Failed to allocate a jit stack for a jit-compiled rule. "
"Not using jit for this call."
);
}
/* If the study data is NULL or size zero, don't use it. */
else if (operator_data->cpdata->study_data_sz > 0) {
edata = operator_data->cpdata->edata;
}
if (edata != NULL) {
pcre_assign_jit_stack(edata, NULL, jit_stack);
}
#else
edata = NULL;
#endif
}
else if (operator_data->cpdata->study_data_sz > 0) {
edata = operator_data->cpdata->edata;
}
else {
edata = NULL;
}
matches = pcre_exec(operator_data->cpdata->cpatt,
edata,
subject,
subject_len,
0, /* Starting offset. */
0, /* Options. */
ovector,
ovecsize);
#ifdef PCRE_JIT_STACK
if (jit_stack != NULL) {
pcre_jit_stack_free(jit_stack);
}
#endif
if (matches > 0) {
if (capture != NULL) {
pcre_set_matches(tx, capture, ovector, matches, subject);
}
ib_rc = IB_OK;
*result = 1;
}
else if (matches == PCRE_ERROR_NOMATCH) {
ib_rc = IB_OK;
*result = 0;
}
else {
/* Some other error occurred. Set the status to false return the
* error. */
ib_rc = IB_EUNKNOWN;
*result = 0;
}
free(ovector);
return ib_rc;
}
/**
* @brief Execute the rule.
*
* @param[in] ib Ironbee engine
* @param[in] tx The transaction.
* @param[in,out] User data. A @c pcre_rule_data_t.
* @param[in] flags Operator instance flags
* @param[in] field The field content.
* @param[in] result The result.
* @returns IB_OK most times. IB_EALLOC when a memory allocation error handles.
*/
static ib_status_t pcre_operator_execute(ib_engine_t *ib,
ib_tx_t *tx,
void *data,
ib_flags_t flags,
ib_field_t *field,
ib_num_t *result)
{
IB_FTRACE_INIT();
assert(ib!=NULL);
assert(tx!=NULL);
assert(tx->dpi!=NULL);
assert(data!=NULL);
int matches;
ib_status_t ib_rc;
const int ovecsize = 3 * MATCH_MAX;
int *ovector = (int *)malloc(ovecsize*sizeof(*ovector));
const char* subject;
size_t subject_len;
const ib_bytestr_t* bytestr;
pcre_rule_data_t *rule_data = (pcre_rule_data_t *)data;
pcre *regex;
pcre_extra *regex_extra = NULL;
#ifdef PCRE_JIT_STACK
pcre_jit_stack *jit_stack = pcre_jit_stack_alloc(PCRE_JIT_MIN_STACK_SZ,
PCRE_JIT_MAX_STACK_SZ);
#endif
if (ovector==NULL) {
IB_FTRACE_RET_STATUS(IB_EALLOC);
}
if (field->type == IB_FTYPE_NULSTR) {
ib_rc = ib_field_value(field, ib_ftype_nulstr_out(&subject));
if (ib_rc != IB_OK) {
IB_FTRACE_RET_STATUS(ib_rc);
}
subject_len = strlen(subject);
}
else if (field->type == IB_FTYPE_BYTESTR) {
ib_rc = ib_field_value(field, ib_ftype_bytestr_out(&bytestr));
if (ib_rc != IB_OK) {
IB_FTRACE_RET_STATUS(ib_rc);
}
subject_len = ib_bytestr_length(bytestr);
subject = (const char *) ib_bytestr_const_ptr(bytestr);
}
else {
free(ovector);
IB_FTRACE_RET_STATUS(IB_EALLOC);
}
/* Debug block. Escapes a string and prints it to the log.
* Memory is freed. */
if (ib_log_get_level(ib) >= 9) {
/* Worst case, we can have a string that is 4x larger.
* Consider if a string of 0xF7 is passed. That single character
* will expand to a string of 4 printed characters +1 for the \0
* character. */
char *debug_str = ib_util_hex_escape(subject, subject_len);
if ( debug_str != NULL ) {
ib_log_debug3_tx(tx, "Matching against: %s", debug_str);
free( debug_str );
}
}
/* Alloc space to copy regex. */
regex = (pcre *)malloc(rule_data->cpatt_sz);
if (regex == NULL ) {
free(ovector);
IB_FTRACE_RET_STATUS(IB_EALLOC);
}
memcpy(regex, rule_data->cpatt, rule_data->cpatt_sz);
if (rule_data->study_data_sz == 0 ) {
regex_extra = NULL;
}
else {
regex_extra = (pcre_extra *) malloc(sizeof(*regex_extra));
if (regex_extra == NULL ) {
free(ovector);
free(regex);
IB_FTRACE_RET_STATUS(IB_EALLOC);
}
*regex_extra = *rule_data->edata;
if ( rule_data->study_data_sz == 0 ) {
regex_extra->study_data = NULL;
}
else {
regex_extra->study_data = malloc(rule_data->study_data_sz);
if (regex_extra->study_data == NULL ) {
free(ovector);
if (regex_extra != NULL) {
free(regex_extra);
}
free(regex);
IB_FTRACE_RET_STATUS(IB_EALLOC);
}
memcpy(regex_extra->study_data,
rule_data->edata->study_data,
rule_data->study_data_sz);
}
/* Put some modest limits on our regex. */
regex_extra->match_limit = 1000;
regex_extra->match_limit_recursion = 1000;
regex_extra->flags = regex_extra->flags |
PCRE_EXTRA_MATCH_LIMIT |
PCRE_EXTRA_MATCH_LIMIT_RECURSION;
}
#ifdef PCRE_JIT_STACK
if (jit_stack == NULL) {
if ( regex_extra != NULL ) {
if ( regex_extra->study_data != NULL ) {
free(regex_extra->study_data);
}
free(regex_extra);
}
free(ovector);
free(regex);
IB_FTRACE_RET_STATUS(IB_EALLOC);
}
pcre_assign_jit_stack(regex_extra, NULL, jit_stack);
#endif
matches = pcre_exec(regex,
regex_extra,
subject,
subject_len,
0, /* Starting offset. */
0, /* Options. */
ovector,
ovecsize);
#ifdef PCRE_JIT_STACK
pcre_jit_stack_free(jit_stack);
#endif
if (matches > 0) {
pcre_set_matches(ib, tx, "TX", ovector, matches, subject);
ib_rc = IB_OK;
*result = 1;
}
else if (matches == PCRE_ERROR_NOMATCH) {
if (ib_log_get_level(ib) >= 7) {
char* tmp_c = malloc(subject_len+1);
memcpy(tmp_c, subject, subject_len);
tmp_c[subject_len] = '\0';
/* No match. Return false to the caller (*result = 0). */
ib_log_debug2_tx(tx, "No match for [%s] using pattern [%s].",
tmp_c,
rule_data->patt);
free(tmp_c);
}
ib_rc = IB_OK;
*result = 0;
}
else {
/* Some other error occurred. Set the status to false and
report the error. */
ib_rc = IB_EUNKNOWN;
*result = 0;
}
if ( regex_extra != NULL ) {
if ( regex_extra->study_data != NULL ) {
free(regex_extra->study_data);
}
free(regex_extra);
}
free(ovector);
free(regex);
IB_FTRACE_RET_STATUS(ib_rc);
}
void pcre_find_all(char* pattern, char* subject, int subject_len, int repeat, int mode)
{
pcre *re;
const char *error;
int err_val, match[64];
pcre_extra *extra;
pcre_jit_stack *stack = NULL;
char *ptr;
int len;
clock_t best_time = 0, time = 0;
int found;
static int work_space[4096];
re = pcre_compile(
pattern, /* the pattern */
PCRE_MULTILINE | PCRE_NEWLINE_ANYCRLF, /* options */
&error, /* for error message */
&err_val, /* for error offset */
NULL); /* use default character tables */
if (!re) {
printf("PCRE compilation failed at offset %d: %s\n", err_val, error);
return;
}
error = NULL;
extra = pcre_study(re, mode == 2 ? PCRE_STUDY_JIT_COMPILE : 0, &error);
if (error) {
printf("PCRE study failed: %s\n", error);
return;
}
if (mode == 2) {
found = 0;
pcre_fullinfo(re, extra, PCRE_INFO_JIT, &found);
if (!found) {
printf("PCRE JIT compilation failed: %s\n", error);
return;
}
stack = pcre_jit_stack_alloc(65536, 65536);
}
do {
found = 0;
ptr = subject;
len = subject_len;
switch (mode) {
case 0:
time = clock();
while (1) {
err_val = pcre_exec(
re, /* the compiled pattern */
extra, /* extra data */
ptr, /* the subject string */
len, /* the length of the subject */
0, /* start at offset 0 in the subject */
0, /* default options */
match, /* output vector for substring information */
64); /* number of elements in the output vector */
if (err_val <= 0) {
if (err_val == PCRE_ERROR_NOMATCH)
break;
printf("PCRE pcre_exec failed with: %d\n", err_val);
break;
}
// printf("match: %d %d\n", (ptr - subject) + match[0], (ptr - subject) + match[1]);
ptr += match[1];
len -= match[1];
found++;
}
time = clock() - time;
break;
case 1:
time = clock();
while (1) {
err_val = pcre_dfa_exec(
re, /* the compiled pattern */
extra, /* extra data */
ptr, /* the subject string */
len, /* the length of the subject */
0, /* start at offset 0 in the subject */
0, /* default options */
match, /* output vector for substring information */
2, /* number of elements in the output vector */
work_space, /* number of elements (NOT size in bytes) */
4096);
if (err_val < 0) {
if (err_val == PCRE_ERROR_NOMATCH)
break;
printf("PCRE pcre_exec failed\n");
break;
}
// printf("match: %d %d\n", (ptr - subject) + match[0], (ptr - subject) + match[1]);
ptr += match[1];
len -= match[1];
found++;
}
time = clock() - time;
break;
case 2:
time = clock();
while (1) {
err_val = pcre_jit_exec(
re, /* the compiled pattern */
extra, /* extra data */
ptr, /* the subject string */
len, /* the length of the subject */
0, /* start at offset 0 in the subject */
0, /* default options */
match, /* output vector for substring information */
64, /* number of elements in the output vector */
stack); /* jit stack */
if (err_val <= 0) {
if (err_val == PCRE_ERROR_NOMATCH)
break;
printf("PCRE pcre_exec failed with: %d\n", err_val);
break;
}
// printf("match: %d %d\n", (ptr - subject) + match[0], (ptr - subject) + match[1]);
ptr += match[1];
len -= match[1];
found++;
}
time = clock() - time;
break;
}
if (!best_time || time < best_time)
best_time = time;
} while (--repeat > 0);
printResult(mode == 0 ? "pcre" : (mode == 1 ? "pcre-dfa" : "pcre-jit"), best_time * 1000 / CLOCKS_PER_SEC, found);
if (extra)
pcre_free_study(extra);
if (stack)
pcre_jit_stack_free(stack);
pcre_free(re);
}
void re_drop(re *r)
{
free(r->re);
pcre_jit_stack_free(r->stack);
pcre_free_study(r->extra);
}
