SQLite3_result * Query_Processor::get_current_query_rules() {
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping current query rules, using Global version %d\n", version);
SQLite3_result *result=new SQLite3_result(16);
spin_rdlock(&rwlock);
QP_rule_t *qr1;
result->add_column_definition(SQLITE_TEXT,"rule_id");
result->add_column_definition(SQLITE_TEXT,"active");
result->add_column_definition(SQLITE_TEXT,"username");
result->add_column_definition(SQLITE_TEXT,"schemaname");
result->add_column_definition(SQLITE_TEXT,"flagIN");
result->add_column_definition(SQLITE_TEXT,"match_pattern");
result->add_column_definition(SQLITE_TEXT,"negate_match_pattern");
result->add_column_definition(SQLITE_TEXT,"flagOUT");
result->add_column_definition(SQLITE_TEXT,"replace_pattern");
result->add_column_definition(SQLITE_TEXT,"destination_hostgroup");
result->add_column_definition(SQLITE_TEXT,"cache_ttl");
result->add_column_definition(SQLITE_TEXT,"reconnect");
result->add_column_definition(SQLITE_TEXT,"timeout");
result->add_column_definition(SQLITE_TEXT,"delay");
result->add_column_definition(SQLITE_TEXT,"apply");
result->add_column_definition(SQLITE_TEXT,"hits");
for (std::vector<QP_rule_t *>::iterator it=rules.begin(); it!=rules.end(); ++it) {
qr1=*it;
QP_rule_text *qt=new QP_rule_text(qr1);
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping Query Rule id: %d\n", qr1->rule_id);
result->add_row(qt->pta);
delete qt;
}
spin_rdunlock(&rwlock);
return result;
}
void Query_Processor::update_query_processor_stats() {
// Note:
// this function is called by each thread to update global query statistics
//
// As an extra safety, it checks that the version didn't change
// Yet, if version changed doesn't perfomr any rules update
//
// It acquires a read lock to ensure that the rules table doesn't change
// Yet, because it has to update vales, it uses atomic operations
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "Updating query rules statistics\n");
spin_rdlock(&rwlock);
if (__sync_add_and_fetch(&version,0) == _thr_SQP_version) {
QP_rule_t *qr;
for (std::vector<QP_rule_t *>::iterator it=_thr_SQP_rules->begin(); it!=_thr_SQP_rules->end(); ++it) {
qr=*it;
if (qr->active && qr->hits) {
__sync_fetch_and_add(&qr->parent->hits,qr->hits);
qr->hits=0;
}
}
}
spin_rdunlock(&rwlock);
for (int i=0; i<MYSQL_COM_QUERY___NONE; i++) {
for (int j=0; j<13; j++) {
__sync_fetch_and_add(&commands_counters[i]->counters[j],_thr_commands_counters[i]->counters[j]);
_thr_commands_counters[i]->counters[j]=0;
}
__sync_fetch_and_add(&commands_counters[i]->total_time,_thr_commands_counters[i]->total_time);
_thr_commands_counters[i]->total_time=0;
}
};
QC_entry_t * KV_BtreeArray::lookup(uint64_t key) {
QC_entry_t *entry=NULL;
spin_rdlock(&lock);
THR_UPDATE_CNT(__thr_cntGet,Glo_cntGet,1,100);
btree::btree_map<uint64_t, QC_entry_t *>::iterator lookup;
lookup = bt_map.find(key);
if (lookup != bt_map.end()) {
entry=lookup->second;
__sync_fetch_and_add(&entry->ref_count,1);
THR_UPDATE_CNT(__thr_cntGetOK,Glo_cntGetOK,1,100);
THR_UPDATE_CNT(__thr_dataOUT,Glo_dataOUT,entry->length,10000);
}
spin_rdunlock(&lock);
return entry;
};
void KV_BtreeArray::purge_some(unsigned long long QCnow_ms) {
uint64_t ret=0, i, _size=0;
QC_entry_t *qce;
spin_rdlock(&lock);
for (i=0; i<ptrArray->len;i++) {
qce=(QC_entry_t *)ptrArray->index(i);
if (qce->expire_ms==EXPIRE_DROPIT || qce->expire_ms<QCnow_ms) {
ret++;
_size+=qce->length;
}
}
freeable_memory=_size;
spin_rdunlock(&lock);
if ( (freeable_memory + ret * (sizeof(QC_entry_t)+sizeof(QC_entry_t *)*2+sizeof(uint64_t)*2) ) > get_data_size()*0.01) {
uint64_t removed_entries=0;
uint64_t freed_memory=0;
spin_wrlock(&lock);
for (i=0; i<ptrArray->len;i++) {
qce=(QC_entry_t *)ptrArray->index(i);
if ((qce->expire_ms==EXPIRE_DROPIT || qce->expire_ms<QCnow_ms) && (__sync_fetch_and_add(&qce->ref_count,0)<=1)) {
qce=(QC_entry_t *)ptrArray->remove_index_fast(i);
btree::btree_map<uint64_t, QC_entry_t *>::iterator lookup;
lookup = bt_map.find(qce->key);
if (lookup != bt_map.end()) {
bt_map.erase(lookup);
}
i--;
freed_memory+=qce->length;
removed_entries++;
free(qce->value);
free(qce);
}
}
spin_wrunlock(&lock);
THR_DECREASE_CNT(__thr_num_deleted,Glo_num_entries,removed_entries,1);
if (removed_entries) {
__sync_fetch_and_add(&Glo_total_freed_memory,freed_memory);
__sync_fetch_and_sub(&Glo_size_values,freed_memory);
__sync_fetch_and_add(&Glo_cntPurge,removed_entries);
// if (removed_entries) fprintf(stderr,"Removed: %lu, total: %lu, arraylen: %d\n", removed_entries, __sync_fetch_and_sub(&Glo_num_entries,0), ptrArray.len);
// if (removed_entries) firintf(stderr,"Size of KVBtreeArray:%d , freed_memory:%lu, Glo_cntGet:%lu, Glo_cntGetOK:%lu, Glo_cntSet:%lu, cntPurge:%lu, dataIN:%lu, dataOUT:%lu\n", cnt() , Glo_total_freed_memory, Glo_cntGet, Glo_cntGetOK, Glo_cntSet, Glo_cntPurge, Glo_dataIN, Glo_dataOUT);
}
}
};
SQLite3_result * Query_Processor::get_stats_query_rules() {
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping query rules statistics, using Global version %d\n", version);
SQLite3_result *result=new SQLite3_result(2);
spin_rdlock(&rwlock);
QP_rule_t *qr1;
result->add_column_definition(SQLITE_TEXT,"rule_id");
result->add_column_definition(SQLITE_TEXT,"hits");
for (std::vector<QP_rule_t *>::iterator it=rules.begin(); it!=rules.end(); ++it) {
qr1=*it;
if (qr1->active) {
QP_rule_text_hitsonly *qt=new QP_rule_text_hitsonly(qr1);
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping Query Rule id: %d\n", qr1->rule_id);
result->add_row(qt->pta);
delete qt;
}
}
spin_rdunlock(&rwlock);
return result;
}
SQLite3_result * Query_Processor::get_query_digests() {
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping current query digest\n");
SQLite3_result *result=new SQLite3_result(10);
spin_rdlock(&digest_rwlock);
result->add_column_definition(SQLITE_TEXT,"schemaname");
result->add_column_definition(SQLITE_TEXT,"usernname");
result->add_column_definition(SQLITE_TEXT,"digest");
result->add_column_definition(SQLITE_TEXT,"digest_text");
result->add_column_definition(SQLITE_TEXT,"count_star");
result->add_column_definition(SQLITE_TEXT,"first_seen");
result->add_column_definition(SQLITE_TEXT,"last_seen");
result->add_column_definition(SQLITE_TEXT,"sum_time");
result->add_column_definition(SQLITE_TEXT,"min_time");
result->add_column_definition(SQLITE_TEXT,"max_time");
for (btree::btree_map<uint64_t, void *>::iterator it=digest_bt_map.begin(); it!=digest_bt_map.end(); ++it) {
QP_query_digest_stats *qds=(QP_query_digest_stats *)it->second;
char **pta=qds->get_row();
result->add_row(pta);
qds->free_row(pta);
}
spin_rdunlock(&digest_rwlock);
return result;
}
Query_Processor_Output * Query_Processor::process_mysql_query(MySQL_Session *sess, void *ptr, unsigned int size, bool delete_original) {
Query_Processor_Output *ret=NULL;
unsigned int len=size-sizeof(mysql_hdr)-1;
char *query=(char *)l_alloc(len+1);
memcpy(query,(char *)ptr+sizeof(mysql_hdr)+1,len);
query[len]=0;
if (__sync_add_and_fetch(&version,0) > _thr_SQP_version) {
// update local rules;
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Detected a changed in version. Global:%d , local:%d . Refreshing...\n", version, _thr_SQP_version);
spin_rdlock(&rwlock);
_thr_SQP_version=__sync_add_and_fetch(&version,0);
__reset_rules(_thr_SQP_rules);
QP_rule_t *qr1;
QP_rule_t *qr2;
for (std::vector<QP_rule_t *>::iterator it=rules.begin(); it!=rules.end(); ++it) {
qr1=*it;
if (qr1->active) {
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Copying Query Rule id: %d\n", qr1->rule_id);
qr2=new_query_rule(qr1->rule_id, qr1->active, qr1->username, qr1->schemaname, qr1->flagIN, qr1->match_pattern, qr1->negate_match_pattern, qr1->flagOUT, qr1->replace_pattern, qr1->destination_hostgroup, qr1->cache_ttl, qr1->reconnect, qr1->timeout, qr1->delay, qr1->apply);
qr2->parent=qr1; // pointer to parent to speed up parent update (hits)
if (qr2->match_pattern) {
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Compiling regex for rule_id: %d, match_pattern: \n", qr2->rule_id, qr2->match_pattern);
qr2->regex_engine=(void *)compile_query_rule(qr2);
}
_thr_SQP_rules->push_back(qr2);
}
}
spin_rdunlock(&rwlock); // unlock should be after the copy
}
QP_rule_t *qr;
re2_t *re2p;
int flagIN=0;
for (std::vector<QP_rule_t *>::iterator it=_thr_SQP_rules->begin(); it!=_thr_SQP_rules->end(); ++it) {
qr=*it;
if (qr->flagIN != flagIN) {
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 6, "query rule %d has no matching flagIN\n", qr->rule_id);
continue;
}
if (qr->username && strlen(qr->username)) {
if (strcmp(qr->username,sess->client_myds->myconn->userinfo->username)!=0) {
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has no matching username\n", qr->rule_id);
continue;
}
}
if (qr->schemaname && strlen(qr->schemaname)) {
if (strcmp(qr->schemaname,sess->client_myds->myconn->userinfo->schemaname)!=0) {
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has no matching schemaname\n", qr->rule_id);
continue;
}
}
re2p=(re2_t *)qr->regex_engine;
if (qr->match_pattern) {
bool rc;
if (ret && ret->new_query) {
// if we already rewrote the query, process the new query
//std::string *s=ret->new_query;
rc=RE2::PartialMatch(ret->new_query->c_str(),*re2p->re);
} else {
// we never rewrote the query
rc=RE2::PartialMatch(query,*re2p->re);
}
if ((rc==true && qr->negate_match_pattern==true) || ( rc==false && qr->negate_match_pattern==false )) {
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has no matching pattern\n", qr->rule_id);
continue;
}
}
// if we arrived here, we have a match
if (ret==NULL) {
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "this is the first time we find a match\n");
// create struct
//ret=(QP_out_t *)l_alloc(sizeof(QP_out_t));
ret=new Query_Processor_Output();
// initalized all values
ret->ptr=NULL;
ret->size=0;
ret->destination_hostgroup=-1;
ret->cache_ttl=-1;
ret->reconnect=-1;
ret->timeout=-1;
ret->delay=-1;
ret->new_query=NULL;
}
qr->hits++; // this is done without atomic function because it updates only the local variables
/*
{
// FIXME: this block of code is only for testing
if ((qr->hits%20)==0) {
spin_rdlock(&rwlock);
if (__sync_add_and_fetch(&version,0) == _thr_SQP_version) { // extra safety check to avoid race conditions
__sync_fetch_and_add(&qr->parent->hits,20);
}
*/
/*
QP_rule_t *qrg;
for (std::vector<QP_rule_t *>::iterator it=rules.begin(); it!=rules.end(); ++it) {
qrg=*it;
if (qrg->rule_id==qr->rule_id) {
__sync_fetch_and_add(&qrg->hits,20);
break;
}
}
*/
/*
spin_rdunlock(&rwlock);
}
}
*/
if (qr->flagOUT >= 0) {
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has changed flagOUT\n", qr->rule_id);
flagIN=qr->flagOUT;
//sess->query_info.flagOUT=flagIN;
}
if (qr->reconnect >= 0) {
// Note: negative reconnect means this rule doesn't change
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has set reconnect: %d. Query will%s be rexecuted if connection is lost\n", qr->rule_id, qr->reconnect, (qr->reconnect == 0 ? " NOT" : "" ));
ret->reconnect=qr->reconnect;
}
if (qr->timeout >= 0) {
// Note: negative timeout means this rule doesn't change
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has set timeout: %d. Query will%s be interrupted if exceeding %dms\n", qr->rule_id, qr->timeout, (qr->timeout == 0 ? " NOT" : "" ) , qr->timeout);
ret->timeout=qr->timeout;
}
if (qr->delay >= 0) {
// Note: negative delay means this rule doesn't change
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has set delay: %d. Session will%s be paused for %dms\n", qr->rule_id, qr->delay, (qr->delay == 0 ? " NOT" : "" ) , qr->delay);
ret->delay=qr->delay;
}
if (qr->cache_ttl >= 0) {
// Note: negative TTL means this rule doesn't change
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has set cache_ttl: %d. Query will%s hit the cache\n", qr->rule_id, qr->cache_ttl, (qr->cache_ttl == 0 ? " NOT" : "" ));
ret->cache_ttl=qr->cache_ttl;
}
if (qr->destination_hostgroup >= 0) {
// Note: negative hostgroup means this rule doesn't change
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has set destination hostgroup: %d\n", qr->rule_id, qr->destination_hostgroup);
ret->destination_hostgroup=qr->destination_hostgroup;
}
if (qr->replace_pattern) {
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d on match_pattern \"%s\" has a replace_pattern \"%s\" to apply\n", qr->rule_id, qr->match_pattern, qr->replace_pattern);
if (ret->new_query==NULL) ret->new_query=new std::string(query);
RE2::Replace(ret->new_query,qr->match_pattern,qr->replace_pattern);
}
if (qr->apply==true) {
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d is the last one to apply: exit!\n", qr->rule_id);
goto __exit_process_mysql_query;
}
}
__exit_process_mysql_query:
// FIXME : there is too much data being copied around
l_free(len+1,query);
return ret;
};