int lazy_semisupervised_classification() {
__START_TIMER__
CACHE.locked=1;
evaluation_t evaluator;
prediction_t prediction;
initialize_evaluation(&evaluator);
int abstain=0, n_points=1, remaining_points=N_POINTS;
float min_level=MIN_LEVEL;
list_t* unlabeled=UNLABELED;
while(unlabeled) {
reset_rules();
prediction=get_THE_prediction(unlabeled->instance, unlabeled->size, unlabeled->label);
if(prediction.score.points[prediction.score.ordered_labels[0]]>min_level || prediction.score.points[prediction.score.ordered_labels[0]]/(float)prediction.score.points[prediction.score.ordered_labels[1]]>=FACTOR) {
update_evaluation(&evaluator, prediction.label, unlabeled->label);
print_statistics(n_points, unlabeled->label, unlabeled->id, prediction, evaluator);
N_TRANSACTIONS++;
COUNT_TARGET[prediction.label]++;
ITEMSETS[TARGET_ID[prediction.label]].list[ITEMSETS[TARGET_ID[prediction.label]].count]=N_TRANSACTIONS;
ITEMSETS[TARGET_ID[prediction.label]].count++;
for(int j=0;j<unlabeled->size;j++) {
ITEMSETS[unlabeled->instance[j]].list[ITEMSETS[unlabeled->instance[j]].count]=N_TRANSACTIONS;
ITEMSETS[unlabeled->instance[j]].count++;
}
n_points++;
remaining_points--;
abstain=0;
list_t* x=unlabeled->next;
free(unlabeled->instance);
free(unlabeled);
unlabeled=x;
UNLABELED=unlabeled;
}
else if(abstain<remaining_points) {
printf("abstaining %d %d %f %f %f\n", abstain, remaining_points, min_level, prediction.score.points[prediction.score.ordered_labels[0]]/(float)prediction.score.points[prediction.score.ordered_labels[1]], avg_size);
abstain++;
list_t* q=UNLABELED;
while(q->next!=NULL) q=q->next;
list_t* t=(list_t*)malloc(sizeof(list_t));
t->instance=(int*)malloc(sizeof(int)*unlabeled->size);
t->size=0;
for(int i=0;i<unlabeled->size;i++) t->instance[t->size++]=unlabeled->instance[i];
t->label=unlabeled->label;
t->next=NULL;
q->next=t;
list_t* x=unlabeled->next;
free(unlabeled->instance);
free(unlabeled);
unlabeled=x;
UNLABELED=unlabeled;
}
else break;
}
printf("\n");
for(int i=0;i<MAX_CLASSES;i++) printf("CLASS(%d)= %d Prec= %f Rec= %f F1= %f ", i, evaluator.true_labels[i], evaluator.precision[i], evaluator.recall[i], evaluator.f1[i]);
printf("Acc= %f MF1= %f * hits= %ld misses= %ld\n", evaluator.acc, evaluator.mf1, CACHE.hits, CACHE.misses);
//finalize_evaluation(&evaluator);
__FINISH_TIMER__
return(0);
}
void lzw_encode_initialize(LZWEncoderData *ld, int initial_lzw_dictionary_size){
int input_length = 100;
memset(ld,0,sizeof(LZWEncoderData));
ld->initial_dictionary_size = initial_lzw_dictionary_size;
ld->output = malloc(sizeof(uint8_t)*input_length);
ld->output_size=input_length;
ld->next_rule_id = reset_rules(ld);
ld->clear_code_number = ld->next_rule_id;
ld->end_code_number = ld->next_rule_id+1;
ld->next_rule_id+=2;
ld->LZWmin = (int)ceil(log2(ld->next_rule_id));
ld->start_index = 0;
ld->resolved_indexes = 0;
ld->dictionary_branch = &ld->dictionary;
ld->last_matched_rule = NULL;
}
/**
INDUCES A RULE SET. THE RULE SET KEEPS GROWING UNTIL A CLASS IS SUFFICIENTLY BETTER THAN ANOTHER.
THEN, PERFORMS THE PREDICTION.
*/
prediction_t get_THE_prediction(int* instance, int instance_size, int true_label) {
prediction_t prediction;
set<int> relevant_classes;
int n_rules[2]={0,0}, min_count=MIN_COUNT, init=1, size=MIN_SIZE, n_items=0, default_prediction=0;
int* items=(int*) malloc(sizeof(int)*instance_size);
for(int i=0;i<instance_size;i++) {
if(instance[i]<N_ITEMSETS) items[n_items++]=instance[i];
}
qsort((int*) items, n_items, sizeof(int), item_cmp);
reset_rules();
while(size<MAX_SIZE) {
if(init) {
init=0;
project_training(items, n_items, &projection_size, &n_classes, relevant_classes);
for(int i=0;i<MAX_CLASSES;i++) count_target[i]=0;
for(int i=1;i<MAX_CLASSES;i++) {
if(count_target[i]>=count_target[default_prediction]) default_prediction=i;
}
}
if(RELATIVE) min_count=(int)(MIN_SUPP*projection_size);
induce_rules(items, n_items, count_target, size, projection_size, min_count, relevant_classes);
//for(int i=0;i<N_RULES;i++) {
// print_rule(RULES[i]);
// printf("\n");
//}
n_rules[0]=n_rules[1];
n_rules[1]=N_RULES;
size++;
if(size==MAX_SIZE || size>n_items || n_rules[0]==n_rules[1]) break;
}
prediction.score=get_TOTAL_score();
prediction.label=prediction.score.ordered_labels[0];
free(items);
if(N_RULES==0) prediction.label=default_prediction;
avg_size=0;
for(int i=0;i<N_RULES;i++) avg_size+=RULES[i].size;
if(N_RULES>0) avg_size/=N_RULES;
prediction.correct=(prediction.label==true_label);
return(prediction);
}
static void
check_rules(Port *port, int status)
{
int r;
/* index of the backend process */
int index;
/* limits */
bool per_user_overriden = false,
per_database_overriden = false,
per_ip_overriden = false;
/* counters */
int per_user = 0,
per_database = 0,
per_ip = 0;
/*
* Any other plugins which use ClientAuthentication_hook.
*/
if (prev_client_auth_hook)
prev_client_auth_hook(port, status);
/* No point in checkin the connection rules after failed authentication. */
if (status != STATUS_OK)
return;
/*
* Lock ProcArray (serialize the processes, so that we can use the
* counters stored in the rule_r struct).
*
* TODO Use a private array of counters (same number of rules), so
* that we don't need an exclusive lock.
*/
LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
/* reset the rule counters */
reset_rules();
/* attach the shared segment */
attach_procarray();
/*
* Perform the actual check - loop through the backends (procArray), and
* compare each valid backend against each rule. If it matches, increment
* the counter (and if value exceeds the limit, make a failure).
*
* TODO First check the rules for the current backend, and then only check
* those rules that match (because those are the only rules that may
* be violated by this new connection).
*/
for (index = 0; index < procArray->numProcs; index++)
{
#if (PG_VERSION_NUM <= 90200)
volatile PGPROC *proc = procArray->procs[index];
#else
volatile PGPROC *proc = &ProcGlobal->allProcs[procArray->procs[index]];
#endif
/* do not count prepared xacts */
if (proc->pid == 0)
continue;
/*
* If this is the current backend, then update the local info. This
* effectively resets info for crashed backends.
*
* FIXME Maybe this should happen explicitly before the loop.
*/
if (proc->backendId == MyBackendId)
{
/* lookup remote host name (unless already done) */
if (! port->remote_hostname)
{
char remote_hostname[NI_MAXHOST];
if (! pg_getnameinfo_all(&port->raddr.addr, port->raddr.salen,
remote_hostname, sizeof(remote_hostname),
NULL, 0, 0))
port->remote_hostname = pstrdup(remote_hostname);
}
/* store the backend info into a cache */
backend_update_info(&backends[proc->backendId], proc,
port->database_name, port->user_name,
port->raddr, port->remote_hostname);
}
/* if the backend info is valid, */
if (backend_info_is_valid(backends[proc->backendId], proc))
{
/* see if the database/user/IP matches */
per_database += (strcmp(backends[proc->backendId].database, port->database_name) == 0) ? 1 : 0;
per_user += (strcmp(backends[proc->backendId].role, port->user_name) == 0) ? 1 : 0;
per_ip += (memcmp(&backends[proc->backendId].socket, &port->raddr, sizeof(SockAddr)) == 0) ? 1 : 0;
/* check all the rules for this backend */
for (r = 0; r < rules->n_rules; r++)
{
/*
* The rule has to be matched by both the current and new session, otherwise
* it can't be violated by the new one.
*
* FIXME This repeatedly checks all the rules for the current backend, which is not
* needed. We only need to do this check (for the new session) once, and then
* walk only the rules that match it. Althouth that may not detect the
* default rules (per db, ...).
*/
if (rule_matches(rules->rules[r], port->database_name, port->user_name, port->raddr, port->remote_host))
{
/* check if this rule overrides per-db, per-user or per-ip limits */
per_database_overriden |= rule_is_per_database(&rules->rules[r]);
per_user_overriden |= rule_is_per_user(&rules->rules[r]);
per_ip_overriden |= rule_is_per_ip(&rules->rules[r]);
/* Check the rule for a existing backend (we assume it's valid thanks to backend_info_is_valid()). */
if (rule_matches(rules->rules[r], backends[proc->backendId].database,
backends[proc->backendId].role, backends[proc->backendId].socket,
backends[proc->backendId].hostname))
{
/* increment the match count for this rule */
++rules->rules[r].count;
/*
* We're looping over all backends (including the current backend), so the
* rule is only violated if the limit is actually exceeded.
*/
if (rules->rules[r].count > rules->rules[r].limit)
{
if (! is_super_user(port->user_name))
elog(ERROR, "connection limit reached (rule %d, line %d, limit %d)",
r, rules->rules[r].line, rules->rules[r].limit);
else
elog(WARNING, "connection limit reached (rule %d, line %d, limit %d), but the user is a superuser",
r, rules->rules[r].line, rules->rules[r].limit);
}
}
}
}
}
}
/*
* Check the per-db/user/IP limits, unless there was an exact rule overriding
* the defaults for that object, or unless the default was disabled (set to 0).
*/
/* check per-database limit */
if ((! per_database_overriden) && (default_per_database != 0) && (per_database > default_per_database))
{
if (! is_super_user(port->user_name))
elog(ERROR, "per-database connection limit reached (limit %d)",
default_per_database);
else
elog(WARNING, "per-database limit reached (limit %d), but the user is a superuser",
default_per_database);
}
/* check per-user limit */
if ((! per_user_overriden) && (default_per_role != 0) && (per_user > default_per_role))
{
if (! is_super_user(port->user_name))
elog(ERROR, "per-user connection limit reached (limit %d)",
default_per_role);
else
elog(WARNING, "per-user connection limit reached (limit %d), but the user is a superuser",
default_per_role);
}
/* check per-IP limit */
if ((! per_ip_overriden) && (default_per_ip != 0) && (per_ip > default_per_ip))
{
if (! is_super_user(port->user_name))
elog(ERROR, "per-IP connection limit reached (limit %d)",
default_per_ip);
else
elog(WARNING, "per-IP connection limit reached (limit %d), but the user is a superuser",
default_per_ip);
}
LWLockRelease(ProcArrayLock);
}
Datum
connection_limits(PG_FUNCTION_ARGS)
{
FuncCallContext *funcctx;
TupleDesc tupdesc;
AttInMetadata *attinmeta;
/* init on the first call */
if (SRF_IS_FIRSTCALL())
{
MemoryContext oldcontext;
funcctx = SRF_FIRSTCALL_INIT();
oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
reset_rules();
check_all_rules();
/* number of rules */
funcctx->max_calls = rules->n_rules;
/* Build a tuple descriptor for our result type */
if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("function returning record called in context "
"that cannot accept type record")));
/*
* generate attribute metadata needed later to produce tuples from raw
* C strings
*/
attinmeta = TupleDescGetAttInMetadata(tupdesc);
funcctx->attinmeta = attinmeta;
funcctx->tuple_desc = tupdesc;
/* switch back to the old context */
MemoryContextSwitchTo(oldcontext);
}
/* init the context */
funcctx = SRF_PERCALL_SETUP();
/* check if we have more data */
if (funcctx->max_calls > funcctx->call_cntr)
{
HeapTuple tuple;
Datum result;
Datum values[6];
bool nulls[6];
rule_t * rule = &(rules->rules[funcctx->call_cntr]);
memset(nulls, 0, sizeof(nulls));
/* rule line */
values[0] = UInt32GetDatum(rule->line);
/* database */
if (rule->fields & CHECK_DBNAME)
values[1] = CStringGetTextDatum(rule->database);
else
nulls[1] = TRUE;
/* username */
if (rule->fields & CHECK_USER)
values[2] = CStringGetTextDatum(rule->user);
else
nulls[2] = TRUE;
/* hostname or IP address */
if (rule->fields & CHECK_HOST)
values[3] = CStringGetTextDatum(rule->hostname);
else if (rule->fields & CHECK_IP)
{
char buffer[256];
memset(buffer, 0, 256);
format_address(buffer, 256, (struct sockaddr*)&rule->ip, (struct sockaddr*)&rule->mask);
values[3] = CStringGetTextDatum(buffer);
}
else
nulls[3] = TRUE;
/* count and limit */
values[4] = UInt32GetDatum(rule->count);
values[5] = UInt32GetDatum(rule->limit);
/* Build and return the tuple. */
tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
/* make the tuple into a datum */
result = HeapTupleGetDatum(tuple);
/* Here we want to return another item: */
SRF_RETURN_NEXT(funcctx, result);
}
else
{
/* lock ProcArray (serialize the processes) */
LWLockRelease(ProcArrayLock);
/* Here we are done returning items and just need to clean up: */
SRF_RETURN_DONE(funcctx);
}
}
void lzw_encode(LZWEncoderData *ld, uint8_t *input, int input_length){
outputCode(ld, ld->clear_code_number);
for(uint8_t end_index=1; end_index<=input_length; end_index++){
#ifdef DEBUG
printf("\nStarting Search from %d to %d: ", ld->start_index, end_index);
for(int input_byte_index=0; input_byte_index<input_length; input_byte_index++){
if(input_byte_index==ld->start_index)printf("\e[1;34m");
printf("%d ",input[input_byte_index]);
if(input_byte_index+1==end_index)printf("\e[0m");
}
printf("\e[0m\n");
for(int j=0; j<end_index-ld->start_index; j++) printf("%d", input[ld->start_index+j]);
printf("\n");
#endif
LinkedListItem *item = ld->dictionary_branch->first;
LinkedListItem *nextitem;
for(int subselect_index=ld->resolved_indexes; subselect_index<end_index-ld->start_index; subselect_index++){
#ifdef DEBUG
printf("Checking symbol #%d (%d)\n", ld->start_index+subselect_index, input[ld->start_index+subselect_index]);
#endif
uint8_t found = 0;
do{
if(item==NULL){
#ifdef DEBUG
printf("NO CHILDREN FOUND\n");
#endif
break;
}
LZWTreeEntry *treeentry = (LZWTreeEntry*)item->data;
#ifdef DEBUG
printf("checking against level %d, rule id: %d value: %d\n", treeentry->level, treeentry->code_number, treeentry->data);
#endif
if(treeentry->data==input[ld->start_index+subselect_index]){
#ifdef DEBUG
printf("FOUND level %d rule %d; id %d\n", treeentry->level, treeentry->data, treeentry->code_number);
#endif
found=1;
ld->resolved_indexes++;
ld->dictionary_branch=(LinkedList *)treeentry->children;
ld->last_matched_rule = treeentry;
break;
}
nextitem = item->next;
item=nextitem;
}while(item!=0);
if(found!=1){
#ifdef DEBUG
printf("\e[1;33mADDING RULE id: %d; ", ld->next_rule_id);
for(int rule_data_index=0; rule_data_index<end_index-ld->start_index; rule_data_index++) printf("%d", input[ld->start_index+rule_data_index]);
printf("\e[0m\n");
#endif
outputCode(ld, ld->last_matched_rule->code_number);
LinkedListItem *new_rule_entry = addNewLinkedListItem(ld->dictionary_branch);
LZWTreeEntry *treeentry = malloc(sizeof(LZWTreeEntry));
new_rule_entry->data = (void*)treeentry;
treeentry->children = malloc(sizeof(LinkedList));
memset(treeentry->children, 0, sizeof(LinkedList));
treeentry->data = input[ld->start_index+(end_index-ld->start_index)-1];
treeentry->code_number = ld->next_rule_id;
treeentry->level = ld->last_matched_rule->level+1;
ld->dictionary_branch = &ld->dictionary;
ld->start_index=end_index-1;
end_index--;
ld->resolved_indexes = 0;
ld->last_matched_rule=NULL;
ld->next_rule_id++;
int nextLZWmin = (int)ceil(log2(ld->next_rule_id));
if(nextLZWmin>12){
outputCode(ld, ld->clear_code_number);
freeRuleDictionary(&ld->dictionary);
ld->next_rule_id = reset_rules(ld);
ld->clear_code_number = ld->next_rule_id;
ld->end_code_number = ld->next_rule_id+1;
ld->next_rule_id+=2;
ld->LZWmin = (int)ceil(log2(ld->next_rule_id));
}else
ld->LZWmin = nextLZWmin;
break;
}
}
}
int lazy_transductive_classification() {
__START_TIMER__
CACHE.locked=1;
evaluation_t evaluator;
prediction_t prediction;
initialize_evaluation(&evaluator);
int abstain=0, n_tests=1, remaining_tests=N_TESTS, locked=1;
float min_level=MIN_LEVEL;
list_t* test=TEST;
while(test) {
reset_rules();
prediction=get_THE_prediction(test->instance, test->size, test->label);
if(!locked || min_level<0.1 || (prediction.score.points[prediction.score.ordered_labels[0]]>min_level && prediction.score.points[prediction.score.ordered_labels[0]]<1)) {
update_evaluation(&evaluator, prediction.label, test->label);
print_statistics(n_tests, test->label, test->id, prediction, evaluator);
free(prediction.score.ordered_labels);
free(prediction.score.points);
if(locked) {
N_TRANSACTIONS++;
COUNT_TARGET[prediction.label]++;
if(COUNT_TARGET[prediction.label]==1) {
ITEMSETS[TARGET_ID[prediction.label]].count=1;
ITEMSETS[prediction.label].list[0]=N_TRANSACTIONS;
}
else {
ITEMSETS[TARGET_ID[prediction.label]].list[ITEMSETS[TARGET_ID[prediction.label]].count]=N_TRANSACTIONS;
ITEMSETS[TARGET_ID[prediction.label]].count++;
}
for(int j=0;j<test->size;j++) {
ITEMSETS[test->instance[j]].list[ITEMSETS[test->instance[j]].count]=N_TRANSACTIONS;
ITEMSETS[test->instance[j]].count++;
}
}
n_tests++;
remaining_tests--;
abstain=0;
list_t* x=test->next;
free(test->instance);
free(test);
test=x;
TEST=test;
}
else if(abstain<remaining_tests) {
printf("abstaining %d %d %f %f %f\n", abstain, remaining_tests, min_level, prediction.score.points[prediction.score.ordered_labels[0]]/(float)prediction.score.points[prediction.score.ordered_labels[1]], avg_size);
abstain++;
list_t* q=TEST;
while(q->next!=NULL) q=q->next;
list_t* t=(list_t*)malloc(sizeof(list_t));
t->instance=(int*)malloc(sizeof(int)*test->size);
t->size=0;
for(int i=0;i<test->size;i++) t->instance[t->size++]=test->instance[i];
t->id=test->id;
t->label=test->label;
t->next=NULL;
q->next=t;
list_t* x=test->next;
free(test->instance);
free(test);
test=x;
TEST=test;
}
else locked=0;
}
printf("\n");
for(int i=0;i<MAX_CLASSES;i++) printf("CLASS(%d)= %d Prec= %f Rec= %f F1= %f ", i, evaluator.true_labels[i], evaluator.precision[i], evaluator.recall[i], evaluator.f1[i]);
printf("Acc= %f MF1= %f * hits= %ld misses= %ld\n", evaluator.acc, evaluator.mf1, CACHE.hits, CACHE.misses);
//finalize_evaluation(&evaluator);
__FINISH_TIMER__
return(0);
}
void mrp_debug_reset(void)
{
debug_enabled = FALSE;
reset_rules();
}
