void analyse_buffer(const unsigned char *buffer, const unsigned int length_in_bytes)
{
enhanced_packet_block_t *epb = (enhanced_packet_block_t *)buffer;
uint16_t ethertype;
void *payload;
ethernet_hdr_t *hdr = (ethernet_hdr_t *) &(epb->data);
ethertype = ntoh16(hdr->ethertype);
// Packet must be VLAN tagged
if (ethertype != 0x8100)
return;
tagged_ethernet_hdr_t *tagged_hdr = (tagged_ethernet_hdr_t *) &(epb->data);
ethertype = ntoh16(tagged_hdr->ethertype);
payload = &(tagged_hdr->payload);
if (ethertype != AVB_1722_ETHERTYPE)
return;
AVB_DataHeader_t *avb_hdr = (AVB_DataHeader_t *)payload;
unsigned int subtype = AVBTP_SUBTYPE(avb_hdr);
if (subtype == 0) {
stream_id_t id;
id.low = AVBTP_STREAM_ID0(avb_hdr);
id.high = AVBTP_STREAM_ID1(avb_hdr);
if (id.low != 0 || id.high != 0)
increment_count(&id, epb->packet_len);
}
}
int
vga_op_list_write_bassign (vga_op_list_t* vol,
size_t address,
size_t size,
bd_t bd)
{
if (reset (vol) != 0) {
return -1;
}
size_t bd_size = buffer_size (bd);
if (bd_size == -1 || size > bd_size * pagesize ()) {
/* The buffer was too small. */
return -1;
}
/* Find the offset in pages of the data. */
size_t bd_offset = buffer_size (vol->bdb);
/* Append the data. */
if (buffer_append (vol->bdb, bd) != 0) {
return -1;
}
vga_op_type_t type = VGA_BASSIGN;
if (buffer_file_write (&vol->bf, &type, sizeof (vga_op_type_t)) != 0 ||
buffer_file_write (&vol->bf, &address, sizeof (size_t)) != 0 ||
buffer_file_write (&vol->bf, &size, sizeof (size_t)) != 0 ||
buffer_file_write (&vol->bf, &bd_offset, sizeof (size_t)) != 0) {
return -1;
}
return increment_count (vol);
}
// Increment count in _access_count[tag]
void ScanBlocks::accumulate_access(int index, ValueTag tag, int count) {
increment_count(tag, index, count);
update_type(index, tag);
if (tag == doubleTag || tag == longTag) {
update_type(index + 1, tag);
}
}
void AdaptiveWeightedAverage::sample(float new_sample) {
increment_count();
assert(count() != 0,
"Wraparound -- history would be incorrectly discarded");
// Compute the new weighted average
float new_avg = compute_adaptive_average(new_sample, average());
set_average(new_avg);
_last_sample = new_sample;
}
int
vga_op_list_write_set_cursor_location (vga_op_list_t* vol,
size_t location)
{
if (reset (vol) != 0) {
return -1;
}
vga_op_type_t type = VGA_SET_CURSOR_LOCATION;
if (buffer_file_write (&vol->bf, &type, sizeof (vga_op_type_t)) != 0 ||
buffer_file_write (&vol->bf, &location, sizeof (size_t)) != 0) {
return -1;
}
return increment_count (vol);
}
int
vga_op_list_write_assign (vga_op_list_t* vol,
size_t address,
const void* data,
size_t size)
{
if (reset (vol) != 0) {
return -1;
}
vga_op_type_t type = VGA_ASSIGN;
if (buffer_file_write (&vol->bf, &type, sizeof (vga_op_type_t)) != 0 ||
buffer_file_write (&vol->bf, &address, sizeof (size_t)) != 0 ||
buffer_file_write (&vol->bf, &size, sizeof (size_t)) != 0 ||
buffer_file_write (&vol->bf, data, size) != 0) {
return -1;
}
return increment_count (vol);
}
OopMap::OopMap(OopMap::DeepCopyToken, OopMap* source) {
// This constructor does a deep copy
// of the source OopMap.
set_write_stream(new CompressedWriteStream(source->omv_count() * 2));
set_omv_count(0);
set_offset(source->offset());
#ifdef ASSERT
_locs_length = source->_locs_length;
_locs_used = NEW_RESOURCE_ARRAY(OopMapValue::oop_types, _locs_length);
for(int i = 0; i < _locs_length; i++) _locs_used[i] = OopMapValue::unused_value;
#endif
// We need to copy the entries too.
for (OopMapStream oms(source); !oms.is_done(); oms.next()) {
OopMapValue omv = oms.current();
omv.write_on(write_stream());
increment_count();
}
}
/* Estimate entropy and Markov models */
void get_statistics(struct Statistics *stats, FILE* fp)
{
unsigned char *buffer;
unsigned char old_buffer[BUFFER_SIZE+2];
int total = 0;
double sum = 0.0;
double probability;
size_t i, bytes_read;
init_statistics(stats);
rewind(fp);
buffer = old_buffer+2;
/* Read the first two characters */
old_buffer[0] = (unsigned char)fgetc(fp);
stats->occurrence.single[old_buffer[0]]++;
old_buffer[1] = (unsigned char)fgetc(fp);
stats->occurrence.single[old_buffer[1]]++;
increment_count(2, old_buffer, stats->occurrence.multi[0]);
total += 2;
while(!feof(fp))
{
/* Fill buffer */
bytes_read = fread(buffer, 1, BUFFER_SIZE, fp);
for(i=0; i<bytes_read; i++)
{
/* Count occurrence of the byte */
stats->occurrence.single[buffer[i]]++;
/* Count occurrence of the latest two bytes */
increment_count(2, buffer+i-1, stats->occurrence.multi[0]);
/* Count occurrence of the latest three bytes */
increment_count(3, buffer+i-2, stats->occurrence.multi[1]);
/* Count total number of bytes */
total++;
}
}
/* Report occurrences */
for(i=0; i<256; i++)
printf("occurrence[%3d] = %d\n", i, stats->occurrence.single[i]);
/* Report file size */
printf("total = %d\n", total);
/* Estimate memoryless entropy */
for(i=0; i<256; i++)
{
if(stats->occurrence.single[i] > 0)
{
/* Calculate probability of the byte */
probability = stats->occurrence.single[i] / (double)total;
/* Add to sum */
sum = sum - (probability * log2(probability));
}
}
stats->entropy.memoryless = sum;
/* Calculate entropy rate of first-order Markov model */
sum = 0;
for(i=0; i<256*256; i++)
{
if(stats->occurrence.multi[0]->mem[i] > 0)
{
/* Calculate probability of the sequence */
probability = stats->occurrence.multi[0]->mem[i] / (double)(total-1);
/* Add to sum */
sum = sum - (probability * log2(probability));
}
}
stats->entropy.markov1 = sum - stats->entropy.memoryless;
/* Calculate entropy rate of second-order Markov model */
sum = 0;
for(i=0; i<256*256*256; i++)
{
if(stats->occurrence.multi[1]->mem[i] > 0)
{
/* Calculate probability of the sequence */
probability = stats->occurrence.multi[1]->mem[i] / (double)(total-2);
/* Add to sum */
sum = sum - (probability * log2(probability));
}
}
stats->entropy.markov2 = sum
- stats->entropy.markov1
- stats->entropy.memoryless;
/* Report memoryless entropy */
printf("Memoryless entropy: %f bit(s)\n", stats->entropy.memoryless);
/* Report entropy rate of first-order Markov model */
printf("Markov model (Order 1) entropy rate: %f bit(s)\n", stats->entropy.markov1);
/* Report entropy rate of second-order Markov model */
printf("Markov model (Order 2) entropy rate: %f bit(s)\n", stats->entropy.markov2);
free_statistics(stats);
return;
}
static t_block allocate_block(t_mchecker mc, t_block list, x_size size, x_boolean cleared) {
x_ubyte * data;
x_ubyte pattern;
t_block b;
x_size i;
x_word tag;
x_size which = x_random() % 5;
tag = ID_anon;
x_mutex_lock(test_mutex, x_eternal);
/*
** We use the which parameter to bring some variation in the __LINE__ number
** for checking the memory dumper.
*/
if (cleared) {
if (which == 0) {
b = allocClearedMem(sizeof(t_Block) + size);
}
else if (which == 1) {
b = allocClearedMem(sizeof(t_Block) + size);
}
else if (which == 2) {
b = allocClearedMem(sizeof(t_Block) + size);
}
else if (which == 3) {
b = allocClearedMem(sizeof(t_Block) + size);
}
else {
b = allocClearedMem(sizeof(t_Block) + size);
}
}
else {
if (which == 0) {
b = allocMem(sizeof(t_Block) + size);
}
else if (which == 1) {
b = allocMem(sizeof(t_Block) + size);
}
else if (which == 2) {
b = allocMem(sizeof(t_Block) + size);
}
else if (which == 3) {
b = allocMem(sizeof(t_Block) + size);
}
else {
b = allocMem(sizeof(t_Block) + size);
}
}
if (b == NULL) {
x_mutex_unlock(test_mutex);
return NULL;
}
/*
** Check block is cleared correctly.
*/
if (cleared) {
data = (x_ubyte *)b;
for (i = 0; i < sizeof(t_Block) + size; i++) {
if (*data++ != 0x00) {
oempa("Block of size %d not cleared properly. data[%d] == 0x%02x\n", sizeof(t_Block) + size, i, data[i]);
exit(0);
}
}
}
b->size = size;
pattern = (x_ubyte) x_random();
b->pattern = pattern;
/*
** Write the pattern in the block.
*/
data = b->data;
set_data(data, size, pattern);
x_list_insert(list, b);
increment_count(b, size);
setMemTag(b, 33);
x_mutex_unlock(test_mutex);
return b;
}
static void increment_pin_toggle(uint32_t pin)
{
assert(pin < PIN_COUNT);
increment_count(toggles, pin);
}
static void increment_pin_hit(uint32_t pin)
{
assert(pin < PIN_COUNT);
increment_count(hits, pin);
}
void perform_verification(storm_file_handle_t *s_handle,
track_file_handle_t *t_handle,
date_time_t *scan_time,
vt_count_t *total_count,
vt_stats_t *total_stats)
{
static ui08 **forecast_grid;
static ui08 **truth_grid;
static int first_call = TRUE;
static long nbytes_grid;
int verify_valid;
long icomplex, isimple, istorm;
long iscan, jscan;
long nvalid, nverify;
long nstorms;
long n_scans;
long forecast_scan;
long complex_track_num;
long simple_track_num;
long n_simple_tracks;
long first_scan_searched, last_scan_searched;
double time_diff, min_time_diff;
double now, forecast_time;
double forecast_lead_time, closest_lead_time;
double verify_min_history;
complex_track_params_t *ct_params;
storm_file_params_t *sparams;
track_file_params_t *tparams;
track_file_forecast_props_t props_current;
track_file_forecast_props_t props_forecast;
track_file_forecast_props_t props_verify;
vt_stats_t complex_stats, file_stats;
vt_simple_track_t *stracks, *strack;
vt_storm_t *storm;
vt_entry_index_t valid[MAX_BRANCHES];
vt_count_t count;
vt_count_t file_count;
vt_count_t complex_count;
n_scans = s_handle->header->n_scans;
sparams = &s_handle->header->params;
tparams = &t_handle->header->params;
/*
* allocate grids
*/
if (first_call) {
nbytes_grid = Glob->nx * Glob->ny;
forecast_grid = (ui08 **) ucalloc2
((ui32) Glob->ny, (ui32) Glob->nx, (ui32) sizeof(ui08));
truth_grid = (ui08 **) ucalloc2
((ui32) Glob->ny, (ui32) Glob->nx, (ui32) sizeof(ui08));
first_call = FALSE;
} /* if (first_call) */
memset ((void *) &file_count,
(int) 0, (size_t) sizeof(vt_count_t));
memset ((void *) &file_stats,
(int) 0, (size_t) sizeof(vt_stats_t));
/*
* Set the verify_min_history. If the 'verify_before_forecast_time'
* flag is set, verify_min_history is set to 0. This allows the
* inclusion of storms which are too young to have been forecast using
* this system, and hence verifies all convective activity. If the
* flag is not set, verify_min_history is set to the sum of the
* forecast_lead_time and the min_valid_history, since this is
* the minimum history for a forecast to be valid.
*/
if (Glob->verify_before_forecast_time)
verify_min_history = 0.0;
else
verify_min_history =
Glob->forecast_lead_time + Glob->forecast_min_history;
/*
* loop through the complex tracks
*/
for (icomplex = 0;
icomplex < t_handle->header->n_complex_tracks; icomplex++) {
/*
* initialize
*/
memset ((void *) &complex_count,
(int) 0, (size_t) sizeof(vt_count_t));
memset ((void *) &complex_stats,
(int) 0, (size_t) sizeof(vt_stats_t));
if (Glob->debug) {
fprintf(stderr, "=========================================================\n");
}
/*
* read in the complex track params
*/
complex_track_num = t_handle->complex_track_nums[icomplex];
if(RfReadComplexTrackParams(t_handle, complex_track_num, TRUE,
"perform_verification"))
tidy_and_exit(-1);
ct_params = t_handle->complex_params;
/*
* initialize flags and counters for track problems
*/
ct_params->n_top_missing = 0;
ct_params->n_range_limited = 0;
ct_params->start_missing = FALSE;
ct_params->end_missing = FALSE;
ct_params->volume_at_start_of_sampling = 0;
ct_params->volume_at_end_of_sampling = 0;
/*
* set flags if track existed at start or end of sampling
*/
if (t_handle->complex_params->start_time ==
s_handle->header->start_time) {
ct_params->start_missing = TRUE;
}
if (t_handle->complex_params->end_time ==
s_handle->header->end_time) {
ct_params->end_missing = TRUE;
}
/*
* allocate array for simple tracks
*/
n_simple_tracks = t_handle->complex_params->n_simple_tracks;
stracks = (vt_simple_track_t *) umalloc
((ui32) (n_simple_tracks * sizeof(vt_simple_track_t)));
/*
* read in simple tracks
*/
for (isimple = 0; isimple < n_simple_tracks; isimple++) {
strack = stracks + isimple;
simple_track_num =
t_handle->simples_per_complex[complex_track_num][isimple];
/*
* read in simple track params and prepare entries for reading
*/
if(RfRewindSimpleTrack(t_handle, simple_track_num,
"perform_verification"))
tidy_and_exit(-1);
/*
* find last descendant, insert relevant values in the
* simple params struct and store
*/
find_last_descendant(t_handle, 0);
/*
* copy simple params
*/
memcpy ((void *) &strack->params,
(void *) t_handle->simple_params,
(size_t) sizeof(simple_track_params_t));
nstorms = t_handle->simple_params->duration_in_scans;
/*
* allocate space for the entries
*/
strack->storms = (vt_storm_t *) umalloc
((ui32) (nstorms * sizeof(vt_storm_t)));
for (istorm = 0; istorm < nstorms; istorm++) {
storm = strack->storms + istorm;
/*
* read in track entry, copy the structs to the local
* array elements
*/
if (RfReadTrackEntry(t_handle, "perform_verification"))
tidy_and_exit(-1);
if (RfReadStormScan(s_handle, t_handle->entry->scan_num,
"perform_verification"))
tidy_and_exit(-1);
if (RfReadStormProps(s_handle, t_handle->entry->storm_num,
"perform_verification"))
tidy_and_exit(-1);
umalloc_verify();
memcpy ((void *) &storm->entry,
(void *) t_handle->entry,
(size_t) sizeof(track_file_entry_t));
memcpy ((void *) &storm->gprops,
(void *) (s_handle->gprops + t_handle->entry->storm_num),
(size_t) sizeof(storm_file_global_props_t));
storm->runs = (storm_file_run_t *) umalloc
((ui32) (storm->gprops.n_runs * sizeof(storm_file_run_t)));
memcpy ((void *) storm->runs,
(void *) s_handle->runs,
(size_t) (storm->gprops.n_runs * sizeof(storm_file_run_t)));
/*
* allocate the lookup tables which relate the scan cartesian
* grid to the verification grid
*/
storm->x_lookup = (long *) umalloc
((ui32) s_handle->scan->grid.nx * sizeof(long));
storm->y_lookup = (long *) umalloc
((ui32) s_handle->scan->grid.ny * sizeof(long));
/*
* compute the lookup tables
*/
compute_lookup(&s_handle->scan->grid,
storm->x_lookup, storm->y_lookup);
/*
* update flags for storm status
*/
if (storm->gprops.top_missing)
ct_params->n_top_missing++;
if (storm->gprops.range_limited)
ct_params->n_range_limited++;
if (storm->entry.time == s_handle->header->start_time) {
ct_params->volume_at_start_of_sampling += storm->gprops.volume;
}
if (storm->entry.time == s_handle->header->end_time) {
ct_params->volume_at_end_of_sampling += storm->gprops.volume;
}
} /* istorm */
} /* isimple */
/*
* Set the first_scan_searched variable. If the
* verify_before_forecast_time flag is set, the search begins at
* the first scan in the storm file, because we need to consider
* forecasts made before the track starts.
* If the flag is not set, the search starts at the
* start of the complex track
*/
if (Glob->verify_before_forecast_time)
first_scan_searched = 0;
else
first_scan_searched = t_handle->complex_params->start_scan;
/*
* now loop through all of the scans in the complex track
*/
for (iscan = first_scan_searched;
iscan <= t_handle->complex_params->end_scan; iscan++) {
/*
* initialize forecast and verification grids, etc
*/
memset ((void *) *forecast_grid,
(int) 0, (size_t) nbytes_grid);
memset ((void *) *truth_grid,
(int) 0, (size_t) nbytes_grid);
memset ((void *) &props_current,
(int) 0, (size_t) sizeof(track_file_forecast_props_t));
memset ((void *) &props_forecast,
(int) 0, (size_t) sizeof(track_file_forecast_props_t));
memset ((void *) &props_verify,
(int) 0, (size_t) sizeof(track_file_forecast_props_t));
nvalid = 0;
nverify = 0;
/*
* compute times
*/
now = (double) scan_time[iscan].unix_time;
forecast_time = now + (double) Glob->forecast_lead_time;
/*
* Set the last_scan_searched variable. If the verify_after_track_dies
* flag is set, the search ends at the last scan in the
* storm file, because we need to consider scans even after the
* track has terminated. If the flag is not set, the search ends
* at the end of the complex track
*/
if (Glob->verify_after_track_dies)
last_scan_searched = n_scans - 1;
else
last_scan_searched = t_handle->complex_params->end_scan;
/*
* find scan number which best corresponds to the forecast time
*/
min_time_diff = LARGE_DOUBLE;
for (jscan = iscan;
jscan <= last_scan_searched; jscan++) {
time_diff =
fabs((double) scan_time[jscan].unix_time - forecast_time);
if (time_diff < min_time_diff) {
forecast_scan = jscan;
min_time_diff = time_diff;
} /* if (time_diff < min_time_diff) */
} /* jscan */
closest_lead_time =
(double) scan_time[forecast_scan].unix_time - now;
/*
* check the forecast lead time to determine whether there is
* track data close to that time - if not, set the verify_valid
* flag to FALSE
*/
if (fabs(closest_lead_time - Glob->forecast_lead_time) <=
Glob->forecast_lead_time_margin) {
forecast_lead_time = closest_lead_time;
verify_valid = TRUE;
} else {
forecast_lead_time = Glob->forecast_lead_time;
verify_valid = FALSE;
}
if (Glob->debug) {
fprintf(stderr, "=========================\n");
fprintf(stderr, "forecast_lead_time : %g\n", forecast_lead_time);
fprintf(stderr, "forecast_scan : %ld\n", forecast_scan);
fprintf(stderr, "verify_valid : %d\n", verify_valid);
}
if (verify_valid) {
/*
* search through all of the entries in the track
*/
for (isimple = 0; isimple < n_simple_tracks; isimple++) {
strack = stracks + isimple;
for (istorm = 0;
istorm < strack->params.duration_in_scans; istorm++) {
storm = strack->storms + istorm;
if (storm->entry.scan_num == iscan &&
storm->entry.history_in_secs >=
Glob->forecast_min_history) {
/*
* this storm is at the current scan number, and its
* duration exceeds that for starting the forecast, so
* compute the forecast and update the forecast grid
*/
if (Glob->debug) {
fprintf(stderr, "Computing Forecast\n");
fprintf(stderr,
"Simple num %ld, ientry %ld, scan %ld, time %s\n",
(long) strack->params.simple_track_num, istorm,
(long) storm->entry.scan_num,
utimstr(storm->entry.time));
} /* if (Glob->debug) */
/*
* load up the forecast props
*/
if (load_props(s_handle, t_handle,
&storm->entry,
&storm->gprops,
forecast_lead_time,
&props_forecast)) {
/*
* load up the current props
*/
load_props(s_handle, t_handle,
&storm->entry,
&storm->gprops,
0.0,
&props_current);
/*
* there is a valid forecast, so proceed.
* The forecast is considered valid it the
* forecast volume exceeds the volume
* threshold
*/
load_forecast_grid(s_handle, t_handle,
&storm->entry,
&storm->gprops,
forecast_lead_time,
forecast_grid);
valid[nvalid].isimple = isimple;
valid[nvalid].istorm = istorm;
nvalid++;
} /* if (load_props (...... */
} /* if (storm->entry.scan_num .... */
if ((storm->entry.scan_num == forecast_scan) &&
storm->entry.history_in_secs >= verify_min_history) {
/*
* this storm is at the forecast time, so
* update the truth grids
*/
if (Glob->debug) {
fprintf(stderr, "Computing Verification\n");
fprintf(stderr,
"Simple num %ld, ientry %ld, scan %ld, time %s\n",
(long) strack->params.simple_track_num, istorm,
(long) storm->entry.scan_num,
utimstr(storm->entry.time));
} /* if (Glob->debug) */
/*
* load props for this scan - the lead time is set to zero
*/
load_props(s_handle, t_handle,
&storm->entry,
&storm->gprops,
0.0,
&props_verify);
nverify++;
load_truth_grid(s_handle, storm, truth_grid);
} /* if (storm->entry.scan_num .... */
} /* istorm */
} /* isimple */
if ((nvalid > 0) ||
(nverify > 0 && Glob->verify_before_forecast_time)) {
/*
* compute the contingency data, summing counters for the
* complex track params
*/
compute_contingency_data(forecast_grid,
truth_grid,
nbytes_grid,
&count);
increment_count(&complex_count, &count);
if (Glob->debug)
debug_print(t_handle, stracks, nvalid, valid,
forecast_grid,
truth_grid,
&count);
/*
* Compute the errors between the forecast and
* verification data
*
* The storm entries are updated to include the verification
* results in the track file - this is then rewritten.
*/
compute_errors(s_handle,
nverify,
&props_current,
&props_forecast,
&props_verify,
&complex_stats,
&file_stats,
total_stats);
} /* if (nvalid > 0 ... ) */
} /* if (verify_valid) */
} /* iscan */
/*
* rewrite the track entries
*/
strack = stracks;
for (isimple = 0; isimple < n_simple_tracks; isimple++) {
storm = strack->storms;
for (istorm = 0;
istorm < strack->params.duration_in_scans; istorm++) {
memcpy ((void *) t_handle->entry,
(void *) &storm->entry,
(size_t) sizeof(track_file_entry_t));
if (RfRewriteTrackEntry(t_handle,
"perform_verification"))
tidy_and_exit(-1);
storm++;
} /* istorm */
strack++;
} /* isimple */
/*
* update counts for the file
*/
increment_count(&file_count, &complex_count);
/*
* set contingency data in complex params
*/
set_counts_to_longs(&complex_count,
&t_handle->complex_params->ellipse_verify);
/*
* compute the root mean squared error for the forecast data
*/
compute_stats(&complex_stats);
load_file_stats(tparams,
&complex_stats,
&t_handle->complex_params->n_samples_for_forecast_stats,
&t_handle->complex_params->forecast_bias,
&t_handle->complex_params->forecast_rmse);
/*
* rewrite complex track params
*/
if(RfWriteComplexTrackParams(t_handle, t_handle->complex_track_nums[icomplex],
"perform_verification"))
tidy_and_exit(-1);
/*
* free up structs
*/
strack = stracks;
for (isimple = 0; isimple < n_simple_tracks; isimple++) {
storm = strack->storms;
for (istorm = 0;
istorm < strack->params.duration_in_scans; istorm++) {
ufree((char *) storm->runs);
ufree((char *) storm->x_lookup);
ufree((char *) storm->y_lookup);
storm++;
} /* istorm */
ufree((char *) strack->storms);
strack++;
} /* isimple */
ufree((char *) stracks);
} /* icomplex */
/*
* update total counts
*/
increment_count(total_count, &file_count);
/*
* set contingency data in file header
*/
set_counts_to_longs(&file_count,
&t_handle->header->ellipse_verify);
/*
* compute the root mean squared error for the file forecast data,
* store in header
*/
compute_stats(&file_stats);
load_file_stats(tparams,
&file_stats,
&t_handle->header->n_samples_for_forecast_stats,
&t_handle->header->forecast_bias,
&t_handle->header->forecast_rmse);
/*
* rewrite header
*/
if(RfWriteTrackHeader(t_handle, "perform_verification"))
tidy_and_exit(-1);
}
//process a new token read from the stream
void Estimator_Update(Estimator_type * est, int token)
{
int old_cs0pos, old_backupminuswait, wait;
est->count++;
Freq_Update(est->freq, token);
//end of Misra-Gries part of algorithm
//In the case that a sampler is scheduled to take a new backup and
//primary sample at the same time, we should use more random bits to
//break the tie. But for now, for simplicity, we'll break all such
//ties by having the sampler take a new *primary* sample
//increment count of token, sets processing to 1
c_a* counter = increment_count(est->hashtable, token);
//check for special cases
if(est->count == 1)
{
est->first = counter;
handle_first(est, counter);
return;
}
if(counter->count == est->count)
{
handle_nondistinct(est, counter);
return;
}
if(est->two_distinct_tokens == 0)
{
handle_second_distinct(est, counter);
//indicate that we are done for the time being with two
//distinct tokens in the stream so they can be removed from
//the hashtable if no samplers are sampling them
done_processing(est->hashtable, counter);
done_processing(est->hashtable, est->first);
return;
}
//only restore heap prop if samplers have been put in bheap
restore_bheap_property(est->bheap, counter->backup_pos);
Sample_type* min;
c_a* old_c_s1 = NULL;
while(((Sample_type*) peek_min(est->prim_heap))->prim <= est->count)
{
min=delete_min(est->prim_heap);
if(min->prim < est->count)
{
fprintf(stderr, "a sampler's prim decreased. fatal error\n");
fprintf(stderr, "min->c_s0_key: %d, min->prim %d, est->count %d\n",
min->c_s0->key, min->prim, est->count);
exit(1);
}
//have min take a new primary sample
if(min->c_s0 == counter)
{
min->val_c_s0 = counter->count;
min->t0 *= prng_float(est->prng);
//resample primary and backup wait times using new values of t0 and t1
reset_wait_times(min, est);
restore_c_a_heap_property(min->c_s0->sample_heap, min->c_s0_pos);
restore_bheap_property(est->bheap, min->c_s0->backup_pos);
}
else
{
old_c_s1 = min->c_s1;
min->c_s1 = min->c_s0;
min->val_c_s1 = min->val_c_s0;
min->t1 = min->t0;
min->c_s0 = counter;
min->val_c_s0 = counter->count;
min->t0 *= prng_float(est->prng);
//resample primary and backup wait times using new values of t0 and t1
old_cs0pos = min->c_s0_pos;
old_backupminuswait = min->backup_minus_delay;
reset_wait_times(min, est);
//increment backup samplers for c_s1 first, b/c if we decremented
//prim samplers first and min was the only primary sampler of c_s1 and
//c_s1 had no backup samplers, then c_s1 would be removed from the hashtable
//which we don't want. Note increment_backup_samplers does *not* change
//min->c_s0_pos, so the subsequent call to decrement_prim_samplers will work fine
//when it tries to remove min from c_s1's heap of samplers
increment_backup_samplers(min->c_s1);
decrement_backup_samplers(est->hashtable, old_c_s1);
decrement_prim_samplers(est->hashtable, min->c_s1, est->bheap, min);
increment_prim_samplers(counter, est->bheap, min);
}
//reinsert min into primary heap
insert_heap(est->prim_heap, min);
}
c_a* min2 = peek_min_bheap(est->bheap);
min = peek_min_c_a_heap(min2->sample_heap);
double r1;
while(min->backup_minus_delay + min2->count <= est->count)
{
if(min->backup_minus_delay + min2->count < est->count)
{ //error check
fprintf(stderr, "error: sampler's backup wait time decreased\n");
fprintf(stderr, "bminusd %d, min2->count %d est->count %d\n",
min->backup_minus_delay, min2->count, est->count);
exit(1);
}
decrement_backup_samplers(est->hashtable, min->c_s1);
increment_backup_samplers(counter);
min->t1 -= prng_float(est->prng) * (min->t1-min->t0);
min->c_s1 = counter;
min->val_c_s1 = counter->count;
//recalculate just min's backup wait time
r1 = prng_float(est->prng);
if(r1 == 0) min->backup_minus_delay = est->count + 1 - min->c_s0->count;
else
{
if(min->t1-min->t0 == 0)
{ //t0 == t1 should cause longest possible wait time
min->backup_minus_delay = MAX_WAIT-min->c_s0->count;
}
else
{
wait = ceil(log(r1)/log(1.0-(min->t1-min->t0)));
if(wait < 0 || wait > MAX_WAIT) //check for overflow
min->backup_minus_delay = MAX_WAIT-min->c_s0->count;
else
min->backup_minus_delay = wait + est->count - min->c_s0->count;
}
}
//fprintf(stderr, "%d ", min->backup_minus_delay);
//put min in proper position in its primary sample's heap
restore_c_a_heap_property(min->c_s0->sample_heap, min->c_s0_pos);
//put min's primary sample in proper position in backup heap
restore_bheap_property(est->bheap, min->c_s0->backup_pos);
min2 = peek_min_bheap(est->bheap);
min = peek_min_c_a_heap(min2->sample_heap);
}
done_processing(est->hashtable, counter);
}
