static void smq_destroy(struct dm_cache_policy *p)
{
struct smq_policy *mq = to_smq_policy(p);
h_exit(&mq->hotspot_table);
h_exit(&mq->table);
free_bitset(mq->hotspot_hit_bits);
free_bitset(mq->cache_hit_bits);
space_exit(&mq->es);
kfree(mq);
}
/**
* Calculates matrix P(s,alpha,B).
*
* The result will be a one-dimensional array. Entries will be in the
* following order: The first entry is for the first state, first
* non-deterministic selection, then the one for the second
* non-deterministic decision follows and so forth. Then the ones for
* the second state follow, etc.
*
* @param sparse sparse matrix to calculate for
* @param B set of states in B
* @return P(s,alpha,B)
*/
static double * calculate_P_s_alpha_B(const NDSparseMatrix * sparse_local,
const bitset * B)
{
double *P_s_alpha_B;
unsigned ps_index;
unsigned * row_starts = (unsigned *) sparse_local->row_counts;
unsigned * choice_starts = (unsigned *) sparse_local->choice_counts;
double * non_zeros = sparse_local->non_zeros;
unsigned * cols = sparse_local->cols;
/* calculate final size of P(s,alpha,B) */
unsigned ps_size = 0;
state_index state_nr;
bitset * not_B = not(B);
/* get_idx_next_non_zero() is more efficient than testing every bit in
B individually. David N. Jansen. */
state_nr = state_index_NONE;
while ( (state_nr = get_idx_next_non_zero(not_B, state_nr))
!= state_index_NONE )
{
ps_size += row_starts[state_nr + 1] - row_starts[state_nr];
}
/* allocate and fill P(s,alpha,B). We precompute it for later
* usage in main part of the algorithm. */
P_s_alpha_B = (double *) malloc(ps_size * sizeof(double));
ps_index = 0;
state_nr = state_index_NONE;
while ( (state_nr = get_idx_next_non_zero(not_B, state_nr))
!= state_index_NONE )
{
BITSET_BLOCK_TYPE s_in_B;
unsigned state_start = row_starts[state_nr];
unsigned state_end = row_starts[state_nr + 1];
unsigned choice_nr;
for (choice_nr = state_start; choice_nr < state_end; choice_nr++) {
unsigned i;
unsigned i_start = choice_starts[choice_nr];
unsigned i_end = choice_starts[choice_nr + 1];
P_s_alpha_B[ps_index] = 0.0;
for (i = i_start; i < i_end; i++) {
s_in_B = get_bit_val(B, cols[i]);
if ( !(s_in_B == BIT_OFF) ) {
P_s_alpha_B[ps_index] += non_zeros[i];
}
}
ps_index++;
}
}
free_bitset(not_B);
return P_s_alpha_B;
}
static int* _factoradic_to_permutation_step(int *data, int length, int *result, int index, Bitset *old) {
if (index < length) {
int nth = bitset_nth_bit_true(old, data[index] + 1);
result[index] = nth;
bitset_reset(old, nth);
return _factoradic_to_permutation_step(data, length, result, index + 1, old);
}
free_bitset(old);
return result;
}
static int* _permutation_to_factoradic_step(int *data, int length, int index, int *result, Bitset* old) {
if (index < length) {
int count = bitset_count_part(old, 0, data[index] - 1);
if (count < 0) count = 0;
result[index] = count;
bitset_reset(old, data[index]);
return _permutation_to_factoradic_step(data, length, index + 1, result, old);
}
free_bitset(old);
return result;
}
static struct dm_cache_policy *smq_create(dm_cblock_t cache_size,
sector_t origin_size,
sector_t cache_block_size)
{
unsigned i;
unsigned nr_sentinels_per_queue = 2u * NR_CACHE_LEVELS;
unsigned total_sentinels = 2u * nr_sentinels_per_queue;
struct smq_policy *mq = kzalloc(sizeof(*mq), GFP_KERNEL);
if (!mq)
return NULL;
init_policy_functions(mq);
mq->cache_size = cache_size;
mq->cache_block_size = cache_block_size;
calc_hotspot_params(origin_size, cache_block_size, from_cblock(cache_size),
&mq->hotspot_block_size, &mq->nr_hotspot_blocks);
mq->cache_blocks_per_hotspot_block = div64_u64(mq->hotspot_block_size, mq->cache_block_size);
mq->hotspot_level_jump = 1u;
if (space_init(&mq->es, total_sentinels + mq->nr_hotspot_blocks + from_cblock(cache_size))) {
DMERR("couldn't initialize entry space");
goto bad_pool_init;
}
init_allocator(&mq->writeback_sentinel_alloc, &mq->es, 0, nr_sentinels_per_queue);
for (i = 0; i < nr_sentinels_per_queue; i++)
get_entry(&mq->writeback_sentinel_alloc, i)->sentinel = true;
init_allocator(&mq->demote_sentinel_alloc, &mq->es, nr_sentinels_per_queue, total_sentinels);
for (i = 0; i < nr_sentinels_per_queue; i++)
get_entry(&mq->demote_sentinel_alloc, i)->sentinel = true;
init_allocator(&mq->hotspot_alloc, &mq->es, total_sentinels,
total_sentinels + mq->nr_hotspot_blocks);
init_allocator(&mq->cache_alloc, &mq->es,
total_sentinels + mq->nr_hotspot_blocks,
total_sentinels + mq->nr_hotspot_blocks + from_cblock(cache_size));
mq->hotspot_hit_bits = alloc_bitset(mq->nr_hotspot_blocks);
if (!mq->hotspot_hit_bits) {
DMERR("couldn't allocate hotspot hit bitset");
goto bad_hotspot_hit_bits;
}
clear_bitset(mq->hotspot_hit_bits, mq->nr_hotspot_blocks);
if (from_cblock(cache_size)) {
mq->cache_hit_bits = alloc_bitset(from_cblock(cache_size));
if (!mq->cache_hit_bits) {
DMERR("couldn't allocate cache hit bitset");
goto bad_cache_hit_bits;
}
clear_bitset(mq->cache_hit_bits, from_cblock(mq->cache_size));
} else
mq->cache_hit_bits = NULL;
mq->tick = 0;
spin_lock_init(&mq->lock);
q_init(&mq->hotspot, &mq->es, NR_HOTSPOT_LEVELS);
mq->hotspot.nr_top_levels = 8;
mq->hotspot.nr_in_top_levels = min(mq->nr_hotspot_blocks / NR_HOTSPOT_LEVELS,
from_cblock(mq->cache_size) / mq->cache_blocks_per_hotspot_block);
q_init(&mq->clean, &mq->es, NR_CACHE_LEVELS);
q_init(&mq->dirty, &mq->es, NR_CACHE_LEVELS);
stats_init(&mq->hotspot_stats, NR_HOTSPOT_LEVELS);
stats_init(&mq->cache_stats, NR_CACHE_LEVELS);
if (h_init(&mq->table, &mq->es, from_cblock(cache_size)))
goto bad_alloc_table;
if (h_init(&mq->hotspot_table, &mq->es, mq->nr_hotspot_blocks))
goto bad_alloc_hotspot_table;
sentinels_init(mq);
mq->write_promote_level = mq->read_promote_level = NR_HOTSPOT_LEVELS;
mq->next_hotspot_period = jiffies;
mq->next_cache_period = jiffies;
return &mq->policy;
bad_alloc_hotspot_table:
h_exit(&mq->table);
bad_alloc_table:
free_bitset(mq->cache_hit_bits);
bad_cache_hit_bits:
free_bitset(mq->hotspot_hit_bits);
bad_hotspot_hit_bits:
space_exit(&mq->es);
bad_pool_init:
kfree(mq);
return NULL;
}
/**
* Actual CTMDPI bounded reachability algorithm.
*
* @param unsigned num_states number of states of CTMDPI
* @param fg Poisson values by Fox-Glynn algorithm
* @param left_end use Poisson probabilities up to left limit
* @param min true for minimal probability, false for maximal one
* @param row_starts see CTMDPI documentation
* @param choice_starts see CTMDPI documentation
* @param non_zeros see CTMDPI documentation
* @param q current probability vector
* @param q_primed next probabilities vector
* @param cols see CTMDPI documentation
* @param P_s_alpha_B P(s,alpha,B) (see paper)
* @param B target states
*/
static double *ctmdpi_iter( const unsigned num_states, const FoxGlynn *fg, const unsigned left_end,
const BOOL min, const int * row_starts,
const int * choice_starts,
const double *non_zeros, double *q, double *q_primed, const unsigned *cols,
const double *P_s_alpha_B, const bitset *B) {
/* in the algorithm the main iteration is in lines 3-12. For each
* jump probability k starting from the right bound in the Fox-Glynn
* algorithm down to 1 the loop body is to be executed once. In the
* implementation here we split this loop into two parts: One for
* the non-negligible Poisson probabilities between left and right
* bound of the Fox-Glynn algorithm and one for the jump
* probabilities below left bound, if any.
*/
unsigned i;
state_index row;
bitset * not_B;
/* first part - use poisson probabilities */
/* line 3 in paper */
for (i = fg->right; i >= left_end; i--) {
unsigned ps_index = 0;
double psi = fg->weights[i-fg->left] / fg->total_weight;
/* lines 4-12 in paper */
for ( row = 0 ; (unsigned) row < num_states ; row++ ) {
BITSET_BLOCK_TYPE s_in_B;
s_in_B = get_bit_val(B, row);
if ( s_in_B == BIT_OFF ) {
/* get maximizing/minimizing decision */
/* lines 4-10 in paper (non-target state) */
double m = min ? 2.0 : -1.0;
unsigned l1 = row_starts[row];
unsigned h1 = row_starts[row+1];
unsigned j;
for (j = l1; j < h1; j++) {
double m_primed = get_choice_probability_psi
(j, choice_starts, non_zeros, cols,q, P_s_alpha_B, ps_index, psi);
m = optimum(min, m, m_primed);
ps_index++;
}
/* in case some non-target state did
* not have any possible decisions we
* set m to zero afterwards */
m = ((-1.0 == m) || (2.0 == m)) ? 0.0 : m;
/* line 9 in paper*/
q_primed[row] = m;
} else {
/* line 11 in paper (target state) */
q_primed[row] = psi + q[row];
}
}
/* swapping is done instead using a new q vector for
* each k */
swap(&q, &q_primed);
}
/* we have to set the q_primed values for target states once
* more now; they won't change afterwards, as psi =
* 0.0. We do here what would be done in the first iteration
* of the next loop. As the values for q_primed do not
* influence the first iteration of the next loop, we can do
* this beforehand to avoid having to do this in each loop. */
row = state_index_NONE;
while ( (row = get_idx_next_non_zero(B, row)) != state_index_NONE )
{ /* target states only */
q_primed[row] = q[row];
}
not_B = not(B);
/* now do part where we are below left bound of poisson probabilities */
for (; i > 0; i--) {
unsigned ps_index = 0;
/* lines 4-12 in paper */
row = state_index_NONE;
while ( (row = get_idx_next_non_zero(not_B, row))
!= state_index_NONE )
{ /* non-target state? */
/* get maximizing/minimizing decision */
/* lines 4-10 in paper (non-target state) */
double m = min ? 2 : -1.0;
unsigned l1 = row_starts[row];
unsigned h1 = row_starts[row+1];
unsigned j;
for (j = l1; j < h1; j++) {
double m_primed = get_choice_probability
(j, choice_starts, non_zeros, cols, q);
m = optimum(min, m, m_primed);
ps_index++;
}
/* in case some non-target state did
* not have any possible decisions we
* set m to zero afterwards */
m = ((-1.0 == m) || (2.0 == m)) ? 0.0 : m;
/* line 9 in paper*/
q_primed[row] = m;
}
/* swapping is done instead using a new q vector for
* each k */
swap(&q, &q_primed);
}
free_bitset(not_B);
return q;
}
