/* * This is a common helper function for find_next_bit, find_next_zero_bit, and * find_next_and_bit. The differences are: * - The "invert" argument, which is XORed with each fetched word before * searching it for one bits. * - The optional "addr2", which is anded with "addr1" if present. */ static inline unsigned long _find_next_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long nbits, unsigned long start, unsigned long invert) { unsigned long tmp; if (unlikely(start >= nbits)) return nbits; tmp = addr1[start / BITS_PER_LONG]; if (addr2) tmp &= addr2[start / BITS_PER_LONG]; tmp ^= invert; /* Handle 1st word. */ tmp &= BITMAP_FIRST_WORD_MASK(start); start = round_down(start, BITS_PER_LONG); while (!tmp) { start += BITS_PER_LONG; if (start >= nbits) return nbits; tmp = addr1[start / BITS_PER_LONG]; if (addr2) tmp &= addr2[start / BITS_PER_LONG]; tmp ^= invert; } return min(start + __ffs(tmp), nbits); }
bool bitmap_test_and_clear_atomic(unsigned long *map, long start, long nr) { unsigned long *p = map + BIT_WORD(start); const long size = start + nr; int bits_to_clear = BITS_PER_LONG - (start % BITS_PER_LONG); unsigned long mask_to_clear = BITMAP_FIRST_WORD_MASK(start); unsigned long dirty = 0; unsigned long old_bits; /* First word */ if (nr - bits_to_clear > 0) { old_bits = atomic_fetch_and(p, ~mask_to_clear); dirty |= old_bits & mask_to_clear; nr -= bits_to_clear; bits_to_clear = BITS_PER_LONG; mask_to_clear = ~0UL; p++; } /* Full words */ if (bits_to_clear == BITS_PER_LONG) { while (nr >= BITS_PER_LONG) { if (*p) { old_bits = atomic_xchg(p, 0); dirty |= old_bits; } nr -= BITS_PER_LONG; p++; } } /* Last word */ if (nr) { mask_to_clear &= BITMAP_LAST_WORD_MASK(size); old_bits = atomic_fetch_and(p, ~mask_to_clear); dirty |= old_bits & mask_to_clear; } else { if (!dirty) { smp_mb(); } } return dirty != 0; }
void bitmap_clear(unsigned long *map, unsigned int start, int len) { unsigned long *p = map + BIT_WORD(start); const unsigned int size = start + len; int bits_to_clear = BITS_PER_LONG - (start % BITS_PER_LONG); unsigned long mask_to_clear = BITMAP_FIRST_WORD_MASK(start); while (len - bits_to_clear >= 0) { *p &= ~mask_to_clear; len -= bits_to_clear; bits_to_clear = BITS_PER_LONG; mask_to_clear = ~0UL; p++; } if (len) { mask_to_clear &= BITMAP_LAST_WORD_MASK(size); *p &= ~mask_to_clear; } }
void bitmap_clear(unsigned long *map, long start, long nr) { unsigned long *p = map + BIT_WORD(start); const long size = start + nr; int bits_to_clear = BITS_PER_LONG - (start % BITS_PER_LONG); unsigned long mask_to_clear = BITMAP_FIRST_WORD_MASK(start); while (nr - bits_to_clear >= 0) { *p &= ~mask_to_clear; nr -= bits_to_clear; bits_to_clear = BITS_PER_LONG; mask_to_clear = ~0UL; p++; } if (nr) { mask_to_clear &= BITMAP_LAST_WORD_MASK(size); *p &= ~mask_to_clear; } }
/* * This is a common helper function for find_next_bit and * find_next_zero_bit. The difference is the "invert" argument, which * is XORed with each fetched word before searching it for one bits. */ static unsigned long _find_next_bit(const unsigned long *addr, unsigned long nbits, unsigned long start, unsigned long invert) { unsigned long tmp; if (!nbits || start >= nbits) return nbits; tmp = addr[start / BITS_PER_LONG] ^ invert; /* Handle 1st word. */ tmp &= BITMAP_FIRST_WORD_MASK(start); start = round_down(start, BITS_PER_LONG); while (!tmp) { start += BITS_PER_LONG; if (start >= nbits) return nbits; tmp = addr[start / BITS_PER_LONG] ^ invert; } return min(start + __ffs(tmp), nbits); }
void bitmap_set_atomic(unsigned long *map, long start, long nr) { unsigned long *p = map + BIT_WORD(start); const long size = start + nr; int bits_to_set = BITS_PER_LONG - (start % BITS_PER_LONG); unsigned long mask_to_set = BITMAP_FIRST_WORD_MASK(start); /* First word */ if (nr - bits_to_set > 0) { atomic_or(p, mask_to_set); nr -= bits_to_set; bits_to_set = BITS_PER_LONG; mask_to_set = ~0UL; p++; } /* Full words */ if (bits_to_set == BITS_PER_LONG) { while (nr >= BITS_PER_LONG) { *p = ~0UL; nr -= BITS_PER_LONG; p++; } } /* Last word */ if (nr) { mask_to_set &= BITMAP_LAST_WORD_MASK(size); atomic_or(p, mask_to_set); } else { /* If we avoided the full barrier in atomic_or(), issue a * barrier to account for the assignments in the while loop. */ smp_mb(); } }