void delete_prio_tracer(pid_t tid)
{
struct prio_tracer *pt;
struct dentry *d;
unsigned long flags;
spin_lock_irqsave(&pts_lock, flags);
pt = query_prio_tracer(tid);
if (!pt) {
spin_unlock_irqrestore(&pts_lock, flags);
return;
}
d = pt->debugfs_entry;
spin_unlock_irqrestore(&pts_lock, flags);
/* debugfs involves mutex... */
debugfs_remove(d);
spin_lock_irqsave(&pts_lock, flags);
rb_erase(&pt->rb_node, &priority_tracers);
kfree(pt);
spin_unlock_irqrestore(&pts_lock, flags);
}
void ion_client_destroy(struct ion_client *client)
{
struct ion_device *dev = client->dev;
struct rb_node *n;
pr_info("%s: destroy ion_client %p (%s)\n", __func__, client, client->name);
while ((n = rb_first(&client->handles))) {
struct ion_handle *handle = rb_entry(n, struct ion_handle,
node);
mutex_lock(&client->lock);
ion_handle_destroy(&handle->ref);
mutex_unlock(&client->lock);
}
mutex_lock(&dev->lock);
if (client->task)
put_task_struct(client->task);
rb_erase(&client->node, &dev->clients);
debugfs_remove_recursive(client->debug_root);
mutex_unlock(&dev->lock);
kfree(client->name);
kfree(client);
}
/*
* Remove all entries past new_clusters, inclusive of an entry that
* contains new_clusters. This is effectively a cache forget.
*
* If you want to also clip the last extent by some number of clusters,
* you need to call ocfs2_extent_map_trunc().
* This code does not check or modify ip_clusters.
*/
int ocfs2_extent_map_drop(struct inode *inode, u32 new_clusters)
{
struct rb_node *free_head = NULL;
struct ocfs2_extent_map *em = &OCFS2_I(inode)->ip_map;
struct ocfs2_extent_map_entry *ent;
spin_lock(&OCFS2_I(inode)->ip_lock);
__ocfs2_extent_map_drop(inode, new_clusters, &free_head, &ent);
if (ent) {
rb_erase(&ent->e_node, &em->em_extents);
ent->e_node.rb_right = free_head;
free_head = &ent->e_node;
}
spin_unlock(&OCFS2_I(inode)->ip_lock);
if (free_head)
__ocfs2_extent_map_drop_cleanup(free_head);
return 0;
}
static int perf_session__add_hist_entry(struct perf_session *self,
struct addr_location *al, u64 count)
{
bool hit;
struct hist_entry *he;
if (sym_hist_filter != NULL &&
(al->sym == NULL || strcmp(sym_hist_filter, al->sym->name) != 0)) {
/* We're only interested in a symbol named sym_hist_filter */
if (al->sym != NULL) {
rb_erase(&al->sym->rb_node,
&al->map->dso->symbols[al->map->type]);
symbol__delete(al->sym);
}
return 0;
}
he = __perf_session__add_hist_entry(self, al, NULL, count, &hit);
if (he == NULL)
return -ENOMEM;
return annotate__hist_hit(he, al->addr);
}
/*
* actually break a callback
*/
static void afs_break_callback(struct afs_server *server,
struct afs_vnode *vnode)
{
_enter("");
set_bit(AFS_VNODE_CB_BROKEN, &vnode->flags);
if (vnode->cb_promised) {
spin_lock(&vnode->lock);
_debug("break callback");
spin_lock(&server->cb_lock);
if (vnode->cb_promised) {
rb_erase(&vnode->cb_promise, &server->cb_promises);
vnode->cb_promised = false;
}
spin_unlock(&server->cb_lock);
queue_work(afs_callback_update_worker, &vnode->cb_broken_work);
spin_unlock(&vnode->lock);
}
}
/* release group, return 1 if this was last release and group is destroyed
* timout work is canceled sync */
static int release_group(struct mcast_group *group, int from_timeout_handler)
{
struct mlx4_ib_demux_ctx *ctx = group->demux;
int nzgroup;
mutex_lock(&ctx->mcg_table_lock);
mutex_lock(&group->lock);
if (atomic_dec_and_test(&group->refcount)) {
if (!from_timeout_handler) {
if (group->state != MCAST_IDLE &&
!cancel_delayed_work(&group->timeout_work)) {
atomic_inc(&group->refcount);
mutex_unlock(&group->lock);
mutex_unlock(&ctx->mcg_table_lock);
return 0;
}
}
nzgroup = memcmp(&group->rec.mgid, &mgid0, sizeof mgid0);
if (nzgroup)
del_sysfs_port_mcg_attr(ctx->dev, ctx->port, &group->dentry.attr);
if (!list_empty(&group->pending_list))
mcg_warn_group(group, "releasing a group with non empty pending list\n");
if (nzgroup)
rb_erase(&group->node, &ctx->mcg_table);
list_del_init(&group->mgid0_list);
mutex_unlock(&group->lock);
mutex_unlock(&ctx->mcg_table_lock);
kfree(group);
return 1;
} else {
mutex_unlock(&group->lock);
mutex_unlock(&ctx->mcg_table_lock);
}
return 0;
}
void unregister_event(int fd)
{
int ret;
struct event_info *ei;
ei = lookup_event(fd);
if (!ei)
return;
ret = epoll_ctl(efd, EPOLL_CTL_DEL, fd, NULL);
if (ret)
sd_err("failed to delete epoll event for fd %d: %m", fd);
rb_erase(&ei->rb, &events_tree);
free(ei);
/*
* Although ei is no longer valid pointer, ei->handler() might be about
* to be called in do_event_loop(). Refreshing the event loop is safe.
*/
event_force_refresh();
tracepoint(event, unregister, fd);
}
/*
* record the callback for breaking
* - the caller must hold server->cb_lock
*/
static void afs_do_give_up_callback(struct afs_server *server,
struct afs_vnode *vnode)
{
struct afs_callback *cb;
_enter("%p,%p", server, vnode);
cb = &server->cb_break[server->cb_break_head];
cb->fid = vnode->fid;
cb->version = vnode->cb_version;
cb->expiry = vnode->cb_expiry;
cb->type = vnode->cb_type;
smp_wmb();
server->cb_break_head =
(server->cb_break_head + 1) &
(ARRAY_SIZE(server->cb_break) - 1);
/* defer the breaking of callbacks to try and collect as many as
* possible to ship in one operation */
switch (atomic_inc_return(&server->cb_break_n)) {
case 1 ... AFSCBMAX - 1:
queue_delayed_work(afs_callback_update_worker,
&server->cb_break_work, HZ * 2);
break;
case AFSCBMAX:
afs_flush_callback_breaks(server);
break;
default:
break;
}
ASSERT(server->cb_promises.rb_node != NULL);
rb_erase(&vnode->cb_promise, &server->cb_promises);
vnode->cb_promised = false;
_leave("");
}
static void id_map_ent_timeout(struct work_struct *work)
{
struct delayed_work *delay = to_delayed_work(work);
struct id_map_entry *ent = container_of(delay, struct id_map_entry, timeout);
struct id_map_entry *db_ent, *found_ent;
struct mlx4_ib_dev *dev = ent->dev;
struct mlx4_ib_sriov *sriov = &dev->sriov;
struct rb_root *sl_id_map = &sriov->sl_id_map;
int pv_id = (int) ent->pv_cm_id;
spin_lock(&sriov->id_map_lock);
db_ent = (struct id_map_entry *)idr_find(&sriov->pv_id_table, pv_id);
if (!db_ent)
goto out;
found_ent = id_map_find_by_sl_id(&dev->ib_dev, ent->slave_id, ent->sl_cm_id);
if (found_ent && found_ent == ent)
rb_erase(&found_ent->node, sl_id_map);
idr_remove(&sriov->pv_id_table, pv_id);
out:
list_del(&ent->list);
spin_unlock(&sriov->id_map_lock);
kfree(ent);
}
static void try_merge_map(struct extent_map_tree *tree, struct extent_map *em)
{
struct extent_map *merge = NULL;
struct rb_node *rb;
if (em->start != 0) {
rb = rb_prev(&em->rb_node);
if (rb)
merge = rb_entry(rb, struct extent_map, rb_node);
if (rb && mergable_maps(merge, em)) {
em->start = merge->start;
em->orig_start = merge->orig_start;
em->len += merge->len;
em->block_len += merge->block_len;
em->block_start = merge->block_start;
merge->in_tree = 0;
em->mod_len = (em->mod_len + em->mod_start) - merge->mod_start;
em->mod_start = merge->mod_start;
em->generation = max(em->generation, merge->generation);
rb_erase(&merge->rb_node, &tree->map);
free_extent_map(merge);
}
}
void uv_teardown_irq(unsigned int irq)
{
struct uv_irq_2_mmr_pnode *e;
struct rb_node *n;
unsigned long irqflags;
spin_lock_irqsave(&uv_irq_lock, irqflags);
n = uv_irq_root.rb_node;
while (n) {
e = rb_entry(n, struct uv_irq_2_mmr_pnode, list);
if (e->irq == irq) {
arch_disable_uv_irq(e->pnode, e->offset);
rb_erase(n, &uv_irq_root);
kfree(e);
break;
}
if (irq < e->irq)
n = n->rb_left;
else
n = n->rb_right;
}
spin_unlock_irqrestore(&uv_irq_lock, irqflags);
destroy_irq(irq);
}
/**
* add_extent_mapping - add new extent map to the extent tree
* @tree: tree to insert new map in
* @em: map to insert
*
* Insert @em into @tree or perform a simple forward/backward merge with
* existing mappings. The extent_map struct passed in will be inserted
* into the tree directly, with an additional reference taken, or a
* reference dropped if the merge attempt was sucessfull.
*/
int add_extent_mapping(struct extent_map_tree *tree,
struct extent_map *em)
{
int ret = 0;
struct extent_map *merge = NULL;
struct rb_node *rb;
struct extent_map *exist;
exist = lookup_extent_mapping(tree, em->start, em->len);
if (exist) {
free_extent_map(exist);
ret = -EEXIST;
goto out;
}
assert_spin_locked(&tree->lock);
rb = tree_insert(&tree->map, em->start, &em->rb_node);
if (rb) {
ret = -EEXIST;
free_extent_map(merge);
goto out;
}
atomic_inc(&em->refs);
if (em->start != 0) {
rb = rb_prev(&em->rb_node);
if (rb)
merge = rb_entry(rb, struct extent_map, rb_node);
if (rb && mergable_maps(merge, em)) {
em->start = merge->start;
em->len += merge->len;
em->block_len += merge->block_len;
em->block_start = merge->block_start;
merge->in_tree = 0;
rb_erase(&merge->rb_node, &tree->map);
free_extent_map(merge);
}
}
static int diff__process_sample_event(event_t *event, struct perf_session *session)
{
struct addr_location al;
struct sample_data data = { .period = 1, };
if (event__preprocess_sample(event, session, &al, &data, NULL) < 0) {
pr_warning("problem processing %d event, skipping it.\n",
event->header.type);
return -1;
}
if (al.filtered || al.sym == NULL)
return 0;
if (hists__add_entry(&session->hists, &al, data.period)) {
pr_warning("problem incrementing symbol period, skipping event\n");
return -1;
}
session->hists.stats.total_period += data.period;
return 0;
}
static struct perf_event_ops event_ops = {
.sample = diff__process_sample_event,
.mmap = event__process_mmap,
.comm = event__process_comm,
.exit = event__process_task,
.fork = event__process_task,
.lost = event__process_lost,
};
static void perf_session__insert_hist_entry_by_name(struct rb_root *root,
struct hist_entry *he)
{
struct rb_node **p = &root->rb_node;
struct rb_node *parent = NULL;
struct hist_entry *iter;
while (*p != NULL) {
parent = *p;
iter = rb_entry(parent, struct hist_entry, rb_node);
if (hist_entry__cmp(he, iter) < 0)
p = &(*p)->rb_left;
else
p = &(*p)->rb_right;
}
rb_link_node(&he->rb_node, parent, p);
rb_insert_color(&he->rb_node, root);
}
static void hists__resort_entries(struct hists *self)
{
unsigned long position = 1;
struct rb_root tmp = RB_ROOT;
struct rb_node *next = rb_first(&self->entries);
while (next != NULL) {
struct hist_entry *n = rb_entry(next, struct hist_entry, rb_node);
next = rb_next(&n->rb_node);
rb_erase(&n->rb_node, &self->entries);
n->position = position++;
perf_session__insert_hist_entry_by_name(&tmp, n);
}
self->entries = tmp;
}
static void rb_erase_init(struct rb_node *n, struct rb_root *root)
{
rb_erase(n, root);
RB_CLEAR_NODE(n);
}
static void remove_attr(struct rb_root *root, struct ib_sa_attr_list *attr_list)
{
rb_erase(&attr_list->node, root);
free_attr_list(attr_list);
kfree(attr_list);
}
static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_sha512 *vh = hdr_priv(hdr);
struct fio_sha512_ctx sha512_ctx = {
.buf = vh->sha512,
};
fio_sha512_init(&sha512_ctx);
fio_sha512_update(&sha512_ctx, p, len);
}
static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_sha256 *vh = hdr_priv(hdr);
struct fio_sha256_ctx sha256_ctx = {
.buf = vh->sha256,
};
fio_sha256_init(&sha256_ctx);
fio_sha256_update(&sha256_ctx, p, len);
}
static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_sha1 *vh = hdr_priv(hdr);
struct fio_sha1_ctx sha1_ctx = {
.H = vh->sha1,
};
fio_sha1_init(&sha1_ctx);
fio_sha1_update(&sha1_ctx, p, len);
}
static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc7 *vh = hdr_priv(hdr);
vh->crc7 = fio_crc7(p, len);
}
static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc16 *vh = hdr_priv(hdr);
vh->crc16 = fio_crc16(p, len);
}
static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc32 *vh = hdr_priv(hdr);
vh->crc32 = fio_crc32(p, len);
}
static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc32 *vh = hdr_priv(hdr);
vh->crc32 = fio_crc32c(p, len);
}
static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc64 *vh = hdr_priv(hdr);
vh->crc64 = fio_crc64(p, len);
}
static void fill_md5(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_md5 *vh = hdr_priv(hdr);
struct fio_md5_ctx md5_ctx = {
.hash = (uint32_t *) vh->md5_digest,
};
fio_md5_init(&md5_ctx);
fio_md5_update(&md5_ctx, p, len);
}
static void populate_hdr(struct thread_data *td, struct io_u *io_u,
struct verify_header *hdr, unsigned int header_num,
unsigned int header_len)
{
unsigned int data_len;
void *data, *p;
p = (void *) hdr;
hdr->magic = FIO_HDR_MAGIC;
hdr->verify_type = td->o.verify;
hdr->len = header_len;
hdr->rand_seed = io_u->rand_seed;
hdr->crc32 = fio_crc32c(p, offsetof(struct verify_header, crc32));
data_len = header_len - hdr_size(hdr);
data = p + hdr_size(hdr);
switch (td->o.verify) {
case VERIFY_MD5:
dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n",
io_u, hdr->len);
fill_md5(hdr, data, data_len);
break;
case VERIFY_CRC64:
dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n",
io_u, hdr->len);
fill_crc64(hdr, data, data_len);
break;
case VERIFY_CRC32C:
case VERIFY_CRC32C_INTEL:
dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n",
io_u, hdr->len);
fill_crc32c(hdr, data, data_len);
break;
case VERIFY_CRC32:
dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n",
io_u, hdr->len);
fill_crc32(hdr, data, data_len);
break;
case VERIFY_CRC16:
dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n",
io_u, hdr->len);
fill_crc16(hdr, data, data_len);
break;
case VERIFY_CRC7:
dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n",
io_u, hdr->len);
fill_crc7(hdr, data, data_len);
break;
case VERIFY_SHA256:
dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n",
io_u, hdr->len);
fill_sha256(hdr, data, data_len);
break;
case VERIFY_SHA512:
dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n",
io_u, hdr->len);
fill_sha512(hdr, data, data_len);
break;
case VERIFY_META:
dprint(FD_VERIFY, "fill meta io_u %p, len %u\n",
io_u, hdr->len);
fill_meta(hdr, td, io_u, header_num);
break;
case VERIFY_SHA1:
dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n",
io_u, hdr->len);
fill_sha1(hdr, data, data_len);
break;
case VERIFY_PATTERN:
/* nothing to do here */
break;
default:
log_err("fio: bad verify type: %d\n", td->o.verify);
assert(0);
}
if (td->o.verify_offset)
memswp(p, p + td->o.verify_offset, hdr_size(hdr));
}
/*
* fill body of io_u->buf with random data and add a header with the
* checksum of choice
*/
void populate_verify_io_u(struct thread_data *td, struct io_u *io_u)
{
if (td->o.verify == VERIFY_NULL)
return;
fill_pattern_headers(td, io_u, 0, 0);
}
int get_next_verify(struct thread_data *td, struct io_u *io_u)
{
struct io_piece *ipo = NULL;
/*
* this io_u is from a requeue, we already filled the offsets
*/
if (io_u->file)
return 0;
if (!RB_EMPTY_ROOT(&td->io_hist_tree)) {
struct rb_node *n = rb_first(&td->io_hist_tree);
ipo = rb_entry(n, struct io_piece, rb_node);
rb_erase(n, &td->io_hist_tree);
assert(ipo->flags & IP_F_ONRB);
ipo->flags &= ~IP_F_ONRB;
} else if (!flist_empty(&td->io_hist_list)) {
ipo = flist_entry(td->io_hist_list.next, struct io_piece, list);
flist_del(&ipo->list);
assert(ipo->flags & IP_F_ONLIST);
ipo->flags &= ~IP_F_ONLIST;
}
if (ipo) {
td->io_hist_len--;
io_u->offset = ipo->offset;
io_u->buflen = ipo->len;
io_u->file = ipo->file;
io_u->flags |= IO_U_F_VER_LIST;
if (ipo->flags & IP_F_TRIMMED)
io_u->flags |= IO_U_F_TRIMMED;
if (!fio_file_open(io_u->file)) {
int r = td_io_open_file(td, io_u->file);
if (r) {
dprint(FD_VERIFY, "failed file %s open\n",
io_u->file->file_name);
return 1;
}
}
get_file(ipo->file);
assert(fio_file_open(io_u->file));
io_u->ddir = DDIR_READ;
io_u->xfer_buf = io_u->buf;
io_u->xfer_buflen = io_u->buflen;
remove_trim_entry(td, ipo);
free(ipo);
dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u);
return 0;
}
dprint(FD_VERIFY, "get_next_verify: empty\n");
return 1;
}
void fio_verify_init(struct thread_data *td)
{
if (td->o.verify == VERIFY_CRC32C_INTEL ||
td->o.verify == VERIFY_CRC32C) {
crc32c_intel_probe();
}
}
static void *verify_async_thread(void *data)
{
struct thread_data *td = data;
struct io_u *io_u;
int ret = 0;
if (td->o.verify_cpumask_set &&
fio_setaffinity(td->pid, td->o.verify_cpumask)) {
log_err("fio: failed setting verify thread affinity\n");
goto done;
}
do {
FLIST_HEAD(list);
read_barrier();
if (td->verify_thread_exit)
break;
pthread_mutex_lock(&td->io_u_lock);
while (flist_empty(&td->verify_list) &&
!td->verify_thread_exit) {
ret = pthread_cond_wait(&td->verify_cond,
&td->io_u_lock);
if (ret) {
pthread_mutex_unlock(&td->io_u_lock);
break;
}
}
flist_splice_init(&td->verify_list, &list);
pthread_mutex_unlock(&td->io_u_lock);
if (flist_empty(&list))
continue;
while (!flist_empty(&list)) {
io_u = flist_entry(list.next, struct io_u, verify_list);
flist_del(&io_u->verify_list);
ret = verify_io_u(td, io_u);
put_io_u(td, io_u);
if (!ret)
continue;
if (td_non_fatal_error(td, ERROR_TYPE_VERIFY_BIT, ret)) {
update_error_count(td, ret);
td_clear_error(td);
ret = 0;
}
}
} while (!ret);
if (ret) {
td_verror(td, ret, "async_verify");
if (td->o.verify_fatal)
td->terminate = 1;
}
done:
pthread_mutex_lock(&td->io_u_lock);
td->nr_verify_threads--;
pthread_mutex_unlock(&td->io_u_lock);
pthread_cond_signal(&td->free_cond);
return NULL;
}
int verify_async_init(struct thread_data *td)
{
int i, ret;
pthread_attr_t attr;
pthread_attr_init(&attr);
pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN);
td->verify_thread_exit = 0;
td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async);
for (i = 0; i < td->o.verify_async; i++) {
ret = pthread_create(&td->verify_threads[i], &attr,
verify_async_thread, td);
if (ret) {
log_err("fio: async verify creation failed: %s\n",
strerror(ret));
break;
}
ret = pthread_detach(td->verify_threads[i]);
if (ret) {
log_err("fio: async verify thread detach failed: %s\n",
strerror(ret));
break;
}
td->nr_verify_threads++;
}
pthread_attr_destroy(&attr);
if (i != td->o.verify_async) {
log_err("fio: only %d verify threads started, exiting\n", i);
td->verify_thread_exit = 1;
write_barrier();
pthread_cond_broadcast(&td->verify_cond);
return 1;
}
return 0;
}
static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_sha512 *vh = hdr_priv(hdr);
struct fio_sha512_ctx sha512_ctx = {
.buf = vh->sha512,
};
fio_sha512_init(&sha512_ctx);
fio_sha512_update(&sha512_ctx, p, len);
}
static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_sha256 *vh = hdr_priv(hdr);
struct fio_sha256_ctx sha256_ctx = {
.buf = vh->sha256,
};
fio_sha256_init(&sha256_ctx);
fio_sha256_update(&sha256_ctx, p, len);
fio_sha256_final(&sha256_ctx);
}
static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_sha1 *vh = hdr_priv(hdr);
struct fio_sha1_ctx sha1_ctx = {
.H = vh->sha1,
};
fio_sha1_init(&sha1_ctx);
fio_sha1_update(&sha1_ctx, p, len);
fio_sha1_final(&sha1_ctx);
}
static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc7 *vh = hdr_priv(hdr);
vh->crc7 = fio_crc7(p, len);
}
static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc16 *vh = hdr_priv(hdr);
vh->crc16 = fio_crc16(p, len);
}
static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc32 *vh = hdr_priv(hdr);
vh->crc32 = fio_crc32(p, len);
}
static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc32 *vh = hdr_priv(hdr);
vh->crc32 = fio_crc32c(p, len);
}
static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc64 *vh = hdr_priv(hdr);
vh->crc64 = fio_crc64(p, len);
}
static void fill_md5(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_md5 *vh = hdr_priv(hdr);
struct fio_md5_ctx md5_ctx = {
.hash = (uint32_t *) vh->md5_digest,
};
fio_md5_init(&md5_ctx);
fio_md5_update(&md5_ctx, p, len);
fio_md5_final(&md5_ctx);
}
static void __fill_hdr(struct verify_header *hdr, int verify_type,
uint32_t len, uint64_t rand_seed)
{
void *p = hdr;
hdr->magic = FIO_HDR_MAGIC;
hdr->verify_type = verify_type;
hdr->len = len;
hdr->rand_seed = rand_seed;
hdr->crc32 = fio_crc32c(p, offsetof(struct verify_header, crc32));
}
static void fill_hdr(struct verify_header *hdr, int verify_type, uint32_t len,
uint64_t rand_seed)
{
if (verify_type != VERIFY_PATTERN_NO_HDR)
__fill_hdr(hdr, verify_type, len, rand_seed);
}
static void populate_hdr(struct thread_data *td, struct io_u *io_u,
struct verify_header *hdr, unsigned int header_num,
unsigned int header_len)
{
unsigned int data_len;
void *data, *p;
p = (void *) hdr;
fill_hdr(hdr, td->o.verify, header_len, io_u->rand_seed);
data_len = header_len - hdr_size(td, hdr);
data = p + hdr_size(td, hdr);
switch (td->o.verify) {
case VERIFY_MD5:
dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n",
io_u, hdr->len);
fill_md5(hdr, data, data_len);
break;
case VERIFY_CRC64:
dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n",
io_u, hdr->len);
fill_crc64(hdr, data, data_len);
break;
case VERIFY_CRC32C:
case VERIFY_CRC32C_INTEL:
dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n",
io_u, hdr->len);
fill_crc32c(hdr, data, data_len);
break;
case VERIFY_CRC32:
dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n",
io_u, hdr->len);
fill_crc32(hdr, data, data_len);
break;
case VERIFY_CRC16:
dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n",
io_u, hdr->len);
fill_crc16(hdr, data, data_len);
break;
case VERIFY_CRC7:
dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n",
io_u, hdr->len);
fill_crc7(hdr, data, data_len);
break;
case VERIFY_SHA256:
dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n",
io_u, hdr->len);
fill_sha256(hdr, data, data_len);
break;
case VERIFY_SHA512:
dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n",
io_u, hdr->len);
fill_sha512(hdr, data, data_len);
break;
case VERIFY_XXHASH:
dprint(FD_VERIFY, "fill xxhash io_u %p, len %u\n",
io_u, hdr->len);
fill_xxhash(hdr, data, data_len);
break;
case VERIFY_META:
dprint(FD_VERIFY, "fill meta io_u %p, len %u\n",
io_u, hdr->len);
fill_meta(hdr, td, io_u, header_num);
break;
case VERIFY_SHA1:
dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n",
io_u, hdr->len);
fill_sha1(hdr, data, data_len);
break;
case VERIFY_PATTERN:
case VERIFY_PATTERN_NO_HDR:
/* nothing to do here */
break;
default:
log_err("fio: bad verify type: %d\n", td->o.verify);
assert(0);
}
if (td->o.verify_offset && hdr_size(td, hdr))
memswp(p, p + td->o.verify_offset, hdr_size(td, hdr));
}
/*
* fill body of io_u->buf with random data and add a header with the
* checksum of choice
*/
void populate_verify_io_u(struct thread_data *td, struct io_u *io_u)
{
if (td->o.verify == VERIFY_NULL)
return;
io_u->numberio = td->io_issues[io_u->ddir];
fill_pattern_headers(td, io_u, 0, 0);
}
int get_next_verify(struct thread_data *td, struct io_u *io_u)
{
struct io_piece *ipo = NULL;
/*
* this io_u is from a requeue, we already filled the offsets
*/
if (io_u->file)
return 0;
if (!RB_EMPTY_ROOT(&td->io_hist_tree)) {
struct rb_node *n = rb_first(&td->io_hist_tree);
ipo = rb_entry(n, struct io_piece, rb_node);
/*
* Ensure that the associated IO has completed
*/
read_barrier();
if (ipo->flags & IP_F_IN_FLIGHT)
goto nothing;
rb_erase(n, &td->io_hist_tree);
assert(ipo->flags & IP_F_ONRB);
ipo->flags &= ~IP_F_ONRB;
} else if (!flist_empty(&td->io_hist_list)) {
ipo = flist_first_entry(&td->io_hist_list, struct io_piece, list);
/*
* Ensure that the associated IO has completed
*/
read_barrier();
if (ipo->flags & IP_F_IN_FLIGHT)
goto nothing;
flist_del(&ipo->list);
assert(ipo->flags & IP_F_ONLIST);
ipo->flags &= ~IP_F_ONLIST;
}
if (ipo) {
td->io_hist_len--;
io_u->offset = ipo->offset;
io_u->buflen = ipo->len;
io_u->numberio = ipo->numberio;
io_u->file = ipo->file;
io_u_set(io_u, IO_U_F_VER_LIST);
if (ipo->flags & IP_F_TRIMMED)
io_u_set(io_u, IO_U_F_TRIMMED);
if (!fio_file_open(io_u->file)) {
int r = td_io_open_file(td, io_u->file);
if (r) {
dprint(FD_VERIFY, "failed file %s open\n",
io_u->file->file_name);
return 1;
}
}
get_file(ipo->file);
assert(fio_file_open(io_u->file));
io_u->ddir = DDIR_READ;
io_u->xfer_buf = io_u->buf;
io_u->xfer_buflen = io_u->buflen;
remove_trim_entry(td, ipo);
free(ipo);
dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u);
if (!td->o.verify_pattern_bytes) {
io_u->rand_seed = __rand(&td->verify_state);
if (sizeof(int) != sizeof(long *))
io_u->rand_seed *= __rand(&td->verify_state);
}
return 0;
}
nothing:
dprint(FD_VERIFY, "get_next_verify: empty\n");
return 1;
}
/*
* Garbage collector for unused keys.
*
* This is done in process context so that we don't have to disable interrupts
* all over the place. key_put() schedules this rather than trying to do the
* cleanup itself, which means key_put() doesn't have to sleep.
*/
static void key_garbage_collector(struct work_struct *work)
{
static LIST_HEAD(graveyard);
static u8 gc_state; /* Internal persistent state */
#define KEY_GC_REAP_AGAIN 0x01 /* - Need another cycle */
#define KEY_GC_REAPING_LINKS 0x02 /* - We need to reap links */
#define KEY_GC_SET_TIMER 0x04 /* - We need to restart the timer */
#define KEY_GC_REAPING_DEAD_1 0x10 /* - We need to mark dead keys */
#define KEY_GC_REAPING_DEAD_2 0x20 /* - We need to reap dead key links */
#define KEY_GC_REAPING_DEAD_3 0x40 /* - We need to reap dead keys */
#define KEY_GC_FOUND_DEAD_KEY 0x80 /* - We found at least one dead key */
struct rb_node *cursor;
struct key *key;
time_t new_timer, limit;
kenter("[%lx,%x]", key_gc_flags, gc_state);
limit = current_kernel_time().tv_sec;
if (limit > key_gc_delay)
limit -= key_gc_delay;
else
limit = key_gc_delay;
/* Work out what we're going to be doing in this pass */
gc_state &= KEY_GC_REAPING_DEAD_1 | KEY_GC_REAPING_DEAD_2;
gc_state <<= 1;
if (test_and_clear_bit(KEY_GC_KEY_EXPIRED, &key_gc_flags))
gc_state |= KEY_GC_REAPING_LINKS | KEY_GC_SET_TIMER;
if (test_and_clear_bit(KEY_GC_REAP_KEYTYPE, &key_gc_flags))
gc_state |= KEY_GC_REAPING_DEAD_1;
kdebug("new pass %x", gc_state);
new_timer = LONG_MAX;
/* As only this function is permitted to remove things from the key
* serial tree, if cursor is non-NULL then it will always point to a
* valid node in the tree - even if lock got dropped.
*/
spin_lock(&key_serial_lock);
cursor = rb_first(&key_serial_tree);
continue_scanning:
while (cursor) {
key = rb_entry(cursor, struct key, serial_node);
cursor = rb_next(cursor);
if (atomic_read(&key->usage) == 0)
goto found_unreferenced_key;
if (unlikely(gc_state & KEY_GC_REAPING_DEAD_1)) {
if (key->type == key_gc_dead_keytype) {
gc_state |= KEY_GC_FOUND_DEAD_KEY;
set_bit(KEY_FLAG_DEAD, &key->flags);
key->perm = 0;
goto skip_dead_key;
}
}
if (gc_state & KEY_GC_SET_TIMER) {
if (key->expiry > limit && key->expiry < new_timer) {
kdebug("will expire %x in %ld",
key_serial(key), key->expiry - limit);
new_timer = key->expiry;
}
}
if (unlikely(gc_state & KEY_GC_REAPING_DEAD_2))
if (key->type == key_gc_dead_keytype)
gc_state |= KEY_GC_FOUND_DEAD_KEY;
if ((gc_state & KEY_GC_REAPING_LINKS) ||
unlikely(gc_state & KEY_GC_REAPING_DEAD_2)) {
if (key->type == &key_type_keyring)
goto found_keyring;
}
if (unlikely(gc_state & KEY_GC_REAPING_DEAD_3))
if (key->type == key_gc_dead_keytype)
goto destroy_dead_key;
skip_dead_key:
if (spin_is_contended(&key_serial_lock) || need_resched())
goto contended;
}
contended:
spin_unlock(&key_serial_lock);
maybe_resched:
if (cursor) {
cond_resched();
spin_lock(&key_serial_lock);
goto continue_scanning;
}
/* We've completed the pass. Set the timer if we need to and queue a
* new cycle if necessary. We keep executing cycles until we find one
* where we didn't reap any keys.
*/
kdebug("pass complete");
if (gc_state & KEY_GC_SET_TIMER && new_timer != (time_t)LONG_MAX) {
new_timer += key_gc_delay;
key_schedule_gc(new_timer);
}
if (unlikely(gc_state & KEY_GC_REAPING_DEAD_2) ||
!list_empty(&graveyard)) {
/* Make sure that all pending keyring payload destructions are
* fulfilled and that people aren't now looking at dead or
* dying keys that they don't have a reference upon or a link
* to.
*/
kdebug("gc sync");
synchronize_rcu();
}
if (!list_empty(&graveyard)) {
kdebug("gc keys");
key_gc_unused_keys(&graveyard);
}
if (unlikely(gc_state & (KEY_GC_REAPING_DEAD_1 |
KEY_GC_REAPING_DEAD_2))) {
if (!(gc_state & KEY_GC_FOUND_DEAD_KEY)) {
/* No remaining dead keys: short circuit the remaining
* keytype reap cycles.
*/
kdebug("dead short");
gc_state &= ~(KEY_GC_REAPING_DEAD_1 | KEY_GC_REAPING_DEAD_2);
gc_state |= KEY_GC_REAPING_DEAD_3;
} else {
gc_state |= KEY_GC_REAP_AGAIN;
}
}
if (unlikely(gc_state & KEY_GC_REAPING_DEAD_3)) {
kdebug("dead wake");
smp_mb();
clear_bit(KEY_GC_REAPING_KEYTYPE, &key_gc_flags);
wake_up_bit(&key_gc_flags, KEY_GC_REAPING_KEYTYPE);
}
if (gc_state & KEY_GC_REAP_AGAIN)
schedule_work(&key_gc_work);
kleave(" [end %x]", gc_state);
return;
/* We found an unreferenced key - once we've removed it from the tree,
* we can safely drop the lock.
*/
found_unreferenced_key:
kdebug("unrefd key %d", key->serial);
rb_erase(&key->serial_node, &key_serial_tree);
spin_unlock(&key_serial_lock);
list_add_tail(&key->graveyard_link, &graveyard);
gc_state |= KEY_GC_REAP_AGAIN;
goto maybe_resched;
/* We found a keyring and we need to check the payload for links to
* dead or expired keys. We don't flag another reap immediately as we
* have to wait for the old payload to be destroyed by RCU before we
* can reap the keys to which it refers.
*/
found_keyring:
spin_unlock(&key_serial_lock);
kdebug("scan keyring %d", key->serial);
key_gc_keyring(key, limit);
goto maybe_resched;
/* We found a dead key that is still referenced. Reset its type and
* destroy its payload with its semaphore held.
*/
destroy_dead_key:
spin_unlock(&key_serial_lock);
kdebug("destroy key %d", key->serial);
down_write(&key->sem);
key->type = &key_type_dead;
if (key_gc_dead_keytype->destroy)
key_gc_dead_keytype->destroy(key);
memset(&key->payload, KEY_DESTROY, sizeof(key->payload));
up_write(&key->sem);
goto maybe_resched;
}
static void memory_engine_delete_shm_node(struct rb_root *shm_root,
memory_node_t *shm_node)
{
rb_erase(&shm_node->__rb_node, shm_root);
}
/*
* update a bunch of callbacks
*/
static void afs_callback_updater(struct work_struct *work)
{
struct afs_server *server;
struct afs_vnode *vnode, *xvnode;
time_t now;
long timeout;
int ret;
server = container_of(work, struct afs_server, updater);
_enter("");
now = get_seconds();
/* find the first vnode to update */
spin_lock(&server->cb_lock);
for (;;) {
if (RB_EMPTY_ROOT(&server->cb_promises)) {
spin_unlock(&server->cb_lock);
_leave(" [nothing]");
return;
}
vnode = rb_entry(rb_first(&server->cb_promises),
struct afs_vnode, cb_promise);
if (atomic_read(&vnode->usage) > 0)
break;
rb_erase(&vnode->cb_promise, &server->cb_promises);
vnode->cb_promised = false;
}
timeout = vnode->update_at - now;
if (timeout > 0) {
queue_delayed_work(afs_vnode_update_worker,
&afs_vnode_update, timeout * HZ);
spin_unlock(&server->cb_lock);
_leave(" [nothing]");
return;
}
list_del_init(&vnode->update);
atomic_inc(&vnode->usage);
spin_unlock(&server->cb_lock);
/* we can now perform the update */
_debug("update %s", vnode->vldb.name);
vnode->state = AFS_VL_UPDATING;
vnode->upd_rej_cnt = 0;
vnode->upd_busy_cnt = 0;
ret = afs_vnode_update_record(vl, &vldb);
switch (ret) {
case 0:
afs_vnode_apply_update(vl, &vldb);
vnode->state = AFS_VL_UPDATING;
break;
case -ENOMEDIUM:
vnode->state = AFS_VL_VOLUME_DELETED;
break;
default:
vnode->state = AFS_VL_UNCERTAIN;
break;
}
/* and then reschedule */
_debug("reschedule");
vnode->update_at = get_seconds() + afs_vnode_update_timeout;
spin_lock(&server->cb_lock);
if (!list_empty(&server->cb_promises)) {
/* next update in 10 minutes, but wait at least 1 second more
* than the newest record already queued so that we don't spam
* the VL server suddenly with lots of requests
*/
xvnode = list_entry(server->cb_promises.prev,
struct afs_vnode, update);
if (vnode->update_at <= xvnode->update_at)
vnode->update_at = xvnode->update_at + 1;
xvnode = list_entry(server->cb_promises.next,
struct afs_vnode, update);
timeout = xvnode->update_at - now;
if (timeout < 0)
timeout = 0;
} else {
static int diff__process_sample_event(event_t *event, struct perf_session *session)
{
struct addr_location al;
struct sample_data data = { .period = 1, };
dump_printf("(IP, %d): %d: %p\n", event->header.misc,
event->ip.pid, (void *)(long)event->ip.ip);
if (event__preprocess_sample(event, session, &al, NULL) < 0) {
pr_warning("problem processing %d event, skipping it.\n",
event->header.type);
return -1;
}
if (al.filtered)
return 0;
event__parse_sample(event, session->sample_type, &data);
if (al.sym && perf_session__add_hist_entry(session, &al, data.period)) {
pr_warning("problem incrementing symbol count, skipping event\n");
return -1;
}
session->events_stats.total += data.period;
return 0;
}
static struct perf_event_ops event_ops = {
.process_sample_event = diff__process_sample_event,
.process_mmap_event = event__process_mmap,
.process_comm_event = event__process_comm,
.process_exit_event = event__process_task,
.process_fork_event = event__process_task,
.process_lost_event = event__process_lost,
};
static void perf_session__insert_hist_entry_by_name(struct rb_root *root,
struct hist_entry *he)
{
struct rb_node **p = &root->rb_node;
struct rb_node *parent = NULL;
struct hist_entry *iter;
while (*p != NULL) {
int cmp;
parent = *p;
iter = rb_entry(parent, struct hist_entry, rb_node);
cmp = strcmp(he->map->dso->name, iter->map->dso->name);
if (cmp > 0)
p = &(*p)->rb_left;
else if (cmp < 0)
p = &(*p)->rb_right;
else {
cmp = strcmp(he->sym->name, iter->sym->name);
if (cmp > 0)
p = &(*p)->rb_left;
else
p = &(*p)->rb_right;
}
}
rb_link_node(&he->rb_node, parent, p);
rb_insert_color(&he->rb_node, root);
}
static void perf_session__resort_by_name(struct perf_session *self)
{
unsigned long position = 1;
struct rb_root tmp = RB_ROOT;
struct rb_node *next = rb_first(&self->hists);
while (next != NULL) {
struct hist_entry *n = rb_entry(next, struct hist_entry, rb_node);
next = rb_next(&n->rb_node);
rb_erase(&n->rb_node, &self->hists);
n->position = position++;
perf_session__insert_hist_entry_by_name(&tmp, n);
}
self->hists = tmp;
}
void CPageMgr::RbRemove(rb_root * root, void * object,
RbGetNodeFunc getNode)
{
rb_erase((this->*getNode)(object), root);
}
static void __pohmelfs_name_del(struct pohmelfs_inode *parent, struct pohmelfs_name *node)
{
rb_erase(&node->hash_node, &parent->hash_root);
}
void *async_loop(void *arg1) {
struct uwsgi_async_fd *tmp_uaf;
int interesting_fd, i;
struct uwsgi_rb_timer *min_timeout;
int timeout;
int is_a_new_connection;
int proto_parser_status;
time_t now, last_now = 0;
static struct uwsgi_async_request *current_request = NULL, *next_async_request = NULL;
void *events = event_queue_alloc(64);
struct uwsgi_socket *uwsgi_sock;
uwsgi.async_runqueue = NULL;
uwsgi.async_runqueue_cnt = 0;
if (uwsgi.signal_socket > -1) {
event_queue_add_fd_read(uwsgi.async_queue, uwsgi.signal_socket);
event_queue_add_fd_read(uwsgi.async_queue, uwsgi.my_signal_socket);
}
// set a default request manager
if (!uwsgi.schedule_to_req) uwsgi.schedule_to_req = async_schedule_to_req;
while (uwsgi.workers[uwsgi.mywid].manage_next_request) {
if (uwsgi.async_runqueue_cnt) {
timeout = 0;
}
else {
min_timeout = uwsgi_min_rb_timer(uwsgi.rb_async_timeouts);
if (uwsgi.async_runqueue_cnt) {
timeout = 0;
}
if (min_timeout) {
timeout = min_timeout->key - time(NULL);
if (timeout <= 0) {
async_expire_timeouts();
timeout = 0;
}
}
else {
timeout = -1;
}
}
uwsgi.async_nevents = event_queue_wait_multi(uwsgi.async_queue, timeout, events, 64);
// timeout ???
if (uwsgi.async_nevents == 0) {
async_expire_timeouts();
}
for(i=0;i<uwsgi.async_nevents;i++) {
// manage events
interesting_fd = event_queue_interesting_fd(events, i);
if (uwsgi.signal_socket > -1 && (interesting_fd == uwsgi.signal_socket || interesting_fd == uwsgi.my_signal_socket)) {
uwsgi_receive_signal(interesting_fd, "worker", uwsgi.mywid);
continue;
}
is_a_new_connection = 0;
// new request coming in ?
uwsgi_sock = uwsgi.sockets;
while(uwsgi_sock) {
if (interesting_fd == uwsgi_sock->fd) {
is_a_new_connection = 1;
uwsgi.wsgi_req = find_first_available_wsgi_req();
if (uwsgi.wsgi_req == NULL) {
now = time(NULL);
if (now > last_now) {
uwsgi_log("async queue is full !!!\n");
last_now = now;
}
break;
}
wsgi_req_setup(uwsgi.wsgi_req, uwsgi.wsgi_req->async_id, uwsgi_sock );
if (wsgi_req_simple_accept(uwsgi.wsgi_req, interesting_fd)) {
#ifdef UWSGI_EVENT_USE_PORT
event_queue_add_fd_read(uwsgi.async_queue, interesting_fd);
#endif
uwsgi.async_queue_unused_ptr++;
uwsgi.async_queue_unused[uwsgi.async_queue_unused_ptr] = uwsgi.wsgi_req;
break;
}
#ifdef UWSGI_EVENT_USE_PORT
event_queue_add_fd_read(uwsgi.async_queue, interesting_fd);
#endif
// on linux we do not need to reset the socket to blocking state
#ifndef __linux__
/* re-set blocking socket */
int arg = uwsgi_sock->arg;
arg &= (~O_NONBLOCK);
if (fcntl(uwsgi.wsgi_req->poll.fd, F_SETFL, arg) < 0) {
uwsgi_error("fcntl()");
uwsgi.async_queue_unused_ptr++;
uwsgi.async_queue_unused[uwsgi.async_queue_unused_ptr] = uwsgi.wsgi_req;
break;
}
#endif
if (wsgi_req_async_recv(uwsgi.wsgi_req)) {
uwsgi.async_queue_unused_ptr++;
uwsgi.async_queue_unused[uwsgi.async_queue_unused_ptr] = uwsgi.wsgi_req;
break;
}
if (uwsgi.wsgi_req->do_not_add_to_async_queue) {
runqueue_push(uwsgi.wsgi_req);
}
break;
}
uwsgi_sock = uwsgi_sock->next;
}
if (!is_a_new_connection) {
// proto event
uwsgi.wsgi_req = find_wsgi_req_proto_by_fd(interesting_fd);
if (uwsgi.wsgi_req) {
proto_parser_status = uwsgi.wsgi_req->socket->proto(uwsgi.wsgi_req);
// reset timeout
rb_erase(&uwsgi.wsgi_req->async_timeout->rbt, uwsgi.rb_async_timeouts);
free(uwsgi.wsgi_req->async_timeout);
uwsgi.wsgi_req->async_timeout = NULL;
// parsing complete
if (!proto_parser_status) {
// remove fd from event poll and fd proto table
#ifndef UWSGI_EVENT_USE_PORT
event_queue_del_fd(uwsgi.async_queue, interesting_fd, event_queue_read());
#endif
uwsgi.async_proto_fd_table[interesting_fd] = NULL;
// put request in the runqueue
runqueue_push(uwsgi.wsgi_req);
continue;
}
else if (proto_parser_status < 0) {
if (proto_parser_status == -1)
uwsgi_log("error parsing request\n");
uwsgi.async_proto_fd_table[interesting_fd] = NULL;
close(interesting_fd);
continue;
}
// re-add timer
async_add_timeout(uwsgi.wsgi_req, uwsgi.shared->options[UWSGI_OPTION_SOCKET_TIMEOUT]);
continue;
}
// app event
uwsgi.wsgi_req = find_wsgi_req_by_fd(interesting_fd);
// unknown fd, remove it (for safety)
if (uwsgi.wsgi_req == NULL) {
close(interesting_fd);
continue;
}
// remove all the fd monitors and timeout
while(uwsgi.wsgi_req->waiting_fds) {
#ifndef UWSGI_EVENT_USE_PORT
event_queue_del_fd(uwsgi.async_queue, uwsgi.wsgi_req->waiting_fds->fd, uwsgi.wsgi_req->waiting_fds->event);
#endif
tmp_uaf = uwsgi.wsgi_req->waiting_fds;
uwsgi.async_waiting_fd_table[tmp_uaf->fd] = NULL;
uwsgi.wsgi_req->waiting_fds = tmp_uaf->next;
free(tmp_uaf);
}
uwsgi.wsgi_req->waiting_fds = NULL;
if (uwsgi.wsgi_req->async_timeout) {
rb_erase(&uwsgi.wsgi_req->async_timeout->rbt, uwsgi.rb_async_timeouts);
free(uwsgi.wsgi_req->async_timeout);
uwsgi.wsgi_req->async_timeout = NULL;
}
uwsgi.wsgi_req->async_ready_fd = 1;
uwsgi.wsgi_req->async_last_ready_fd = interesting_fd;
// put the request in the runqueue again
runqueue_push(uwsgi.wsgi_req);
}
}
// event queue managed, give cpu to runqueue
if (!current_request)
current_request = uwsgi.async_runqueue;
if (uwsgi.async_runqueue_cnt) {
uwsgi.wsgi_req = current_request->wsgi_req;
uwsgi.schedule_to_req();
uwsgi.wsgi_req->switches++;
next_async_request = current_request->next;
// request ended ?
if (uwsgi.wsgi_req->async_status <= UWSGI_OK) {
// remove all the monitored fds and timeout
while(uwsgi.wsgi_req->waiting_fds) {
#ifndef UWSGI_EVENT_USE_PORT
event_queue_del_fd(uwsgi.async_queue, uwsgi.wsgi_req->waiting_fds->fd, uwsgi.wsgi_req->waiting_fds->event);
#endif
tmp_uaf = uwsgi.wsgi_req->waiting_fds;
uwsgi.async_waiting_fd_table[tmp_uaf->fd] = NULL;
uwsgi.wsgi_req->waiting_fds = tmp_uaf->next;
free(tmp_uaf);
}
uwsgi.wsgi_req->waiting_fds = NULL;
if (uwsgi.wsgi_req->async_timeout) {
rb_erase(&uwsgi.wsgi_req->async_timeout->rbt, uwsgi.rb_async_timeouts);
free(uwsgi.wsgi_req->async_timeout);
uwsgi.wsgi_req->async_timeout = NULL;
}
// remove from the list
runqueue_remove(current_request);
uwsgi_close_request(uwsgi.wsgi_req);
// push wsgi_request in the unused stack
uwsgi.async_queue_unused_ptr++;
uwsgi.async_queue_unused[uwsgi.async_queue_unused_ptr] = uwsgi.wsgi_req;
}
else if (uwsgi.wsgi_req->waiting_fds || uwsgi.wsgi_req->async_timeout) {
// remove this request from suspended list
runqueue_remove(current_request);
}
current_request = next_async_request;
}
}
return NULL;
}
static inline void delete_dtask_from_tree(struct dead_task_struct* dtask)
{
rb_erase(&dtask->rb_node, &dtask_root);
RB_CLEAR_NODE(&dtask->rb_node);
}
static int nova_failure_insert_inodetree(struct super_block *sb,
unsigned long ino_low, unsigned long ino_high)
{
struct nova_sb_info *sbi = NOVA_SB(sb);
struct inode_map *inode_map;
struct nova_range_node *prev = NULL, *next = NULL;
struct nova_range_node *new_node;
unsigned long internal_low, internal_high;
int cpu;
struct rb_root *tree;
int ret;
if (ino_low > ino_high) {
nova_err(sb, "%s: ino low %lu, ino high %lu\n",
__func__, ino_low, ino_high);
BUG();
}
cpu = ino_low % sbi->cpus;
if (ino_high % sbi->cpus != cpu) {
nova_err(sb, "%s: ino low %lu, ino high %lu\n",
__func__, ino_low, ino_high);
BUG();
}
internal_low = ino_low / sbi->cpus;
internal_high = ino_high / sbi->cpus;
inode_map = &sbi->inode_maps[cpu];
tree = &inode_map->inode_inuse_tree;
mutex_lock(&inode_map->inode_table_mutex);
ret = nova_find_free_slot(sbi, tree, internal_low, internal_high,
&prev, &next);
if (ret) {
nova_dbg("%s: ino %lu - %lu already exists!: %d\n",
__func__, ino_low, ino_high, ret);
mutex_unlock(&inode_map->inode_table_mutex);
return ret;
}
if (prev && next && (internal_low == prev->range_high + 1) &&
(internal_high + 1 == next->range_low)) {
/* fits the hole */
rb_erase(&next->node, tree);
inode_map->num_range_node_inode--;
prev->range_high = next->range_high;
nova_free_inode_node(sb, next);
goto finish;
}
if (prev && (internal_low == prev->range_high + 1)) {
/* Aligns left */
prev->range_high += internal_high - internal_low + 1;
goto finish;
}
if (next && (internal_high + 1 == next->range_low)) {
/* Aligns right */
next->range_low -= internal_high - internal_low + 1;
goto finish;
}
/* Aligns somewhere in the middle */
new_node = nova_alloc_inode_node(sb);
NOVA_ASSERT(new_node);
new_node->range_low = internal_low;
new_node->range_high = internal_high;
ret = nova_insert_inodetree(sbi, new_node, cpu);
if (ret) {
nova_err(sb, "%s failed\n", __func__);
nova_free_inode_node(sb, new_node);
goto finish;
}
inode_map->num_range_node_inode++;
finish:
mutex_unlock(&inode_map->inode_table_mutex);
return ret;
}
/*
* Manage a cell record, initialising and destroying it, maintaining its DNS
* records.
*/
static void afs_manage_cell(struct work_struct *work)
{
struct afs_cell *cell = container_of(work, struct afs_cell, manager);
struct afs_net *net = cell->net;
bool deleted;
int ret, usage;
_enter("%s", cell->name);
again:
_debug("state %u", cell->state);
switch (cell->state) {
case AFS_CELL_INACTIVE:
case AFS_CELL_FAILED:
write_seqlock(&net->cells_lock);
usage = 1;
deleted = atomic_try_cmpxchg_relaxed(&cell->usage, &usage, 0);
if (deleted)
rb_erase(&cell->net_node, &net->cells);
write_sequnlock(&net->cells_lock);
if (deleted)
goto final_destruction;
if (cell->state == AFS_CELL_FAILED)
goto done;
cell->state = AFS_CELL_UNSET;
goto again;
case AFS_CELL_UNSET:
cell->state = AFS_CELL_ACTIVATING;
goto again;
case AFS_CELL_ACTIVATING:
ret = afs_activate_cell(net, cell);
if (ret < 0)
goto activation_failed;
cell->state = AFS_CELL_ACTIVE;
smp_wmb();
clear_bit(AFS_CELL_FL_NOT_READY, &cell->flags);
wake_up_bit(&cell->flags, AFS_CELL_FL_NOT_READY);
goto again;
case AFS_CELL_ACTIVE:
if (atomic_read(&cell->usage) > 1) {
time64_t now = ktime_get_real_seconds();
if (cell->dns_expiry <= now && net->live)
afs_update_cell(cell);
goto done;
}
cell->state = AFS_CELL_DEACTIVATING;
goto again;
case AFS_CELL_DEACTIVATING:
set_bit(AFS_CELL_FL_NOT_READY, &cell->flags);
if (atomic_read(&cell->usage) > 1)
goto reverse_deactivation;
afs_deactivate_cell(net, cell);
cell->state = AFS_CELL_INACTIVE;
goto again;
default:
break;
}
_debug("bad state %u", cell->state);
BUG(); /* Unhandled state */
activation_failed:
cell->error = ret;
afs_deactivate_cell(net, cell);
cell->state = AFS_CELL_FAILED;
smp_wmb();
if (test_and_clear_bit(AFS_CELL_FL_NOT_READY, &cell->flags))
wake_up_bit(&cell->flags, AFS_CELL_FL_NOT_READY);
goto again;
reverse_deactivation:
cell->state = AFS_CELL_ACTIVE;
smp_wmb();
clear_bit(AFS_CELL_FL_NOT_READY, &cell->flags);
wake_up_bit(&cell->flags, AFS_CELL_FL_NOT_READY);
_leave(" [deact->act]");
return;
done:
_leave(" [done %u]", cell->state);
return;
final_destruction:
call_rcu(&cell->rcu, afs_cell_destroy);
afs_dec_cells_outstanding(net);
_leave(" [destruct %d]", atomic_read(&net->cells_outstanding));
}
/*
* Simple rule: on any return code other than -EAGAIN, anything left
* in the insert_context will be freed.
*
* Simple rule #2: A return code of -EEXIST from this function or
* its calls to ocfs2_extent_map_insert_entry() signifies that another
* thread beat us to the insert. It is not an actual error, but it
* tells the caller we have no more work to do.
*/
static int ocfs2_extent_map_try_insert(struct inode *inode,
struct ocfs2_extent_rec *rec,
int tree_depth,
struct ocfs2_em_insert_context *ctxt)
{
int ret;
struct ocfs2_extent_map *em = &OCFS2_I(inode)->ip_map;
struct ocfs2_extent_map_entry *old_ent;
ctxt->need_left = 0;
ctxt->need_right = 0;
ctxt->old_ent = NULL;
spin_lock(&OCFS2_I(inode)->ip_lock);
ret = ocfs2_extent_map_insert_entry(em, ctxt->new_ent);
if (!ret) {
ctxt->new_ent = NULL;
goto out_unlock;
}
/* Since insert_entry failed, the map MUST have old_ent */
old_ent = ocfs2_extent_map_lookup(em, le32_to_cpu(rec->e_cpos),
le32_to_cpu(rec->e_clusters),
NULL, NULL);
BUG_ON(!old_ent);
if (old_ent->e_tree_depth < tree_depth) {
/* Another thread beat us to the lower tree_depth */
ret = -EEXIST;
goto out_unlock;
}
if (old_ent->e_tree_depth == tree_depth) {
/*
* Another thread beat us to this tree_depth.
* Let's make sure we agree with that thread (the
* extent_rec should be identical).
*/
if (!memcmp(rec, &old_ent->e_rec,
sizeof(struct ocfs2_extent_rec)))
ret = 0;
else
/* FIXME: Should this be ESRCH/EBADR??? */
ret = -EEXIST;
goto out_unlock;
}
/*
* We do it in this order specifically so that no actual tree
* changes occur until we have all the pieces we need. We
* don't want malloc failures to leave an inconsistent tree.
* Whenever we drop the lock, another process could be
* inserting. Also note that, if another process just beat us
* to an insert, we might not need the same pieces we needed
* the first go round. In the end, the pieces we need will
* be used, and the pieces we don't will be freed.
*/
ctxt->need_left = !!(le32_to_cpu(rec->e_cpos) >
le32_to_cpu(old_ent->e_rec.e_cpos));
ctxt->need_right = !!((le32_to_cpu(old_ent->e_rec.e_cpos) +
le32_to_cpu(old_ent->e_rec.e_clusters)) >
(le32_to_cpu(rec->e_cpos) + le32_to_cpu(rec->e_clusters)));
ret = -EAGAIN;
if (ctxt->need_left) {
if (!ctxt->left_ent)
goto out_unlock;
*(ctxt->left_ent) = *old_ent;
ctxt->left_ent->e_rec.e_clusters =
cpu_to_le32(le32_to_cpu(rec->e_cpos) -
le32_to_cpu(ctxt->left_ent->e_rec.e_cpos));
}
if (ctxt->need_right) {
if (!ctxt->right_ent)
goto out_unlock;
*(ctxt->right_ent) = *old_ent;
ctxt->right_ent->e_rec.e_cpos =
cpu_to_le32(le32_to_cpu(rec->e_cpos) +
le32_to_cpu(rec->e_clusters));
ctxt->right_ent->e_rec.e_clusters =
cpu_to_le32((le32_to_cpu(old_ent->e_rec.e_cpos) +
le32_to_cpu(old_ent->e_rec.e_clusters)) -
le32_to_cpu(ctxt->right_ent->e_rec.e_cpos));
}
rb_erase(&old_ent->e_node, &em->em_extents);
/* Now that he's erased, set him up for deletion */
ctxt->old_ent = old_ent;
if (ctxt->need_left) {
ret = ocfs2_extent_map_insert_entry(em,
ctxt->left_ent);
if (ret)
goto out_unlock;
ctxt->left_ent = NULL;
}
if (ctxt->need_right) {
ret = ocfs2_extent_map_insert_entry(em,
ctxt->right_ent);
if (ret)
goto out_unlock;
ctxt->right_ent = NULL;
}
ret = ocfs2_extent_map_insert_entry(em, ctxt->new_ent);
if (!ret)
ctxt->new_ent = NULL;
out_unlock:
spin_unlock(&OCFS2_I(inode)->ip_lock);
return ret;
}
/*
* log a successful write, so we can unwind the log for verify
*/
void log_io_piece(struct thread_data *td, struct io_u *io_u)
{
struct rb_node **p, *parent;
struct io_piece *ipo, *__ipo;
ipo = malloc(sizeof(struct io_piece));
init_ipo(ipo);
ipo->file = io_u->file;
ipo->offset = io_u->offset;
ipo->len = io_u->buflen;
ipo->numberio = io_u->numberio;
ipo->flags = IP_F_IN_FLIGHT;
io_u->ipo = ipo;
if (io_u_should_trim(td, io_u)) {
flist_add_tail(&ipo->trim_list, &td->trim_list);
td->trim_entries++;
}
/*
* We don't need to sort the entries, if:
*
* Sequential writes, or
* Random writes that lay out the file as it goes along
*
* For both these cases, just reading back data in the order we
* wrote it out is the fastest.
*
* One exception is if we don't have a random map AND we are doing
* verifies, in that case we need to check for duplicate blocks and
* drop the old one, which we rely on the rb insert/lookup for
* handling.
*/
if (((!td->o.verifysort) || !td_random(td) || !td->o.overwrite) &&
(file_randommap(td, ipo->file) || td->o.verify == VERIFY_NONE)) {
INIT_FLIST_HEAD(&ipo->list);
flist_add_tail(&ipo->list, &td->io_hist_list);
ipo->flags |= IP_F_ONLIST;
td->io_hist_len++;
return;
}
RB_CLEAR_NODE(&ipo->rb_node);
/*
* Sort the entry into the verification list
*/
restart:
p = &td->io_hist_tree.rb_node;
parent = NULL;
while (*p) {
parent = *p;
__ipo = rb_entry(parent, struct io_piece, rb_node);
if (ipo->file < __ipo->file)
p = &(*p)->rb_left;
else if (ipo->file > __ipo->file)
p = &(*p)->rb_right;
else if (ipo->offset < __ipo->offset)
p = &(*p)->rb_left;
else if (ipo->offset > __ipo->offset)
p = &(*p)->rb_right;
else {
dprint(FD_IO, "iolog: overlap %llu/%lu, %llu/%lu",
__ipo->offset, __ipo->len,
ipo->offset, ipo->len);
td->io_hist_len--;
rb_erase(parent, &td->io_hist_tree);
remove_trim_entry(td, __ipo);
free(__ipo);
goto restart;
}
}
rb_link_node(&ipo->rb_node, parent, p);
rb_insert_color(&ipo->rb_node, &td->io_hist_tree);
ipo->flags |= IP_F_ONRB;
td->io_hist_len++;
}
void rblist__remove_node(struct rblist *rblist, struct rb_node *rb_node)
{
rb_erase(rb_node, &rblist->entries);
--rblist->nr_entries;
rblist->node_delete(rblist, rb_node);
}
