int blk_register_queue(struct gendisk *disk) { int ret; struct device *dev = disk_to_dev(disk); struct request_queue *q = disk->queue; if (WARN_ON(!q)) return -ENXIO; ret = blk_trace_init_sysfs(dev); if (ret) return ret; ret = kobject_add(&q->kobj, kobject_get(&dev->kobj), "%s", "queue"); if (ret < 0) return ret; kobject_uevent(&q->kobj, KOBJ_ADD); if (!q->request_fn) return 0; ret = elv_register_queue(q); if (ret) { kobject_uevent(&q->kobj, KOBJ_REMOVE); kobject_del(&q->kobj); blk_trace_remove_sysfs(disk_to_dev(disk)); return ret; } return 0; }
/* always call with the tx_lock held */ static int nbd_send_req(struct nbd_device *nbd, struct request *req) { int result, flags; struct nbd_request request; unsigned long size = blk_rq_bytes(req); u32 type; if (req->cmd_type == REQ_TYPE_DRV_PRIV) type = NBD_CMD_DISC; else if (req->cmd_flags & REQ_DISCARD) type = NBD_CMD_TRIM; else if (req->cmd_flags & REQ_FLUSH) type = NBD_CMD_FLUSH; else if (rq_data_dir(req) == WRITE) type = NBD_CMD_WRITE; else type = NBD_CMD_READ; memset(&request, 0, sizeof(request)); request.magic = htonl(NBD_REQUEST_MAGIC); request.type = htonl(type); if (type != NBD_CMD_FLUSH && type != NBD_CMD_DISC) { request.from = cpu_to_be64((u64)blk_rq_pos(req) << 9); request.len = htonl(size); } memcpy(request.handle, &req, sizeof(req)); dev_dbg(nbd_to_dev(nbd), "request %p: sending control (%s@%llu,%uB)\n", req, nbdcmd_to_ascii(type), (unsigned long long)blk_rq_pos(req) << 9, blk_rq_bytes(req)); result = sock_xmit(nbd, 1, &request, sizeof(request), (type == NBD_CMD_WRITE) ? MSG_MORE : 0); if (result <= 0) { dev_err(disk_to_dev(nbd->disk), "Send control failed (result %d)\n", result); return -EIO; } if (type == NBD_CMD_WRITE) { struct req_iterator iter; struct bio_vec bvec; /* * we are really probing at internals to determine * whether to set MSG_MORE or not... */ rq_for_each_segment(bvec, req, iter) { flags = 0; if (!rq_iter_last(bvec, iter)) flags = MSG_MORE; dev_dbg(nbd_to_dev(nbd), "request %p: sending %d bytes data\n", req, bvec.bv_len); result = sock_send_bvec(nbd, &bvec, flags); if (result <= 0) { dev_err(disk_to_dev(nbd->disk), "Send data failed (result %d)\n", result); return -EIO; } } }
/* always call with the tx_lock held */ static int nbd_send_req(struct nbd_device *nbd, struct request *req) { int result, flags; struct nbd_request request; unsigned long size = blk_rq_bytes(req); request.magic = htonl(NBD_REQUEST_MAGIC); request.type = htonl(nbd_cmd(req)); if (nbd_cmd(req) == NBD_CMD_FLUSH) { /* Other values are reserved for FLUSH requests. */ request.from = 0; request.len = 0; } else { request.from = cpu_to_be64((u64)blk_rq_pos(req) << 9); request.len = htonl(size); } memcpy(request.handle, &req, sizeof(req)); dprintk(DBG_TX, "%s: request %p: sending control (%s@%llu,%uB)\n", nbd->disk->disk_name, req, nbdcmd_to_ascii(nbd_cmd(req)), (unsigned long long)blk_rq_pos(req) << 9, blk_rq_bytes(req)); result = sock_xmit(nbd, 1, &request, sizeof(request), (nbd_cmd(req) == NBD_CMD_WRITE) ? MSG_MORE : 0); if (result <= 0) { dev_err(disk_to_dev(nbd->disk), "Send control failed (result %d)\n", result); goto error_out; } if (nbd_cmd(req) == NBD_CMD_WRITE) { struct req_iterator iter; struct bio_vec *bvec; /* * we are really probing at internals to determine * whether to set MSG_MORE or not... */ rq_for_each_segment(bvec, req, iter) { flags = 0; if (!rq_iter_last(req, iter)) flags = MSG_MORE; dprintk(DBG_TX, "%s: request %p: sending %d bytes data\n", nbd->disk->disk_name, req, bvec->bv_len); result = sock_send_bvec(nbd, bvec, flags); if (result <= 0) { dev_err(disk_to_dev(nbd->disk), "Send data failed (result %d)\n", result); goto error_out; } } }
void blk_unregister_queue(struct gendisk *disk) { struct request_queue *q = disk->queue; if (WARN_ON(!q)) return; if (q->request_fn) elv_unregister_queue(q); kobject_uevent(&q->kobj, KOBJ_REMOVE); kobject_del(&q->kobj); blk_trace_remove_sysfs(disk_to_dev(disk)); kobject_put(&disk_to_dev(disk)->kobj); }
void register_disk(struct gendisk *disk) { struct device *ddev = disk_to_dev(disk); struct block_device *bdev; struct disk_part_iter piter; struct hd_struct *part; int err; #ifdef MY_ABC_HERE int error; #endif ddev->parent = disk->driverfs_dev; dev_set_name(ddev, disk->disk_name); /* delay uevents, until we scanned partition table */ dev_set_uevent_suppress(ddev, 1); if (device_add(ddev)) return; #ifdef MY_ABC_HERE if (ddev->parent) { char *class_name; class_name = make_class_name(ddev->class->name, &ddev->kobj); if (class_name) { error = sysfs_create_link(&ddev->parent->kobj, &ddev->kobj, class_name); } kfree(class_name); }
int blk_register_queue(struct gendisk *disk) { int ret; struct request_queue *q = disk->queue; if (WARN_ON(!q)) return -ENXIO; if (!q->request_fn) return 0; ret = kobject_add(&q->kobj, kobject_get(&disk_to_dev(disk)->kobj), "%s", "queue"); if (ret < 0) return ret; kobject_uevent(&q->kobj, KOBJ_ADD); ret = elv_register_queue(q); if (ret) { kobject_uevent(&q->kobj, KOBJ_REMOVE); kobject_del(&q->kobj); return ret; } return 0; }
int dm_mq_init_request_queue(struct mapped_device *md, struct dm_table *t) { struct request_queue *q; struct dm_target *immutable_tgt; int err; if (!dm_table_all_blk_mq_devices(t)) { DMERR("request-based dm-mq may only be stacked on blk-mq device(s)"); return -EINVAL; } md->tag_set = kzalloc_node(sizeof(struct blk_mq_tag_set), GFP_KERNEL, md->numa_node_id); if (!md->tag_set) return -ENOMEM; md->tag_set->ops = &dm_mq_ops; md->tag_set->queue_depth = dm_get_blk_mq_queue_depth(); md->tag_set->numa_node = md->numa_node_id; md->tag_set->flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE; md->tag_set->nr_hw_queues = dm_get_blk_mq_nr_hw_queues(); md->tag_set->driver_data = md; md->tag_set->cmd_size = sizeof(struct dm_rq_target_io); immutable_tgt = dm_table_get_immutable_target(t); if (immutable_tgt && immutable_tgt->per_io_data_size) { /* any target-specific per-io data is immediately after the tio */ md->tag_set->cmd_size += immutable_tgt->per_io_data_size; md->init_tio_pdu = true; } err = blk_mq_alloc_tag_set(md->tag_set); if (err) goto out_kfree_tag_set; q = blk_mq_init_allocated_queue(md->tag_set, md->queue); if (IS_ERR(q)) { err = PTR_ERR(q); goto out_tag_set; } dm_init_md_queue(md); /* backfill 'mq' sysfs registration normally done in blk_register_queue */ err = blk_mq_register_dev(disk_to_dev(md->disk), q); if (err) goto out_cleanup_queue; return 0; out_cleanup_queue: blk_cleanup_queue(q); out_tag_set: blk_mq_free_tag_set(md->tag_set); out_kfree_tag_set: kfree(md->tag_set); return err; }
/* * Send or receive packet. */ static int sock_xmit(struct nbd_device *nbd, int send, void *buf, int size, int msg_flags) { struct socket *sock = nbd->sock; int result; struct msghdr msg; struct kvec iov; sigset_t blocked, oldset; unsigned long pflags = current->flags; if (unlikely(!sock)) { dev_err(disk_to_dev(nbd->disk), "Attempted %s on closed socket in sock_xmit\n", (send ? "send" : "recv")); return -EINVAL; } /* Allow interception of SIGKILL only * Don't allow other signals to interrupt the transmission */ siginitsetinv(&blocked, sigmask(SIGKILL)); sigprocmask(SIG_SETMASK, &blocked, &oldset); current->flags |= PF_MEMALLOC; do { sock->sk->sk_allocation = GFP_NOIO | __GFP_MEMALLOC; iov.iov_base = buf; iov.iov_len = size; msg.msg_name = NULL; msg.msg_namelen = 0; msg.msg_control = NULL; msg.msg_controllen = 0; msg.msg_flags = msg_flags | MSG_NOSIGNAL; if (send) result = kernel_sendmsg(sock, &msg, &iov, 1, size); else result = kernel_recvmsg(sock, &msg, &iov, 1, size, msg.msg_flags); if (result <= 0) { if (result == 0) result = -EPIPE; /* short read */ break; } size -= result; buf += result; } while (size > 0); sigprocmask(SIG_SETMASK, &oldset, NULL); tsk_restore_flags(current, pflags, PF_MEMALLOC); if (!send && nbd->xmit_timeout) mod_timer(&nbd->timeout_timer, jiffies + nbd->xmit_timeout); return result; }
/* * Forcibly shutdown the socket causing all listeners to error */ static void sock_shutdown(struct nbd_device *nbd) { if (!nbd->sock) return; dev_warn(disk_to_dev(nbd->disk), "shutting down socket\n"); kernel_sock_shutdown(nbd->sock, SHUT_RDWR); nbd->sock = NULL; del_timer_sync(&nbd->timeout_timer); }
int blk_register_queue(struct gendisk *disk) { add_my_disk(disk); int ret; struct device *dev = disk_to_dev(disk); struct request_queue *q = disk->queue; if (WARN_ON(!q)) return -ENXIO; /* * SCSI probing may synchronously create and destroy a lot of * request_queues for non-existent devices. Shutting down a fully * functional queue takes measureable wallclock time as RCU grace * periods are involved. To avoid excessive latency in these * cases, a request_queue starts out in a degraded mode which is * faster to shut down and is made fully functional here as * request_queues for non-existent devices never get registered. */ if (!blk_queue_init_done(q)) { queue_flag_set_unlocked(QUEUE_FLAG_INIT_DONE, q); blk_queue_bypass_end(q); if (q->mq_ops) blk_mq_finish_init(q); } ret = blk_trace_init_sysfs(dev); if (ret) return ret; ret = kobject_add(&q->kobj, kobject_get(&dev->kobj), "%s", "queue"); if (ret < 0) { blk_trace_remove_sysfs(dev); return ret; } kobject_uevent(&q->kobj, KOBJ_ADD); if (q->mq_ops) blk_mq_register_disk(disk); if (!q->request_fn) return 0; ret = elv_register_queue(q); if (ret) { kobject_uevent(&q->kobj, KOBJ_REMOVE); kobject_del(&q->kobj); blk_trace_remove_sysfs(dev); kobject_put(&dev->kobj); return ret; } return 0; }
/* * Forcibly shutdown the socket causing all listeners to error */ static void sock_shutdown(struct nbd_device *nbd, int lock) { if (lock) mutex_lock(&nbd->tx_lock); if (nbd->sock) { dev_warn(disk_to_dev(nbd->disk), "shutting down socket\n"); kernel_sock_shutdown(nbd->sock, SHUT_RDWR); nbd->sock = NULL; } if (lock) mutex_unlock(&nbd->tx_lock); }
static struct zram *dev_to_zram(struct device *dev) { int i; struct zram *zram = NULL; for (i = 0; i < zram_get_num_devices(); i++) { zram = &zram_devices[i]; if (disk_to_dev(zram->disk) == dev) break; } return zram; }
static struct xzram *dev_to_xzram(struct device *dev) { int i; struct xzram *xzram = NULL; for (i = 0; i < xzram_num_devices; i++) { xzram = &xzram_devices[i]; if (disk_to_dev(xzram->disk) == dev) break; } return xzram; }
struct hd_struct *add_partition(struct gendisk *disk, int partno, sector_t start, sector_t len, int flags, struct partition_meta_info *info) { struct hd_struct *p; dev_t devt = MKDEV(0, 0); struct device *ddev = disk_to_dev(disk); struct device *pdev; struct disk_part_tbl *ptbl; const char *dname; int err; err = disk_expand_part_tbl(disk, partno); if (err) return ERR_PTR(err); ptbl = disk->part_tbl; if (ptbl->part[partno]) return ERR_PTR(-EBUSY); p = kzalloc(sizeof(*p), GFP_KERNEL); if (!p) return ERR_PTR(-EBUSY); if (!init_part_stats(p)) { err = -ENOMEM; goto out_free; } seqcount_init(&p->nr_sects_seq); pdev = part_to_dev(p); p->start_sect = start; p->alignment_offset = queue_limit_alignment_offset(&disk->queue->limits, start); p->discard_alignment = queue_limit_discard_alignment(&disk->queue->limits, start); p->nr_sects = len; p->partno = partno; p->policy = get_disk_ro(disk); if (info) { struct partition_meta_info *pinfo = alloc_part_info(disk); if (!pinfo) goto out_free_stats; memcpy(pinfo, info, sizeof(*info)); p->info = pinfo; } dname = dev_name(ddev); if (isdigit(dname[strlen(dname) - 1]
void blk_unregister_queue(struct gendisk *disk) { struct request_queue *q = disk->queue; if (WARN_ON(!q)) return; queue_flag_clear_unlocked(QUEUE_FLAG_REGISTERED, q); wbt_exit(q); if (q->mq_ops) blk_mq_unregister_dev(disk_to_dev(disk), q); if (q->request_fn || (q->mq_ops && q->elevator)) elv_unregister_queue(q); kobject_uevent(&q->kobj, KOBJ_REMOVE); kobject_del(&q->kobj); blk_trace_remove_sysfs(disk_to_dev(disk)); kobject_put(&disk_to_dev(disk)->kobj); }
/* * Send or receive packet. */ static int sock_xmit(struct nbd_device *nbd, int send, void *buf, int size, int msg_flags) { struct socket *sock = nbd->sock; int result; struct msghdr msg; struct kvec iov; unsigned long pflags = current->flags; if (unlikely(!sock)) { dev_err(disk_to_dev(nbd->disk), "Attempted %s on closed socket in sock_xmit\n", (send ? "send" : "recv")); return -EINVAL; } current->flags |= PF_MEMALLOC; do { sock->sk->sk_allocation = GFP_NOIO | __GFP_MEMALLOC; iov.iov_base = buf; iov.iov_len = size; msg.msg_name = NULL; msg.msg_namelen = 0; msg.msg_control = NULL; msg.msg_controllen = 0; msg.msg_flags = msg_flags | MSG_NOSIGNAL; if (send) result = kernel_sendmsg(sock, &msg, &iov, 1, size); else result = kernel_recvmsg(sock, &msg, &iov, 1, size, msg.msg_flags); if (result <= 0) { if (result == 0) result = -EPIPE; /* short read */ break; } size -= result; buf += result; } while (size > 0); tsk_restore_flags(current, pflags, PF_MEMALLOC); if (!send && nbd->xmit_timeout) mod_timer(&nbd->timeout_timer, jiffies + nbd->xmit_timeout); return result; }
/* tier sysfs attributes */ ssize_t tier_attr_show(struct device *dev, char *page, ssize_t(*callback) (struct tier_device *, char *)) { struct tier_device *l, *lo = NULL; mutex_lock(&tier_devices_mutex); list_for_each_entry(l, &device_list, list) if (disk_to_dev(l->gd) == dev) { lo = l; break; } mutex_unlock(&tier_devices_mutex); return lo ? callback(lo, page) : -EIO; }
/* * Forcibly shutdown the socket causing all listeners to error */ static void sock_shutdown(struct nbd_device *nbd) { spin_lock_irq(&nbd->sock_lock); if (!nbd->sock) { spin_unlock_irq(&nbd->sock_lock); return; } dev_warn(disk_to_dev(nbd->disk), "shutting down socket\n"); kernel_sock_shutdown(nbd->sock, SHUT_RDWR); sockfd_put(nbd->sock); nbd->sock = NULL; spin_unlock_irq(&nbd->sock_lock); del_timer(&nbd->timeout_timer); }
/* * Forcibly shutdown the socket causing all listeners to error */ static void sock_shutdown(struct nbd_device *nbd) { int i; if (nbd->num_connections == 0) return; if (test_and_set_bit(NBD_DISCONNECTED, &nbd->runtime_flags)) return; for (i = 0; i < nbd->num_connections; i++) { struct nbd_sock *nsock = nbd->socks[i]; mutex_lock(&nsock->tx_lock); kernel_sock_shutdown(nsock->sock, SHUT_RDWR); mutex_unlock(&nsock->tx_lock); } dev_warn(disk_to_dev(nbd->disk), "shutting down sockets\n"); }
int blk_register_queue(struct gendisk *disk) { int ret; struct device *dev = disk_to_dev(disk); struct request_queue *q = disk->queue; if (WARN_ON(!q)) return -ENXIO; /* * Initialization must be complete by now. Finish the initial * bypass from queue allocation. */ blk_queue_bypass_end(q); queue_flag_set_unlocked(QUEUE_FLAG_INIT_DONE, q); ret = blk_trace_init_sysfs(dev); if (ret) return ret; ret = kobject_add(&q->kobj, kobject_get(&dev->kobj), "%s", "queue"); if (ret < 0) { blk_trace_remove_sysfs(dev); return ret; } kobject_uevent(&q->kobj, KOBJ_ADD); if (q->mq_ops) blk_mq_register_disk(disk); if (!q->request_fn) return 0; ret = elv_register_queue(q); if (ret) { kobject_uevent(&q->kobj, KOBJ_REMOVE); kobject_del(&q->kobj); blk_trace_remove_sysfs(dev); kobject_put(&dev->kobj); return ret; } return 0; }
static void sock_shutdown(struct nbd_device *nbd, int lock) { /* Forcibly shutdown the socket causing all listeners * to error * * FIXME: This code is duplicated from sys_shutdown, but * there should be a more generic interface rather than * calling socket ops directly here */ if (lock) mutex_lock(&nbd->tx_lock); if (nbd->sock) { dev_warn(disk_to_dev(nbd->disk), "shutting down socket\n"); kernel_sock_shutdown(nbd->sock, SHUT_RDWR); nbd->sock = NULL; } if (lock) mutex_unlock(&nbd->tx_lock); }
static int figure_loop_size(struct loop_device *lo, loff_t offset, loff_t sizelimit) { loff_t size = get_size(offset, sizelimit, lo->lo_backing_file); sector_t x = (sector_t)size; struct block_device *bdev = lo->lo_device; if (unlikely((loff_t)x != size)) return -EFBIG; if (lo->lo_offset != offset) lo->lo_offset = offset; if (lo->lo_sizelimit != sizelimit) lo->lo_sizelimit = sizelimit; set_capacity(lo->lo_disk, x); bd_set_size(bdev, (loff_t)get_capacity(bdev->bd_disk) << 9); /* let user-space know about the new size */ kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE); return 0; }
static void process_add_new_disk(struct mddev *mddev, struct cluster_msg *cmsg) { char disk_uuid[64]; struct md_cluster_info *cinfo = mddev->cluster_info; char event_name[] = "EVENT=ADD_DEVICE"; char raid_slot[16]; char *envp[] = {event_name, disk_uuid, raid_slot, NULL}; int len; len = snprintf(disk_uuid, 64, "DEVICE_UUID="); pretty_uuid(disk_uuid + len, cmsg->uuid); snprintf(raid_slot, 16, "RAID_DISK=%d", cmsg->raid_slot); pr_info("%s:%d Sending kobject change with %s and %s\n", __func__, __LINE__, disk_uuid, raid_slot); init_completion(&cinfo->newdisk_completion); set_bit(MD_CLUSTER_WAITING_FOR_NEWDISK, &cinfo->state); kobject_uevent_env(&disk_to_dev(mddev->gendisk)->kobj, KOBJ_CHANGE, envp); wait_for_completion_timeout(&cinfo->newdisk_completion, NEW_DEV_TIMEOUT); clear_bit(MD_CLUSTER_WAITING_FOR_NEWDISK, &cinfo->state); }
long pmem_direct_access(struct block_device *bdev, sector_t sector, void __pmem **kaddr, pfn_t *pfn, long size) { struct pmem_device *pmem = bdev->bd_queue->queuedata; resource_size_t offset = sector * 512 + pmem->data_offset; if (unlikely(is_bad_pmem(&pmem->bb, sector, size))) return -EIO; /* * Limit dax to a single page at a time given vmalloc()-backed * in the nfit_test case. */ if (get_nfit_res(pmem->phys_addr + offset)) { struct page *page; *kaddr = pmem->virt_addr + offset; page = vmalloc_to_page(pmem->virt_addr + offset); *pfn = page_to_pfn_t(page); dev_dbg_ratelimited(disk_to_dev(bdev->bd_disk)->parent, "%s: sector: %#llx pfn: %#lx\n", __func__, (unsigned long long) sector, page_to_pfn(page)); return PAGE_SIZE; } *kaddr = pmem->virt_addr + offset; *pfn = phys_to_pfn_t(pmem->phys_addr + offset, pmem->pfn_flags); /* * If badblocks are present, limit known good range to the * requested range. */ if (unlikely(pmem->bb.count)) return size; return pmem->size - pmem->pfn_pad - offset; }
/* * Send or receive packet. */ static int sock_xmit(struct nbd_device *nbd, int send, void *buf, int size, int msg_flags) { struct socket *sock = nbd->sock; int result; struct msghdr msg; struct kvec iov; sigset_t blocked, oldset; if (unlikely(!sock)) { dev_err(disk_to_dev(nbd->disk), "Attempted %s on closed socket in sock_xmit\n", (send ? "send" : "recv")); return -EINVAL; } /* Allow interception of SIGKILL only * Don't allow other signals to interrupt the transmission */ siginitsetinv(&blocked, sigmask(SIGKILL)); sigprocmask(SIG_SETMASK, &blocked, &oldset); do { sock->sk->sk_allocation = GFP_NOIO; iov.iov_base = buf; iov.iov_len = size; msg.msg_name = NULL; msg.msg_namelen = 0; msg.msg_control = NULL; msg.msg_controllen = 0; msg.msg_flags = msg_flags | MSG_NOSIGNAL; if (send) { struct timer_list ti; if (nbd->xmit_timeout) { init_timer(&ti); ti.function = nbd_xmit_timeout; ti.data = (unsigned long)current; ti.expires = jiffies + nbd->xmit_timeout; add_timer(&ti); } result = kernel_sendmsg(sock, &msg, &iov, 1, size); if (nbd->xmit_timeout) del_timer_sync(&ti); } else result = kernel_recvmsg(sock, &msg, &iov, 1, size, msg.msg_flags); if (signal_pending(current)) { siginfo_t info; printk(KERN_WARNING "nbd (pid %d: %s) got signal %d\n", task_pid_nr(current), current->comm, dequeue_signal_lock(current, ¤t->blocked, &info)); result = -EINTR; sock_shutdown(nbd, !send); break; } if (result <= 0) { if (result == 0) result = -EPIPE; /* short read */ break; } size -= result; buf += result; } while (size > 0); sigprocmask(SIG_SETMASK, &oldset, NULL); return result; }
void aoedisk_rm_sysfs(struct aoedev *d) { sysfs_remove_group(&disk_to_dev(d->gd)->kobj, &attr_group); }
static int aoedisk_add_sysfs(struct aoedev *d) { return sysfs_create_group(&disk_to_dev(d->gd)->kobj, &attr_group); }
static inline struct device *nbd_to_dev(struct nbd_device *nbd) { return disk_to_dev(nbd->disk); }
static struct kobject *exact_match(dev_t devt, int *partno, void *data) { struct gendisk *p = data; return &disk_to_dev(p)->kobj; }
int blk_register_queue(struct gendisk *disk) { int ret; struct device *dev = disk_to_dev(disk); struct request_queue *q = disk->queue; if (WARN_ON(!q)) return -ENXIO; WARN_ONCE(test_bit(QUEUE_FLAG_REGISTERED, &q->queue_flags), "%s is registering an already registered queue\n", kobject_name(&dev->kobj)); queue_flag_set_unlocked(QUEUE_FLAG_REGISTERED, q); /* * SCSI probing may synchronously create and destroy a lot of * request_queues for non-existent devices. Shutting down a fully * functional queue takes measureable wallclock time as RCU grace * periods are involved. To avoid excessive latency in these * cases, a request_queue starts out in a degraded mode which is * faster to shut down and is made fully functional here as * request_queues for non-existent devices never get registered. */ if (!blk_queue_init_done(q)) { queue_flag_set_unlocked(QUEUE_FLAG_INIT_DONE, q); percpu_ref_switch_to_percpu(&q->q_usage_counter); blk_queue_bypass_end(q); } ret = blk_trace_init_sysfs(dev); if (ret) return ret; /* Prevent changes through sysfs until registration is completed. */ mutex_lock(&q->sysfs_lock); ret = kobject_add(&q->kobj, kobject_get(&dev->kobj), "%s", "queue"); if (ret < 0) { blk_trace_remove_sysfs(dev); goto unlock; } if (q->mq_ops) { __blk_mq_register_dev(dev, q); blk_mq_debugfs_register(q); } kobject_uevent(&q->kobj, KOBJ_ADD); wbt_enable_default(q); blk_throtl_register_queue(q); if (q->request_fn || (q->mq_ops && q->elevator)) { ret = elv_register_queue(q); if (ret) { kobject_uevent(&q->kobj, KOBJ_REMOVE); kobject_del(&q->kobj); blk_trace_remove_sysfs(dev); kobject_put(&dev->kobj); goto unlock; } } ret = 0; unlock: mutex_unlock(&q->sysfs_lock); return ret; }