static void mmc_setup_queue(struct mmc_queue *mq, struct mmc_card *card)
{
struct mmc_host *host = card->host;
u64 limit = BLK_BOUNCE_HIGH;
if (mmc_dev(host)->dma_mask && *mmc_dev(host)->dma_mask)
limit = (u64)dma_max_pfn(mmc_dev(host)) << PAGE_SHIFT;
blk_queue_flag_set(QUEUE_FLAG_NONROT, mq->queue);
blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, mq->queue);
if (mmc_can_erase(card))
mmc_queue_setup_discard(mq->queue, card);
blk_queue_bounce_limit(mq->queue, limit);
blk_queue_max_hw_sectors(mq->queue,
min(host->max_blk_count, host->max_req_size / 512));
blk_queue_max_segments(mq->queue, host->max_segs);
blk_queue_max_segment_size(mq->queue, host->max_seg_size);
INIT_WORK(&mq->recovery_work, mmc_mq_recovery_handler);
INIT_WORK(&mq->complete_work, mmc_blk_mq_complete_work);
mutex_init(&mq->complete_lock);
init_waitqueue_head(&mq->wait);
}
int card_init_queue(struct card_queue *cq, struct memory_card *card,
spinlock_t * lock)
{
struct card_host *host = card->host;
u64 limit = BLK_BOUNCE_HIGH;
int ret=0;
if (host->parent->dma_mask && *host->parent->dma_mask)
limit = *host->parent->dma_mask;
cq->card = card;
cq->queue = blk_init_queue(card_request, lock);
if (!cq->queue)
return -ENOMEM;
blk_queue_prep_rq(cq->queue, card_prep_request);
card_init_bounce_buf(cq, card);
if(!cq->bounce_buf){
blk_queue_bounce_limit(cq->queue, limit);
blk_queue_max_hw_sectors(cq->queue, host->max_sectors);
//blk_queue_max_hw_phys_segments(cq->queue, host->max_phys_segs);
blk_queue_max_segments(cq->queue, host->max_hw_segs);
blk_queue_max_segment_size(cq->queue, host->max_seg_size);
cq->queue->queuedata = cq;
cq->req = NULL;
cq->sg = kmalloc(sizeof(struct scatterlist) * host->max_phys_segs, GFP_KERNEL);
if (!cq->sg) {
ret = -ENOMEM;
blk_cleanup_queue(cq->queue);
return ret;
}
}
/*change card io scheduler from cfq to deadline*/
cq->queue->queuedata = cq;
elevator_exit(cq->queue->elevator);
cq->queue->elevator = NULL;
ret = elevator_init(cq->queue, "deadline");
if (ret) {
printk("[card_init_queue] elevator_init deadline fail\n");
blk_cleanup_queue(cq->queue);
return ret;
}
init_MUTEX(&cq->thread_sem);
cq->thread = kthread_run(card_queue_thread, cq, "%s_queue", card->name);
if (IS_ERR(cq->thread)) {
ret = PTR_ERR(cq->thread);
//goto free_bounce_sg;
}
cq->nb.notifier_call = card_reboot_notifier;
register_reboot_notifier(&cq->nb);
return ret;
}
/*
* Alloc bounce buf for read/write numbers of pages in one request
*/
static int card_init_bounce_buf(struct card_queue *cq,
struct memory_card *card)
{
int ret=0;
struct card_host *host = card->host;
unsigned int bouncesz;
bouncesz = CARD_QUEUE_BOUNCESZ;
if (bouncesz > host->max_req_size)
bouncesz = host->max_req_size;
if (bouncesz >= PAGE_CACHE_SIZE) {
//cq->bounce_buf = kmalloc(bouncesz, GFP_KERNEL);
cq->bounce_buf = host->dma_buf;
if (!cq->bounce_buf) {
printk(KERN_WARNING "%s: unable to "
"allocate bounce buffer\n", card->name);
}
}
if (cq->bounce_buf) {
blk_queue_bounce_limit(cq->queue, BLK_BOUNCE_HIGH);
blk_queue_max_hw_sectors(cq->queue, bouncesz / 512);
blk_queue_physical_block_size(cq->queue, bouncesz);
blk_queue_max_segments(cq->queue, bouncesz / PAGE_CACHE_SIZE);
blk_queue_max_segment_size(cq->queue, bouncesz);
cq->queue->queuedata = cq;
cq->req = NULL;
cq->sg = kmalloc(sizeof(struct scatterlist),
GFP_KERNEL);
if (!cq->sg) {
ret = -ENOMEM;
blk_cleanup_queue(cq->queue);
return ret;
}
sg_init_table(cq->sg, 1);
cq->bounce_sg = kmalloc(sizeof(struct scatterlist) *
bouncesz / PAGE_CACHE_SIZE, GFP_KERNEL);
if (!cq->bounce_sg) {
ret = -ENOMEM;
kfree(cq->sg);
cq->sg = NULL;
blk_cleanup_queue(cq->queue);
return ret;
}
sg_init_table(cq->bounce_sg, bouncesz / PAGE_CACHE_SIZE);
}
return 0;
}
/**
* mmc_init_queue - initialise a queue structure.
* @mq: mmc queue
* @card: mmc card to attach this queue
* @lock: queue lock
*
* Initialise a MMC card request queue.
*/
int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card, spinlock_t *lock)
{
struct mmc_host *host = card->host;
u64 limit = BLK_BOUNCE_HIGH;
int ret;
if (host->dev->dma_mask && *host->dev->dma_mask)
limit = *host->dev->dma_mask;
mq->card = card;
mq->queue = blk_init_queue(mmc_request, lock);
if (!mq->queue)
return -ENOMEM;
blk_queue_prep_rq(mq->queue, mmc_prep_request);
blk_queue_bounce_limit(mq->queue, limit);
blk_queue_max_sectors(mq->queue, host->max_sectors);
blk_queue_max_phys_segments(mq->queue, host->max_phys_segs);
blk_queue_max_hw_segments(mq->queue, host->max_hw_segs);
blk_queue_max_segment_size(mq->queue, host->max_seg_size);
mq->queue->queuedata = mq;
mq->req = NULL;
mq->sg = kmalloc(sizeof(struct scatterlist) * host->max_phys_segs,
GFP_KERNEL);
if (!mq->sg) {
ret = -ENOMEM;
goto cleanup;
}
init_completion(&mq->thread_complete);
init_waitqueue_head(&mq->thread_wq);
init_MUTEX(&mq->thread_sem);
ret = kernel_thread(mmc_queue_thread, mq, CLONE_KERNEL);
if (ret >= 0) {
wait_for_completion(&mq->thread_complete);
init_completion(&mq->thread_complete);
ret = 0;
goto out;
}
cleanup:
kfree(mq->sg);
mq->sg = NULL;
blk_cleanup_queue(mq->queue);
out:
return ret;
}
static int __init stheno_module_init( void )
{
stheno_major = register_blkdev( 0, MODNAME );
if( stheno_major <= 0 ){
printk( KERN_WARNING "register_blkdev failed\n" );
return stheno_major;
}
spin_lock_init( &stheno_lock );
stheno_queue = blk_init_queue(
stheno_request, //
&stheno_lock );
if( ! stheno_queue ){
printk( KERN_WARNING "blk_init_queue failed\n" );
unregister_blkdev( stheno_major, MODNAME );
return -ENOMEM;
}
blk_queue_logical_block_size( stheno_queue, SECT_SIZE );
blk_queue_max_sectors( stheno_queue, MAX_SECTORS );
blk_queue_bounce_limit(stheno_queue, BLK_BOUNCE_ANY);
stheno_gd = alloc_disk( MINOR_COUNT );
if( ! stheno_gd ){
printk( KERN_WARNING "alloc_disk failed\n" );
blk_cleanup_queue( stheno_queue );
unregister_blkdev( stheno_major, MODNAME );
return -ENOMEM;
}
sprintf( stheno_gd->disk_name, "%s", MODNAME ); //
stheno_gd->queue = stheno_queue; //
stheno_gd->major = stheno_major;
stheno_gd->first_minor = 0;
stheno_gd->fops = &stheno_fops; //
set_capacity( stheno_gd, SECT_NUM );
sema_init(&stheno_sem, 1);
init_waitqueue_head(&stheno_process_q);
stheno_thread = kthread_create(stheno_do_request, 0, "sthenod");
wake_up_process(stheno_thread);
add_disk( stheno_gd );
printk( KERN_INFO "stheno is loaded\n" );
printk( KERN_INFO "major = %d\n", stheno_major );
return 0;
}
/**
* mmc_init_queue - initialise a queue structure.
* @mq: mmc queue
* @card: mmc card to attach this queue
* @lock: queue lock
*
* Initialise a MMC card request queue.
*/
int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card, spinlock_t *lock)
{
struct mmc_host *host = card->host;
u64 limit = BLK_BOUNCE_HIGH;
int ret;
if (mmc_dev(host)->dma_mask && *mmc_dev(host)->dma_mask)
limit = *mmc_dev(host)->dma_mask;
mq->card = card;
mq->queue = blk_init_queue(mmc_request, lock);
if (!mq->queue)
return -ENOMEM;
blk_queue_prep_rq(mq->queue, mmc_prep_request);
blk_queue_bounce_limit(mq->queue, limit);
blk_queue_max_sectors(mq->queue, host->max_req_size / 512);
blk_queue_max_phys_segments(mq->queue, host->max_phys_segs);
blk_queue_max_hw_segments(mq->queue, host->max_hw_segs);
blk_queue_max_segment_size(mq->queue, host->max_seg_size);
mq->queue->queuedata = mq;
mq->req = NULL;
mq->sg = kmalloc(sizeof(struct scatterlist) * host->max_phys_segs,
GFP_KERNEL);
if (!mq->sg) {
ret = -ENOMEM;
goto cleanup_queue;
}
init_MUTEX(&mq->thread_sem);
mq->thread = kthread_run(mmc_queue_thread, mq, "mmcqd");
if (IS_ERR(mq->thread)) {
ret = PTR_ERR(mq->thread);
goto free_sg;
}
return 0;
free_sg:
kfree(mq->sg);
mq->sg = NULL;
cleanup_queue:
blk_cleanup_queue(mq->queue);
return ret;
}
int cyasblkdev_init_queue(struct cyasblkdev_queue *bq, spinlock_t *lock)
{
int ret;
DBGPRN_FUNC_NAME;
/* 1st param is a function that wakes up the queue thread */
bq->queue = blk_init_queue(cyasblkdev_request, lock);
if (!bq->queue)
return -ENOMEM;
blk_queue_prep_rq(bq->queue, cyasblkdev_prep_request);
blk_queue_bounce_limit(bq->queue, BLK_BOUNCE_ANY);
blk_queue_max_hw_sectors(bq->queue, Q_MAX_SECTORS);
/* As of now, we have the HAL/driver support to
* merge scattered segments and handle them simultaneously.
* so, setting the max_phys_segments to 8. */
/*blk_queue_max_phys_segments(bq->queue, Q_MAX_SGS);
blk_queue_max_hw_segments(bq->queue, Q_MAX_SGS);*/
blk_queue_max_segments(bq->queue, Q_MAX_SGS);
/* should be < then HAL can handle */
blk_queue_max_segment_size(bq->queue, 512*Q_MAX_SECTORS);
bq->queue->queuedata = bq;
bq->req = NULL;
init_completion(&bq->thread_complete);
init_waitqueue_head(&bq->thread_wq);
sema_init(&bq->thread_sem, 1);
ret = kernel_thread(cyasblkdev_queue_thread, bq, CLONE_KERNEL);
if (ret >= 0) {
/* wait until the thread is spawned */
wait_for_completion(&bq->thread_complete);
/* reinitialize the completion */
init_completion(&bq->thread_complete);
ret = 0;
goto out;
}
out:
return ret;
}
static int swim3_attach(struct macio_dev *mdev,
const struct of_device_id *match)
{
struct gendisk *disk;
int index, rc;
index = floppy_count++;
if (index >= MAX_FLOPPIES)
return -ENXIO;
/* Add the drive */
rc = swim3_add_device(mdev, index);
if (rc)
return rc;
/* Now register that disk. Same comment about failure handling */
disk = disks[index] = alloc_disk(1);
if (disk == NULL)
return -ENOMEM;
disk->queue = blk_init_queue(do_fd_request, &swim3_lock);
if (disk->queue == NULL) {
put_disk(disk);
return -ENOMEM;
}
blk_queue_bounce_limit(disk->queue, BLK_BOUNCE_HIGH);
disk->queue->queuedata = &floppy_states[index];
if (index == 0) {
/* If we failed, there isn't much we can do as the driver is still
* too dumb to remove the device, just bail out
*/
if (register_blkdev(FLOPPY_MAJOR, "fd"))
return 0;
}
disk->major = FLOPPY_MAJOR;
disk->first_minor = index;
disk->fops = &floppy_fops;
disk->private_data = &floppy_states[index];
disk->flags |= GENHD_FL_REMOVABLE;
sprintf(disk->disk_name, "fd%d", index);
set_capacity(disk, 2880);
add_disk(disk);
return 0;
}
static void mmc_setup_queue(struct mmc_queue *mq, struct mmc_card *card)
{
struct mmc_host *host = card->host;
u64 limit = BLK_BOUNCE_HIGH;
if (mmc_dev(host)->dma_mask && *mmc_dev(host)->dma_mask)
limit = (u64)dma_max_pfn(mmc_dev(host)) << PAGE_SHIFT;
queue_flag_set_unlocked(QUEUE_FLAG_NONROT, mq->queue);
queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, mq->queue);
if (mmc_can_erase(card))
mmc_queue_setup_discard(mq->queue, card);
blk_queue_bounce_limit(mq->queue, limit);
blk_queue_max_hw_sectors(mq->queue,
min(host->max_blk_count, host->max_req_size / 512));
blk_queue_max_segments(mq->queue, host->max_segs);
blk_queue_max_segment_size(mq->queue, host->max_seg_size);
/* Initialize thread_sem even if it is not used */
sema_init(&mq->thread_sem, 1);
}
static int pmem_attach_disk(struct device *dev,
struct nd_namespace_common *ndns, struct pmem_device *pmem)
{
struct gendisk *disk;
pmem->pmem_queue = blk_alloc_queue(GFP_KERNEL);
if (!pmem->pmem_queue)
return -ENOMEM;
blk_queue_make_request(pmem->pmem_queue, pmem_make_request);
blk_queue_physical_block_size(pmem->pmem_queue, PAGE_SIZE);
blk_queue_max_hw_sectors(pmem->pmem_queue, UINT_MAX);
blk_queue_bounce_limit(pmem->pmem_queue, BLK_BOUNCE_ANY);
queue_flag_set_unlocked(QUEUE_FLAG_NONROT, pmem->pmem_queue);
disk = alloc_disk(0);
if (!disk) {
blk_cleanup_queue(pmem->pmem_queue);
return -ENOMEM;
}
disk->major = pmem_major;
disk->first_minor = 0;
disk->fops = &pmem_fops;
disk->private_data = pmem;
disk->queue = pmem->pmem_queue;
disk->flags = GENHD_FL_EXT_DEVT;
nvdimm_namespace_disk_name(ndns, disk->disk_name);
disk->driverfs_dev = dev;
set_capacity(disk, (pmem->size - pmem->data_offset) / 512);
pmem->pmem_disk = disk;
add_disk(disk);
revalidate_disk(disk);
return 0;
}
/*
* Allocate and initialise a blank device with a given minor.
*/
static struct mapped_device *alloc_dev(int minor)
{
int r;
struct mapped_device *md = kmalloc(sizeof(*md), GFP_KERNEL);
void *old_md;
if (!md) {
DMWARN("unable to allocate device, out of memory.");
return NULL;
}
if (!try_module_get(THIS_MODULE))
goto bad0;
/* get a minor number for the dev */
if (minor == DM_ANY_MINOR)
r = next_free_minor(md, &minor);
else
r = specific_minor(md, minor);
if (r < 0)
goto bad1;
memset(md, 0, sizeof(*md));
init_rwsem(&md->io_lock);
init_MUTEX(&md->suspend_lock);
spin_lock_init(&md->pushback_lock);
rwlock_init(&md->map_lock);
atomic_set(&md->holders, 1);
atomic_set(&md->open_count, 0);
atomic_set(&md->event_nr, 0);
md->queue = blk_alloc_queue(GFP_KERNEL);
if (!md->queue)
goto bad1_free_minor;
md->queue->queuedata = md;
md->queue->backing_dev_info.congested_fn = dm_any_congested;
md->queue->backing_dev_info.congested_data = md;
blk_queue_make_request(md->queue, dm_request);
blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
md->queue->unplug_fn = dm_unplug_all;
md->queue->issue_flush_fn = dm_flush_all;
md->io_pool = mempool_create_slab_pool(MIN_IOS, _io_cache);
if (!md->io_pool)
goto bad2;
md->tio_pool = mempool_create_slab_pool(MIN_IOS, _tio_cache);
if (!md->tio_pool)
goto bad3;
md->bs = bioset_create(16, 16);
if (!md->bs)
goto bad_no_bioset;
md->disk = alloc_disk(1);
if (!md->disk)
goto bad4;
atomic_set(&md->pending, 0);
init_waitqueue_head(&md->wait);
init_waitqueue_head(&md->eventq);
md->disk->major = _major;
md->disk->first_minor = minor;
md->disk->fops = &dm_blk_dops;
md->disk->queue = md->queue;
md->disk->private_data = md;
sprintf(md->disk->disk_name, "dm-%d", minor);
add_disk(md->disk);
format_dev_t(md->name, MKDEV(_major, minor));
/* Populate the mapping, nobody knows we exist yet */
spin_lock(&_minor_lock);
old_md = idr_replace(&_minor_idr, md, minor);
spin_unlock(&_minor_lock);
BUG_ON(old_md != MINOR_ALLOCED);
return md;
bad4:
bioset_free(md->bs);
bad_no_bioset:
mempool_destroy(md->tio_pool);
bad3:
mempool_destroy(md->io_pool);
bad2:
blk_cleanup_queue(md->queue);
bad1_free_minor:
free_minor(minor);
bad1:
module_put(THIS_MODULE);
bad0:
kfree(md);
return NULL;
}
/**
* mmc_init_queue - initialise a queue structure.
* @mq: mmc queue
* @card: mmc card to attach this queue
* @lock: queue lock
* @subname: partition subname
*
* Initialise a MMC card request queue.
*/
int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card,
spinlock_t *lock, const char *subname)
{
struct mmc_host *host = card->host;
u64 limit = BLK_BOUNCE_HIGH;
int ret;
if (mmc_dev(host)->dma_mask && *mmc_dev(host)->dma_mask)
limit = *mmc_dev(host)->dma_mask;
mq->card = card;
mq->queue = blk_init_queue(mmc_request, lock);
if (!mq->queue)
return -ENOMEM;
mq->queue->queuedata = mq;
mq->req = NULL;
blk_queue_prep_rq(mq->queue, mmc_prep_request);
queue_flag_set_unlocked(QUEUE_FLAG_NONROT, mq->queue);
if (mmc_can_erase(card)) {
queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mq->queue);
mq->queue->limits.max_discard_sectors = UINT_MAX;
if (card->erased_byte == 0)
mq->queue->limits.discard_zeroes_data = 1;
mq->queue->limits.discard_granularity = card->pref_erase << 9;
if (mmc_can_secure_erase_trim(card))
queue_flag_set_unlocked(QUEUE_FLAG_SECDISCARD,
mq->queue);
}
#ifdef CONFIG_MMC_BLOCK_BOUNCE
if (host->max_segs == 1) {
unsigned int bouncesz;
bouncesz = MMC_QUEUE_BOUNCESZ;
if (bouncesz > host->max_req_size)
bouncesz = host->max_req_size;
if (bouncesz > host->max_seg_size)
bouncesz = host->max_seg_size;
if (bouncesz > (host->max_blk_count * 512))
bouncesz = host->max_blk_count * 512;
if (bouncesz > 512) {
mq->bounce_buf = kmalloc(bouncesz, GFP_KERNEL);
if (!mq->bounce_buf) {
printk(KERN_WARNING "%s: unable to "
"allocate bounce buffer\n",
mmc_card_name(card));
}
}
if (mq->bounce_buf) {
blk_queue_bounce_limit(mq->queue, BLK_BOUNCE_ANY);
blk_queue_max_hw_sectors(mq->queue, bouncesz / 512);
blk_queue_max_segments(mq->queue, bouncesz / 512);
blk_queue_max_segment_size(mq->queue, bouncesz);
mq->sg = kmalloc(sizeof(struct scatterlist),
GFP_KERNEL);
if (!mq->sg) {
ret = -ENOMEM;
goto cleanup_queue;
}
sg_init_table(mq->sg, 1);
mq->bounce_sg = kmalloc(sizeof(struct scatterlist) *
bouncesz / 512, GFP_KERNEL);
if (!mq->bounce_sg) {
ret = -ENOMEM;
goto cleanup_queue;
}
sg_init_table(mq->bounce_sg, bouncesz / 512);
}
}
#endif
if (!mq->bounce_buf) {
blk_queue_bounce_limit(mq->queue, limit);
blk_queue_max_hw_sectors(mq->queue,
min(host->max_blk_count, host->max_req_size / 512));
blk_queue_max_segments(mq->queue, host->max_segs);
blk_queue_max_segment_size(mq->queue, host->max_seg_size);
mq->sg = kmalloc(sizeof(struct scatterlist) *
host->max_segs, GFP_KERNEL);
if (!mq->sg) {
ret = -ENOMEM;
goto cleanup_queue;
}
sg_init_table(mq->sg, host->max_segs);
}
sema_init(&mq->thread_sem, 1);
mq->thread = kthread_run(mmc_queue_thread, mq, "mmcqd/%d%s",
host->index, subname ? subname : "");
if (IS_ERR(mq->thread)) {
ret = PTR_ERR(mq->thread);
goto free_bounce_sg;
}
return 0;
free_bounce_sg:
if (mq->bounce_sg)
kfree(mq->bounce_sg);
mq->bounce_sg = NULL;
cleanup_queue:
if (mq->sg)
kfree(mq->sg);
mq->sg = NULL;
if (mq->bounce_buf)
kfree(mq->bounce_buf);
mq->bounce_buf = NULL;
blk_cleanup_queue(mq->queue);
return ret;
}
static int iscsi_sw_tcp_slave_configure(struct scsi_device *sdev)
{
blk_queue_bounce_limit(sdev->request_queue, BLK_BOUNCE_ANY);
blk_queue_dma_alignment(sdev->request_queue, 0);
return 0;
}
static int __init stheno_module_init( void )
{
int retval;
print_info( "stheno_module_init was called.\n" );
init_waitqueue_head( &stheno_wait_q );
wake_lock_init( &stheno_wakelock, WAKE_LOCK_SUSPEND, STHENO_NAME );
stheno_major = register_blkdev( 0, STHENO_NAME );
if( stheno_major <= 0 ){
print_error( "stheno register_blkdev failed.\n" );
retval = -EBUSY;
goto error;
}
spin_lock_init( &stheno_lock );
stheno_queue = blk_init_queue( stheno_request, &stheno_lock );
if( stheno_queue == NULL ){
print_error( "stheno blk_init_queue failed.\n" );
retval = -ENOMEM;
goto error;
}
/*blk_queue_hardsect_size( stheno_queue, SECTOR_SIZE );*/
/*blk_queue_max_sectors( stheno_queue, MAX_SECTORS );*/
blk_queue_logical_block_size( stheno_queue, SECTOR_SIZE );
blk_queue_max_hw_sectors( stheno_queue, MAX_SECTORS );
#if defined( STHENO_BLK_BOUNCE_ANY )
blk_queue_bounce_limit( stheno_queue, BLK_BOUNCE_ANY );
#else
blk_queue_bounce_limit( stheno_queue, BLK_BOUNCE_HIGH ); /* default */
#endif
stheno_gd = alloc_disk( STHENO_MINOR_COUNT );
if( stheno_gd == NULL ){
print_error( "stheno alloc_disk failed.\n" );
retval = -ENOMEM;
goto error;
}
stheno_gd->major = stheno_major;
stheno_gd->first_minor = 0;
stheno_gd->fops = &stheno_fops;
stheno_gd->queue = stheno_queue;
/*stheno_gd->flags = GENHD_FL_REMOVABLE;*/
/*stheno_gd->private_data = NULL;*/
snprintf( stheno_gd->disk_name, DISK_NAME_LEN, "%s", STHENO_NAME );
set_capacity( stheno_gd, AMOUNT_OF_SECTORS );
stheno_thread = kthread_create( stheno_request_thread, 0, STHENO_THREAD_NAME );
if( IS_ERR( stheno_thread ) ){
print_error( "stheno kthread_create failed.\n" );
retval = -EBUSY;
goto error;
}
wake_up_process( stheno_thread );
add_disk( stheno_gd );
print_debug( "stheno major = %d\n", stheno_major );
return 0;
error:
if( stheno_gd != NULL ) del_gendisk( stheno_gd );
if( stheno_queue != NULL ) blk_cleanup_queue( stheno_queue );
if( stheno_major > 0 ) unregister_blkdev( stheno_major, STHENO_NAME );
return retval;
}
/* pdev is NULL for eisa */
static int cpqarray_register_ctlr( int i, struct pci_dev *pdev)
{
request_queue_t *q;
int j;
/*
* register block devices
* Find disks and fill in structs
* Get an interrupt, set the Q depth and get into /proc
*/
/* If this successful it should insure that we are the only */
/* instance of the driver */
if (register_blkdev(COMPAQ_SMART2_MAJOR+i, hba[i]->devname)) {
goto Enomem4;
}
hba[i]->access.set_intr_mask(hba[i], 0);
if (request_irq(hba[i]->intr, do_ida_intr,
SA_INTERRUPT|SA_SHIRQ|SA_SAMPLE_RANDOM,
hba[i]->devname, hba[i]))
{
printk(KERN_ERR "cpqarray: Unable to get irq %d for %s\n",
hba[i]->intr, hba[i]->devname);
goto Enomem3;
}
for (j=0; j<NWD; j++) {
ida_gendisk[i][j] = alloc_disk(1 << NWD_SHIFT);
if (!ida_gendisk[i][j])
goto Enomem2;
}
hba[i]->cmd_pool = (cmdlist_t *)pci_alloc_consistent(
hba[i]->pci_dev, NR_CMDS * sizeof(cmdlist_t),
&(hba[i]->cmd_pool_dhandle));
hba[i]->cmd_pool_bits = kmalloc(
((NR_CMDS+BITS_PER_LONG-1)/BITS_PER_LONG)*sizeof(unsigned long),
GFP_KERNEL);
if (!hba[i]->cmd_pool_bits || !hba[i]->cmd_pool)
goto Enomem1;
memset(hba[i]->cmd_pool, 0, NR_CMDS * sizeof(cmdlist_t));
memset(hba[i]->cmd_pool_bits, 0, ((NR_CMDS+BITS_PER_LONG-1)/BITS_PER_LONG)*sizeof(unsigned long));
printk(KERN_INFO "cpqarray: Finding drives on %s",
hba[i]->devname);
spin_lock_init(&hba[i]->lock);
q = blk_init_queue(do_ida_request, &hba[i]->lock);
if (!q)
goto Enomem1;
hba[i]->queue = q;
q->queuedata = hba[i];
getgeometry(i);
start_fwbk(i);
ida_procinit(i);
if (pdev)
blk_queue_bounce_limit(q, hba[i]->pci_dev->dma_mask);
/* This is a hardware imposed limit. */
blk_queue_max_hw_segments(q, SG_MAX);
/* This is a driver limit and could be eliminated. */
blk_queue_max_phys_segments(q, SG_MAX);
init_timer(&hba[i]->timer);
hba[i]->timer.expires = jiffies + IDA_TIMER;
hba[i]->timer.data = (unsigned long)hba[i];
hba[i]->timer.function = ida_timer;
add_timer(&hba[i]->timer);
/* Enable IRQ now that spinlock and rate limit timer are set up */
hba[i]->access.set_intr_mask(hba[i], FIFO_NOT_EMPTY);
for(j=0; j<NWD; j++) {
struct gendisk *disk = ida_gendisk[i][j];
drv_info_t *drv = &hba[i]->drv[j];
sprintf(disk->disk_name, "ida/c%dd%d", i, j);
disk->major = COMPAQ_SMART2_MAJOR + i;
disk->first_minor = j<<NWD_SHIFT;
disk->fops = &ida_fops;
if (j && !drv->nr_blks)
continue;
blk_queue_hardsect_size(hba[i]->queue, drv->blk_size);
set_capacity(disk, drv->nr_blks);
disk->queue = hba[i]->queue;
disk->private_data = drv;
add_disk(disk);
}
/* done ! */
return(i);
Enomem1:
nr_ctlr = i;
kfree(hba[i]->cmd_pool_bits);
if (hba[i]->cmd_pool)
pci_free_consistent(hba[i]->pci_dev, NR_CMDS*sizeof(cmdlist_t),
hba[i]->cmd_pool, hba[i]->cmd_pool_dhandle);
Enomem2:
while (j--) {
put_disk(ida_gendisk[i][j]);
ida_gendisk[i][j] = NULL;
}
free_irq(hba[i]->intr, hba[i]);
Enomem3:
unregister_blkdev(COMPAQ_SMART2_MAJOR+i, hba[i]->devname);
Enomem4:
if (pdev)
pci_set_drvdata(pdev, NULL);
release_io_mem(hba[i]);
free_hba(i);
printk( KERN_ERR "cpqarray: out of memory");
return -1;
}
/**
* mmc_init_queue - initialise a queue structure.
* @mq: mmc queue
* @card: mmc card to attach this queue
* @lock: queue lock
*
* Initialise a MMC card request queue.
*/
int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card, spinlock_t *lock)
{
struct mmc_host *host = card->host;
u64 limit = BLK_BOUNCE_HIGH;
int ret;
if (mmc_dev(host)->dma_mask && *mmc_dev(host)->dma_mask)
limit = *mmc_dev(host)->dma_mask;
mq->card = card;
mq->queue = blk_init_queue(mmc_request, lock);
if (!mq->queue)
return -ENOMEM;
mq->queue->queuedata = mq;
mq->req = NULL;
blk_queue_prep_rq(mq->queue, mmc_prep_request);
blk_queue_ordered(mq->queue, QUEUE_ORDERED_DRAIN, NULL);
queue_flag_set_unlocked(QUEUE_FLAG_NONROT, mq->queue);
/* Set max discard size, << 11 converts to megabytes in sectors */
blk_queue_max_discard_sectors(mq->queue, 16 << 11);
if (card->csd.cmdclass & CCC_ERASE)
queue_flag_set_unlocked(QUEUE_FLAG_DISCARD,
mq->queue);
/*
* Calculating a correct span is way to messy if this
* assumption is broken, so remove the erase support
*/
if (unlikely(mmc_card_blockaddr(card) &&
(card->csd.erase_size % 512)))
queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD,
mq->queue);
#ifdef CONFIG_MMC_BLOCK_BOUNCE
if (host->max_hw_segs == 1) {
unsigned int bouncesz;
bouncesz = MMC_QUEUE_BOUNCESZ;
if (bouncesz > host->max_req_size)
bouncesz = host->max_req_size;
if (bouncesz > host->max_seg_size)
bouncesz = host->max_seg_size;
if (bouncesz > (host->max_blk_count * 512))
bouncesz = host->max_blk_count * 512;
if (bouncesz > 512) {
mq->bounce_buf = kmalloc(bouncesz, GFP_KERNEL);
if (!mq->bounce_buf) {
printk(KERN_WARNING "%s: unable to "
"allocate bounce buffer\n",
mmc_card_name(card));
}
}
if (mq->bounce_buf) {
blk_queue_bounce_limit(mq->queue, BLK_BOUNCE_ANY);
blk_queue_max_hw_sectors(mq->queue, bouncesz / 512);
blk_queue_max_segments(mq->queue, bouncesz / 512);
blk_queue_max_segment_size(mq->queue, bouncesz);
mq->sg = kmalloc(sizeof(struct scatterlist),
GFP_KERNEL);
if (!mq->sg) {
ret = -ENOMEM;
goto cleanup_queue;
}
sg_init_table(mq->sg, 1);
mq->bounce_sg = kmalloc(sizeof(struct scatterlist) *
bouncesz / 512, GFP_KERNEL);
if (!mq->bounce_sg) {
ret = -ENOMEM;
goto cleanup_queue;
}
sg_init_table(mq->bounce_sg, bouncesz / 512);
}
}
#endif
if (!mq->bounce_buf) {
blk_queue_bounce_limit(mq->queue, limit);
blk_queue_max_hw_sectors(mq->queue,
min(host->max_blk_count, host->max_req_size / 512));
blk_queue_max_segments(mq->queue, host->max_hw_segs);
blk_queue_max_segment_size(mq->queue, host->max_seg_size);
mq->sg = kmalloc(sizeof(struct scatterlist) *
host->max_phys_segs, GFP_KERNEL);
if (!mq->sg) {
ret = -ENOMEM;
goto cleanup_queue;
}
sg_init_table(mq->sg, host->max_phys_segs);
}
init_MUTEX(&mq->thread_sem);
mq->thread = kthread_run(mmc_queue_thread, mq, "mmcqd");
if (IS_ERR(mq->thread)) {
ret = PTR_ERR(mq->thread);
goto free_bounce_sg;
}
return 0;
free_bounce_sg:
if (mq->bounce_sg)
kfree(mq->bounce_sg);
mq->bounce_sg = NULL;
cleanup_queue:
if (mq->sg)
kfree(mq->sg);
mq->sg = NULL;
if (mq->bounce_buf)
kfree(mq->bounce_buf);
mq->bounce_buf = NULL;
blk_cleanup_queue(mq->queue);
return ret;
}
int card_init_queue(struct card_queue *cq, struct memory_card *card,
spinlock_t * lock)
{
struct card_host *host = card->host;
u64 limit = BLK_BOUNCE_HIGH;
int ret=0, card_quene_num;
struct card_queue_list *cq_node_current;
struct card_queue_list *cq_node_prev = NULL;
if (host->parent->dma_mask && *host->parent->dma_mask)
limit = *host->parent->dma_mask;
cq->card = card;
cq->queue = blk_init_queue(card_request, lock);
if (!cq->queue)
return -ENOMEM;
blk_queue_prep_rq(cq->queue, card_prep_request);
card_init_bounce_buf(cq, card);
if(!cq->bounce_buf){
blk_queue_bounce_limit(cq->queue, limit);
blk_queue_max_hw_sectors(cq->queue, host->max_sectors);
//blk_queue_max_hw_phys_segments(cq->queue, host->max_phys_segs);
blk_queue_max_segments(cq->queue, host->max_hw_segs);
blk_queue_max_segment_size(cq->queue, host->max_seg_size);
cq->queue->queuedata = cq;
cq->req = NULL;
cq->sg = kmalloc(sizeof(struct scatterlist) * host->max_phys_segs, GFP_KERNEL);
if (!cq->sg) {
ret = -ENOMEM;
blk_cleanup_queue(cq->queue);
return ret;
}
}
if (card_queue_head == NULL)
{
card_queue_head = kmalloc(sizeof(struct card_queue_list), GFP_KERNEL);
if (card_queue_head == NULL)
{
ret = -ENOMEM;
kfree(card_queue_head);
card_queue_head = NULL;
return ret;
}
card_queue_head->cq = cq;
card_queue_head->cq_num = 0;
card_queue_head->cq_flag = 0;
card_queue_head->cq_next = NULL;
init_completion(&card_thread_complete);
init_waitqueue_head(&card_thread_wq);
init_MUTEX(&card_thread_sem);
host->queue_task = kthread_run(card_queue_thread, cq, "card_queue");
if (host->queue_task)
{
wait_for_completion(&card_thread_complete);
init_completion(&card_thread_complete);
ret = 0;
return ret;
}
}
else
{
card_quene_num = 0;
cq_node_current = card_queue_head;
do
{
card_quene_num = cq_node_current->cq_num;
cq_node_prev = cq_node_current;
cq_node_current = cq_node_current->cq_next;
} while (cq_node_current != NULL);
cq_node_current = kmalloc(sizeof(struct card_queue_list), GFP_KERNEL);
if (cq_node_current == NULL)
{
ret = -ENOMEM;
kfree(cq_node_current);
cq_node_current = NULL;
return ret;
}
cq_node_prev->cq_next = cq_node_current;
cq_node_current->cq = cq;
cq_node_current->cq_next = NULL;
cq_node_current->cq_num = (++card_quene_num);
cq_node_current->cq_flag = 0;
ret = 0;
return ret;
}
return ret;
}
/*
* Allocate and initialise a blank device with a given minor.
*/
static struct mapped_device *alloc_dev(unsigned int minor, int persistent)
{
int r;
struct mapped_device *md = kmalloc(sizeof(*md), GFP_KERNEL);
if (!md) {
DMWARN("unable to allocate device, out of memory.");
return NULL;
}
/* get a minor number for the dev */
r = persistent ? specific_minor(md, minor) : next_free_minor(md, &minor);
if (r < 0)
goto bad1;
memset(md, 0, sizeof(*md));
init_rwsem(&md->io_lock);
init_MUTEX(&md->suspend_lock);
rwlock_init(&md->map_lock);
atomic_set(&md->holders, 1);
atomic_set(&md->event_nr, 0);
md->queue = blk_alloc_queue(GFP_KERNEL);
if (!md->queue)
goto bad1;
md->queue->queuedata = md;
md->queue->backing_dev_info.congested_fn = dm_any_congested;
md->queue->backing_dev_info.congested_data = md;
blk_queue_make_request(md->queue, dm_request);
blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
md->queue->unplug_fn = dm_unplug_all;
md->queue->issue_flush_fn = dm_flush_all;
md->io_pool = mempool_create(MIN_IOS, mempool_alloc_slab,
mempool_free_slab, _io_cache);
if (!md->io_pool)
goto bad2;
md->tio_pool = mempool_create(MIN_IOS, mempool_alloc_slab,
mempool_free_slab, _tio_cache);
if (!md->tio_pool)
goto bad3;
md->disk = alloc_disk(1);
if (!md->disk)
goto bad4;
md->disk->major = _major;
md->disk->first_minor = minor;
md->disk->fops = &dm_blk_dops;
md->disk->queue = md->queue;
md->disk->private_data = md;
sprintf(md->disk->disk_name, "dm-%d", minor);
add_disk(md->disk);
atomic_set(&md->pending, 0);
init_waitqueue_head(&md->wait);
init_waitqueue_head(&md->eventq);
return md;
bad4:
mempool_destroy(md->tio_pool);
bad3:
mempool_destroy(md->io_pool);
bad2:
blk_put_queue(md->queue);
free_minor(minor);
bad1:
kfree(md);
return NULL;
}
/*
* Generic MMC request handler. This is called for any queue on a
* particular host. When the host is not busy, we look for a request
* on any queue on this host, and attempt to issue it. This may
* not be the queue we were asked to process.
*/
static void mmc_request(struct request_queue *q)
{
struct mmc_queue *mq = q->queuedata;
struct request *req;
int ret;
#if 0
if (!mq) {
#else
//插着USB线(充电姿态)，拔插卡，有偶尔死机现象。出现mq->thread为空的现象;modifyed by xbw
if (!mq ||!mq->thread) {
#endif
printk(KERN_ERR "MMC: killing requests for dead queue\n");
while ((req = elv_next_request(q)) != NULL) {
do {
ret = __blk_end_request(req, -EIO,
blk_rq_cur_bytes(req));
} while (ret);
}
return;
}
if (!mq->req)
wake_up_process(mq->thread);
}
/**
* mmc_init_queue - initialise a queue structure.
* @mq: mmc queue
* @card: mmc card to attach this queue
* @lock: queue lock
*
* Initialise a MMC card request queue.
*/
int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card, spinlock_t *lock)
{
struct mmc_host *host = card->host;
u64 limit = BLK_BOUNCE_ANY ; // BLK_BOUNCE_HIGH;
int ret;
if (mmc_dev(host)->dma_mask && *mmc_dev(host)->dma_mask)
limit = *mmc_dev(host)->dma_mask;
mq->card = card;
mq->queue = blk_init_queue(mmc_request, lock);
if (!mq->queue)
return -ENOMEM;
mq->queue->queuedata = mq;
mq->req = NULL;
blk_queue_prep_rq(mq->queue, mmc_prep_request);
#ifdef CONFIG_MMC_BLOCK_BOUNCE
if (host->max_hw_segs == 1) {
unsigned int bouncesz;
bouncesz = MMC_QUEUE_BOUNCESZ;
if (bouncesz > host->max_req_size)
bouncesz = host->max_req_size;
if (bouncesz > host->max_seg_size)
bouncesz = host->max_seg_size;
mq->bounce_buf = kmalloc(bouncesz, GFP_KERNEL);
if (!mq->bounce_buf) {
printk(KERN_WARNING "%s: unable to allocate "
"bounce buffer\n", mmc_card_name(card));
} else {
blk_queue_bounce_limit(mq->queue, BLK_BOUNCE_HIGH);
blk_queue_max_sectors(mq->queue, bouncesz / 512);
blk_queue_max_phys_segments(mq->queue, bouncesz / 512);
blk_queue_max_hw_segments(mq->queue, bouncesz / 512);
blk_queue_max_segment_size(mq->queue, bouncesz);
mq->sg = kmalloc(sizeof(struct scatterlist),
GFP_KERNEL);
if (!mq->sg) {
ret = -ENOMEM;
goto cleanup_queue;
}
sg_init_table(mq->sg, 1);
mq->bounce_sg = kmalloc(sizeof(struct scatterlist) *
bouncesz / 512, GFP_KERNEL);
if (!mq->bounce_sg) {
ret = -ENOMEM;
goto cleanup_queue;
}
sg_init_table(mq->bounce_sg, bouncesz / 512);
}
}
#endif
if (!mq->bounce_buf) {
blk_queue_bounce_limit(mq->queue, limit);
blk_queue_max_sectors(mq->queue, host->max_req_size / 512);
blk_queue_max_phys_segments(mq->queue, host->max_phys_segs);
blk_queue_max_hw_segments(mq->queue, host->max_hw_segs);
blk_queue_max_segment_size(mq->queue, host->max_seg_size);
mq->sg = kmalloc(sizeof(struct scatterlist) *
host->max_phys_segs, GFP_KERNEL);
if (!mq->sg) {
ret = -ENOMEM;
goto cleanup_queue;
}
sg_init_table(mq->sg, host->max_phys_segs);
}
init_MUTEX(&mq->thread_sem);
mq->thread = kthread_run(mmc_queue_thread, mq, "mmcqd");
if (IS_ERR(mq->thread)) {
ret = PTR_ERR(mq->thread);
goto free_bounce_sg;
}
return 0;
free_bounce_sg:
if (mq->bounce_sg)
kfree(mq->bounce_sg);
mq->bounce_sg = NULL;
cleanup_queue:
if (mq->sg)
kfree(mq->sg);
mq->sg = NULL;
if (mq->bounce_buf)
kfree(mq->bounce_buf);
mq->bounce_buf = NULL;
blk_cleanup_queue(mq->queue);
return ret;
}
void mmc_cleanup_queue(struct mmc_queue *mq)
{
struct request_queue *q = mq->queue;
unsigned long flags;
/* Mark that we should start throwing out stragglers */
spin_lock_irqsave(q->queue_lock, flags);
q->queuedata = NULL;
spin_unlock_irqrestore(q->queue_lock, flags);
/* Make sure the queue isn't suspended, as that will deadlock */
mmc_queue_resume(mq);
/* Then terminate our worker thread */
kthread_stop(mq->thread);
if (mq->bounce_sg)
kfree(mq->bounce_sg);
mq->bounce_sg = NULL;
kfree(mq->sg);
mq->sg = NULL;
if (mq->bounce_buf)
kfree(mq->bounce_buf);
mq->bounce_buf = NULL;
blk_cleanup_queue(mq->queue);
mq->card = NULL;
}
EXPORT_SYMBOL(mmc_cleanup_queue);
/**
* mmc_queue_suspend - suspend a MMC request queue
* @mq: MMC queue to suspend
*
* Stop the block request queue, and wait for our thread to
* complete any outstanding requests. This ensures that we
* won't suspend while a request is being processed.
*/
void mmc_queue_suspend(struct mmc_queue *mq)
{
struct request_queue *q = mq->queue;
unsigned long flags;
if (!(mq->flags & MMC_QUEUE_SUSPENDED)) {
mq->flags |= MMC_QUEUE_SUSPENDED;
spin_lock_irqsave(q->queue_lock, flags);
blk_stop_queue(q);
spin_unlock_irqrestore(q->queue_lock, flags);
down(&mq->thread_sem);
}
}
/**
* mmc_queue_resume - resume a previously suspended MMC request queue
* @mq: MMC queue to resume
*/
void mmc_queue_resume(struct mmc_queue *mq)
{
struct request_queue *q = mq->queue;
unsigned long flags;
if (mq->flags & MMC_QUEUE_SUSPENDED) {
mq->flags &= ~MMC_QUEUE_SUSPENDED;
up(&mq->thread_sem);
spin_lock_irqsave(q->queue_lock, flags);
blk_start_queue(q);
spin_unlock_irqrestore(q->queue_lock, flags);
}
}
static void copy_sg(struct scatterlist *dst, unsigned int dst_len,
struct scatterlist *src, unsigned int src_len)
{
unsigned int chunk;
char *dst_buf, *src_buf;
unsigned int dst_size, src_size;
dst_buf = NULL;
src_buf = NULL;
dst_size = 0;
src_size = 0;
while (src_len) {
BUG_ON(dst_len == 0);
if (dst_size == 0) {
dst_buf = sg_virt(dst);
dst_size = dst->length;
}
if (src_size == 0) {
src_buf = sg_virt(src);
src_size = src->length;
}
chunk = min(dst_size, src_size);
memcpy(dst_buf, src_buf, chunk);
dst_buf += chunk;
src_buf += chunk;
dst_size -= chunk;
src_size -= chunk;
if (dst_size == 0) {
dst++;
dst_len--;
}
if (src_size == 0) {
src++;
src_len--;
}
}
}
/**
* iscsi_sw_tcp_xmit_segment - transmit segment
* @tcp_conn: the iSCSI TCP connection
* @segment: the buffer to transmnit
*
* This function transmits as much of the buffer as
* the network layer will accept, and returns the number of
* bytes transmitted.
*
* If CRC hashing is enabled, the function will compute the
* hash as it goes. When the entire segment has been transmitted,
* it will retrieve the hash value and send it as well.
*/
static int iscsi_sw_tcp_xmit_segment(struct iscsi_tcp_conn *tcp_conn,
struct iscsi_segment *segment)
{
struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
struct socket *sk = tcp_sw_conn->sock;
unsigned int copied = 0;
int r = 0;
while (!iscsi_tcp_segment_done(tcp_conn, segment, 0, r)) {
struct scatterlist *sg;
unsigned int offset, copy;
int flags = 0;
r = 0;
offset = segment->copied;
copy = segment->size - offset;
if (segment->total_copied + segment->size < segment->total_size)
flags |= MSG_MORE;
/* Use sendpage if we can; else fall back to sendmsg */
if (!segment->data) {
sg = segment->sg;
offset += segment->sg_offset + sg->offset;
r = tcp_sw_conn->sendpage(sk, sg_page(sg), offset,
copy, flags);
} else {
struct msghdr msg = { .msg_flags = flags };
struct kvec iov = {
.iov_base = segment->data + offset,
.iov_len = copy
};
r = kernel_sendmsg(sk, &msg, &iov, 1, copy);
}
if (r < 0) {
iscsi_tcp_segment_unmap(segment);
if (copied || r == -EAGAIN)
break;
return r;
}
copied += r;
}
return copied;
}
/**
* iscsi_sw_tcp_xmit - TCP transmit
**/
static int iscsi_sw_tcp_xmit(struct iscsi_conn *conn)
{
struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
struct iscsi_segment *segment = &tcp_sw_conn->out.segment;
unsigned int consumed = 0;
int rc = 0;
while (1) {
rc = iscsi_sw_tcp_xmit_segment(tcp_conn, segment);
if (rc < 0) {
rc = ISCSI_ERR_XMIT_FAILED;
goto error;
}
if (rc == 0)
break;
consumed += rc;
if (segment->total_copied >= segment->total_size) {
if (segment->done != NULL) {
rc = segment->done(tcp_conn, segment);
if (rc != 0)
goto error;
}
}
}
debug_tcp("xmit %d bytes\n", consumed);
conn->txdata_octets += consumed;
return consumed;
error:
/* Transmit error. We could initiate error recovery
* here. */
debug_tcp("Error sending PDU, errno=%d\n", rc);
iscsi_conn_failure(conn, rc);
return -EIO;
}
/**
* iscsi_tcp_xmit_qlen - return the number of bytes queued for xmit
*/
static inline int iscsi_sw_tcp_xmit_qlen(struct iscsi_conn *conn)
{
struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
struct iscsi_segment *segment = &tcp_sw_conn->out.segment;
return segment->total_copied - segment->total_size;
}
static int iscsi_sw_tcp_pdu_xmit(struct iscsi_task *task)
{
struct iscsi_conn *conn = task->conn;
int rc;
while (iscsi_sw_tcp_xmit_qlen(conn)) {
rc = iscsi_sw_tcp_xmit(conn);
if (rc == 0)
return -EAGAIN;
if (rc < 0)
return rc;
}
return 0;
}
/*
* This is called when we're done sending the header.
* Simply copy the data_segment to the send segment, and return.
*/
static int iscsi_sw_tcp_send_hdr_done(struct iscsi_tcp_conn *tcp_conn,
struct iscsi_segment *segment)
{
struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
tcp_sw_conn->out.segment = tcp_sw_conn->out.data_segment;
debug_tcp("Header done. Next segment size %u total_size %u\n",
tcp_sw_conn->out.segment.size,
tcp_sw_conn->out.segment.total_size);
return 0;
}
static void iscsi_sw_tcp_send_hdr_prep(struct iscsi_conn *conn, void *hdr,
size_t hdrlen)
{
struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
debug_tcp("%s(%p%s)\n", __func__, tcp_conn,
conn->hdrdgst_en? ", digest enabled" : "");
/* Clear the data segment - needs to be filled in by the
* caller using iscsi_tcp_send_data_prep() */
memset(&tcp_sw_conn->out.data_segment, 0,
sizeof(struct iscsi_segment));
/* If header digest is enabled, compute the CRC and
* place the digest into the same buffer. We make
* sure that both iscsi_tcp_task and mtask have
* sufficient room.
*/
if (conn->hdrdgst_en) {
iscsi_tcp_dgst_header(&tcp_sw_conn->tx_hash, hdr, hdrlen,
hdr + hdrlen);
hdrlen += ISCSI_DIGEST_SIZE;
}
/* Remember header pointer for later, when we need
* to decide whether there's a payload to go along
* with the header. */
tcp_sw_conn->out.hdr = hdr;
iscsi_segment_init_linear(&tcp_sw_conn->out.segment, hdr, hdrlen,
iscsi_sw_tcp_send_hdr_done, NULL);
}
/*
* Prepare the send buffer for the payload data.
* Padding and checksumming will all be taken care
* of by the iscsi_segment routines.
*/
static int
iscsi_sw_tcp_send_data_prep(struct iscsi_conn *conn, struct scatterlist *sg,
unsigned int count, unsigned int offset,
unsigned int len)
{
struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
struct hash_desc *tx_hash = NULL;
unsigned int hdr_spec_len;
debug_tcp("%s(%p, offset=%d, datalen=%d%s)\n", __func__,
tcp_conn, offset, len,
conn->datadgst_en? ", digest enabled" : "");
/* Make sure the datalen matches what the caller
said he would send. */
hdr_spec_len = ntoh24(tcp_sw_conn->out.hdr->dlength);
WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len));
if (conn->datadgst_en)
tx_hash = &tcp_sw_conn->tx_hash;
return iscsi_segment_seek_sg(&tcp_sw_conn->out.data_segment,
sg, count, offset, len,
NULL, tx_hash);
}
static void
iscsi_sw_tcp_send_linear_data_prep(struct iscsi_conn *conn, void *data,
size_t len)
{
struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
struct hash_desc *tx_hash = NULL;
unsigned int hdr_spec_len;
debug_tcp("%s(%p, datalen=%d%s)\n", __func__, tcp_conn, len,
conn->datadgst_en? ", digest enabled" : "");
/* Make sure the datalen matches what the caller
said he would send. */
hdr_spec_len = ntoh24(tcp_sw_conn->out.hdr->dlength);
WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len));
if (conn->datadgst_en)
tx_hash = &tcp_sw_conn->tx_hash;
iscsi_segment_init_linear(&tcp_sw_conn->out.data_segment,
data, len, NULL, tx_hash);
}
static int iscsi_sw_tcp_pdu_init(struct iscsi_task *task,
unsigned int offset, unsigned int count)
{
struct iscsi_conn *conn = task->conn;
int err = 0;
iscsi_sw_tcp_send_hdr_prep(conn, task->hdr, task->hdr_len);
if (!count)
return 0;
if (!task->sc)
iscsi_sw_tcp_send_linear_data_prep(conn, task->data, count);
else {
struct scsi_data_buffer *sdb = scsi_out(task->sc);
err = iscsi_sw_tcp_send_data_prep(conn, sdb->table.sgl,
sdb->table.nents, offset,
count);
}
if (err) {
iscsi_conn_failure(conn, err);
return -EIO;
}
return 0;
}
static int iscsi_sw_tcp_pdu_alloc(struct iscsi_task *task, uint8_t opcode)
{
struct iscsi_tcp_task *tcp_task = task->dd_data;
task->hdr = task->dd_data + sizeof(*tcp_task);
task->hdr_max = sizeof(struct iscsi_sw_tcp_hdrbuf) - ISCSI_DIGEST_SIZE;
return 0;
}
static struct iscsi_cls_conn *
iscsi_sw_tcp_conn_create(struct iscsi_cls_session *cls_session,
uint32_t conn_idx)
{
struct iscsi_conn *conn;
struct iscsi_cls_conn *cls_conn;
struct iscsi_tcp_conn *tcp_conn;
struct iscsi_sw_tcp_conn *tcp_sw_conn;
cls_conn = iscsi_tcp_conn_setup(cls_session, sizeof(*tcp_sw_conn),
conn_idx);
if (!cls_conn)
return NULL;
conn = cls_conn->dd_data;
tcp_conn = conn->dd_data;
tcp_sw_conn = tcp_conn->dd_data;
tcp_sw_conn->tx_hash.tfm = crypto_alloc_hash("crc32c", 0,
CRYPTO_ALG_ASYNC);
tcp_sw_conn->tx_hash.flags = 0;
if (IS_ERR(tcp_sw_conn->tx_hash.tfm))
goto free_conn;
tcp_sw_conn->rx_hash.tfm = crypto_alloc_hash("crc32c", 0,
CRYPTO_ALG_ASYNC);
tcp_sw_conn->rx_hash.flags = 0;
if (IS_ERR(tcp_sw_conn->rx_hash.tfm))
goto free_tx_tfm;
tcp_conn->rx_hash = &tcp_sw_conn->rx_hash;
return cls_conn;
free_tx_tfm:
crypto_free_hash(tcp_sw_conn->tx_hash.tfm);
free_conn:
iscsi_conn_printk(KERN_ERR, conn,
"Could not create connection due to crc32c "
"loading error. Make sure the crc32c "
"module is built as a module or into the "
"kernel\n");
iscsi_tcp_conn_teardown(cls_conn);
return NULL;
}
static void iscsi_sw_tcp_release_conn(struct iscsi_conn *conn)
{
struct iscsi_session *session = conn->session;
struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
struct socket *sock = tcp_sw_conn->sock;
if (!sock)
return;
sock_hold(sock->sk);
iscsi_sw_tcp_conn_restore_callbacks(tcp_sw_conn);
sock_put(sock->sk);
spin_lock_bh(&session->lock);
tcp_sw_conn->sock = NULL;
spin_unlock_bh(&session->lock);
sockfd_put(sock);
}
static void iscsi_sw_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn)
{
struct iscsi_conn *conn = cls_conn->dd_data;
struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
iscsi_sw_tcp_release_conn(conn);
if (tcp_sw_conn->tx_hash.tfm)
crypto_free_hash(tcp_sw_conn->tx_hash.tfm);
if (tcp_sw_conn->rx_hash.tfm)
crypto_free_hash(tcp_sw_conn->rx_hash.tfm);
iscsi_tcp_conn_teardown(cls_conn);
}
static void iscsi_sw_tcp_conn_stop(struct iscsi_cls_conn *cls_conn, int flag)
{
struct iscsi_conn *conn = cls_conn->dd_data;
struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
/* userspace may have goofed up and not bound us */
if (!tcp_sw_conn->sock)
return;
/*
* Make sure our recv side is stopped.
* Older tools called conn stop before ep_disconnect
* so IO could still be coming in.
*/
write_lock_bh(&tcp_sw_conn->sock->sk->sk_callback_lock);
set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_rx);
write_unlock_bh(&tcp_sw_conn->sock->sk->sk_callback_lock);
iscsi_conn_stop(cls_conn, flag);
iscsi_sw_tcp_release_conn(conn);
}
static int iscsi_sw_tcp_get_addr(struct iscsi_conn *conn, struct socket *sock,
char *buf, int *port,
int (*getname)(struct socket *,
struct sockaddr *,
int *addrlen))
{
struct sockaddr_storage *addr;
struct sockaddr_in6 *sin6;
struct sockaddr_in *sin;
int rc = 0, len;
addr = kmalloc(sizeof(*addr), GFP_KERNEL);
if (!addr)
return -ENOMEM;
if (getname(sock, (struct sockaddr *) addr, &len)) {
rc = -ENODEV;
goto free_addr;
}
switch (addr->ss_family) {
case AF_INET:
sin = (struct sockaddr_in *)addr;
spin_lock_bh(&conn->session->lock);
sprintf(buf, "%pI4", &sin->sin_addr.s_addr);
*port = be16_to_cpu(sin->sin_port);
spin_unlock_bh(&conn->session->lock);
break;
case AF_INET6:
sin6 = (struct sockaddr_in6 *)addr;
spin_lock_bh(&conn->session->lock);
sprintf(buf, "%pI6", &sin6->sin6_addr);
*port = be16_to_cpu(sin6->sin6_port);
spin_unlock_bh(&conn->session->lock);
break;
}
free_addr:
kfree(addr);
return rc;
}
static int
iscsi_sw_tcp_conn_bind(struct iscsi_cls_session *cls_session,
struct iscsi_cls_conn *cls_conn, uint64_t transport_eph,
int is_leading)
{
struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
struct iscsi_host *ihost = shost_priv(shost);
struct iscsi_conn *conn = cls_conn->dd_data;
struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
struct sock *sk;
struct socket *sock;
int err;
/* lookup for existing socket */
sock = sockfd_lookup((int)transport_eph, &err);
if (!sock) {
iscsi_conn_printk(KERN_ERR, conn,
"sockfd_lookup failed %d\n", err);
return -EEXIST;
}
/*
* copy these values now because if we drop the session
* userspace may still want to query the values since we will
* be using them for the reconnect
*/
err = iscsi_sw_tcp_get_addr(conn, sock, conn->portal_address,
&conn->portal_port, kernel_getpeername);
if (err)
goto free_socket;
err = iscsi_sw_tcp_get_addr(conn, sock, ihost->local_address,
&ihost->local_port, kernel_getsockname);
if (err)
goto free_socket;
err = iscsi_conn_bind(cls_session, cls_conn, is_leading);
if (err)
goto free_socket;
/* bind iSCSI connection and socket */
tcp_sw_conn->sock = sock;
/* setup Socket parameters */
sk = sock->sk;
sk->sk_reuse = 1;
sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */
sk->sk_allocation = GFP_ATOMIC;
iscsi_sw_tcp_conn_set_callbacks(conn);
tcp_sw_conn->sendpage = tcp_sw_conn->sock->ops->sendpage;
/*
* set receive state machine into initial state
*/
iscsi_tcp_hdr_recv_prep(tcp_conn);
return 0;
free_socket:
sockfd_put(sock);
return err;
}
static int iscsi_sw_tcp_conn_set_param(struct iscsi_cls_conn *cls_conn,
enum iscsi_param param, char *buf,
int buflen)
{
struct iscsi_conn *conn = cls_conn->dd_data;
struct iscsi_session *session = conn->session;
struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
int value;
switch(param) {
case ISCSI_PARAM_HDRDGST_EN:
iscsi_set_param(cls_conn, param, buf, buflen);
break;
case ISCSI_PARAM_DATADGST_EN:
iscsi_set_param(cls_conn, param, buf, buflen);
tcp_sw_conn->sendpage = conn->datadgst_en ?
sock_no_sendpage : tcp_sw_conn->sock->ops->sendpage;
break;
case ISCSI_PARAM_MAX_R2T:
sscanf(buf, "%d", &value);
if (value <= 0 || !is_power_of_2(value))
return -EINVAL;
if (session->max_r2t == value)
break;
iscsi_tcp_r2tpool_free(session);
iscsi_set_param(cls_conn, param, buf, buflen);
if (iscsi_tcp_r2tpool_alloc(session))
return -ENOMEM;
break;
default:
return iscsi_set_param(cls_conn, param, buf, buflen);
}
return 0;
}
static int iscsi_sw_tcp_conn_get_param(struct iscsi_cls_conn *cls_conn,
enum iscsi_param param, char *buf)
{
struct iscsi_conn *conn = cls_conn->dd_data;
int len;
switch(param) {
case ISCSI_PARAM_CONN_PORT:
spin_lock_bh(&conn->session->lock);
len = sprintf(buf, "%hu\n", conn->portal_port);
spin_unlock_bh(&conn->session->lock);
break;
case ISCSI_PARAM_CONN_ADDRESS:
spin_lock_bh(&conn->session->lock);
len = sprintf(buf, "%s\n", conn->portal_address);
spin_unlock_bh(&conn->session->lock);
break;
default:
return iscsi_conn_get_param(cls_conn, param, buf);
}
return len;
}
static void
iscsi_sw_tcp_conn_get_stats(struct iscsi_cls_conn *cls_conn,
struct iscsi_stats *stats)
{
struct iscsi_conn *conn = cls_conn->dd_data;
struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
stats->custom_length = 3;
strcpy(stats->custom[0].desc, "tx_sendpage_failures");
stats->custom[0].value = tcp_sw_conn->sendpage_failures_cnt;
strcpy(stats->custom[1].desc, "rx_discontiguous_hdr");
stats->custom[1].value = tcp_sw_conn->discontiguous_hdr_cnt;
strcpy(stats->custom[2].desc, "eh_abort_cnt");
stats->custom[2].value = conn->eh_abort_cnt;
iscsi_tcp_conn_get_stats(cls_conn, stats);
}
static struct iscsi_cls_session *
iscsi_sw_tcp_session_create(struct iscsi_endpoint *ep, uint16_t cmds_max,
uint16_t qdepth, uint32_t initial_cmdsn,
uint32_t *hostno)
{
struct iscsi_cls_session *cls_session;
struct iscsi_session *session;
struct Scsi_Host *shost;
if (ep) {
printk(KERN_ERR "iscsi_tcp: invalid ep %p.\n", ep);
return NULL;
}
shost = iscsi_host_alloc(&iscsi_sw_tcp_sht, 0, qdepth);
if (!shost)
return NULL;
shost->transportt = iscsi_sw_tcp_scsi_transport;
shost->max_lun = iscsi_max_lun;
shost->max_id = 0;
shost->max_channel = 0;
shost->max_cmd_len = SCSI_MAX_VARLEN_CDB_SIZE;
if (iscsi_host_add(shost, NULL))
goto free_host;
*hostno = shost->host_no;
cls_session = iscsi_session_setup(&iscsi_sw_tcp_transport, shost,
cmds_max,
sizeof(struct iscsi_tcp_task) +
sizeof(struct iscsi_sw_tcp_hdrbuf),
initial_cmdsn, 0);
if (!cls_session)
goto remove_host;
session = cls_session->dd_data;
shost->can_queue = session->scsi_cmds_max;
if (iscsi_tcp_r2tpool_alloc(session))
goto remove_session;
return cls_session;
remove_session:
iscsi_session_teardown(cls_session);
remove_host:
iscsi_host_remove(shost);
free_host:
iscsi_host_free(shost);
return NULL;
}
static void iscsi_sw_tcp_session_destroy(struct iscsi_cls_session *cls_session)
{
struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
iscsi_tcp_r2tpool_free(cls_session->dd_data);
iscsi_session_teardown(cls_session);
iscsi_host_remove(shost);
iscsi_host_free(shost);
}
static int iscsi_sw_tcp_slave_configure(struct scsi_device *sdev)
{
blk_queue_bounce_limit(sdev->request_queue, BLK_BOUNCE_ANY);
blk_queue_dma_alignment(sdev->request_queue, 0);
return 0;
}
/**
* mmc_init_queue - initialise a queue structure.
* @mq: mmc queue
* @card: mmc card to attach this queue
* @lock: queue lock
* @subname: partition subname
*
* Initialise a MMC card request queue.
*/
int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card,
spinlock_t *lock, const char *subname)
{
struct mmc_host *host = card->host;
u64 limit = BLK_BOUNCE_HIGH;
bool bounce = false;
int ret = -ENOMEM;
if (mmc_dev(host)->dma_mask && *mmc_dev(host)->dma_mask)
limit = (u64)dma_max_pfn(mmc_dev(host)) << PAGE_SHIFT;
mq->card = card;
mq->queue = blk_init_queue(mmc_request_fn, lock);
if (!mq->queue)
return -ENOMEM;
mq->qdepth = 2;
mq->mqrq = kcalloc(mq->qdepth, sizeof(struct mmc_queue_req),
GFP_KERNEL);
if (!mq->mqrq)
goto blk_cleanup;
mq->mqrq_cur = &mq->mqrq[0];
mq->mqrq_prev = &mq->mqrq[1];
mq->queue->queuedata = mq;
blk_queue_prep_rq(mq->queue, mmc_prep_request);
queue_flag_set_unlocked(QUEUE_FLAG_NONROT, mq->queue);
queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, mq->queue);
if (mmc_can_erase(card))
mmc_queue_setup_discard(mq->queue, card);
#ifdef CONFIG_MMC_BLOCK_BOUNCE
if (host->max_segs == 1) {
unsigned int bouncesz;
bouncesz = MMC_QUEUE_BOUNCESZ;
if (bouncesz > host->max_req_size)
bouncesz = host->max_req_size;
if (bouncesz > host->max_seg_size)
bouncesz = host->max_seg_size;
if (bouncesz > (host->max_blk_count * 512))
bouncesz = host->max_blk_count * 512;
if (bouncesz > 512 &&
mmc_queue_alloc_bounce_bufs(mq, bouncesz)) {
blk_queue_bounce_limit(mq->queue, BLK_BOUNCE_ANY);
blk_queue_max_hw_sectors(mq->queue, bouncesz / 512);
blk_queue_max_segments(mq->queue, bouncesz / 512);
blk_queue_max_segment_size(mq->queue, bouncesz);
ret = mmc_queue_alloc_bounce_sgs(mq, bouncesz);
if (ret)
goto cleanup_queue;
bounce = true;
}
}
#endif
if (!bounce) {
blk_queue_bounce_limit(mq->queue, limit);
blk_queue_max_hw_sectors(mq->queue,
min(host->max_blk_count, host->max_req_size / 512));
blk_queue_max_segments(mq->queue, host->max_segs);
blk_queue_max_segment_size(mq->queue, host->max_seg_size);
ret = mmc_queue_alloc_sgs(mq, host->max_segs);
if (ret)
goto cleanup_queue;
}
sema_init(&mq->thread_sem, 1);
mq->thread = kthread_run(mmc_queue_thread, mq, "mmcqd/%d%s",
host->index, subname ? subname : "");
if (IS_ERR(mq->thread)) {
ret = PTR_ERR(mq->thread);
goto cleanup_queue;
}
return 0;
cleanup_queue:
mmc_queue_reqs_free_bufs(mq);
kfree(mq->mqrq);
mq->mqrq = NULL;
blk_cleanup:
blk_cleanup_queue(mq->queue);
return ret;
}
static struct pmem_device *pmem_alloc(struct device *dev, struct resource *res)
{
struct pmem_device *pmem;
struct gendisk *disk;
int idx, err;
err = -ENOMEM;
pmem = kzalloc(sizeof(*pmem), GFP_KERNEL);
if (!pmem)
goto out;
pmem->phys_addr = res->start;
pmem->size = resource_size(res);
err = -EINVAL;
if (!request_mem_region(pmem->phys_addr, pmem->size, "pmem")) {
dev_warn(dev, "could not reserve region [0x%pa:0x%zx]\n", &pmem->phys_addr, pmem->size);
goto out_free_dev;
}
/*
* Map the memory as write-through, as we can't write back the contents
* of the CPU caches in case of a crash.
*/
err = -ENOMEM;
pmem->virt_addr = ioremap_wt(pmem->phys_addr, pmem->size);
if (!pmem->virt_addr)
goto out_release_region;
pmem->pmem_queue = blk_alloc_queue(GFP_KERNEL);
if (!pmem->pmem_queue)
goto out_unmap;
blk_queue_make_request(pmem->pmem_queue, pmem_make_request);
blk_queue_max_hw_sectors(pmem->pmem_queue, 1024);
blk_queue_bounce_limit(pmem->pmem_queue, BLK_BOUNCE_ANY);
disk = alloc_disk(PMEM_MINORS);
if (!disk)
goto out_free_queue;
idx = atomic_inc_return(&pmem_index) - 1;
disk->major = pmem_major;
disk->first_minor = PMEM_MINORS * idx;
disk->fops = &pmem_fops;
disk->private_data = pmem;
disk->queue = pmem->pmem_queue;
disk->flags = GENHD_FL_EXT_DEVT;
sprintf(disk->disk_name, "pmem%d", idx);
disk->driverfs_dev = dev;
set_capacity(disk, pmem->size >> 9);
pmem->pmem_disk = disk;
add_disk(disk);
return pmem;
out_free_queue:
blk_cleanup_queue(pmem->pmem_queue);
out_unmap:
iounmap(pmem->virt_addr);
out_release_region:
release_mem_region(pmem->phys_addr, pmem->size);
out_free_dev:
kfree(pmem);
out:
return ERR_PTR(err);
}
/**
* mmc_init_queue - initialise a queue structure.
* @mq: mmc queue
* @card: mmc card to attach this queue
* @lock: queue lock
* @subname: partition subname
*
* Initialise a MMC card request queue.
*/
int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card,
spinlock_t *lock, const char *subname)
{
struct mmc_host *host = card->host;
u64 limit = BLK_BOUNCE_HIGH;
int ret;
struct mmc_queue_req *mqrq_cur = &mq->mqrq[0];
struct mmc_queue_req *mqrq_prev = &mq->mqrq[1];
if (mmc_dev(host)->dma_mask && *mmc_dev(host)->dma_mask)
limit = *mmc_dev(host)->dma_mask;
mq->card = card;
mq->queue = blk_init_queue(mmc_request, lock);
if (!mq->queue)
return -ENOMEM;
memset(&mq->mqrq_cur, 0, sizeof(mq->mqrq_cur));
memset(&mq->mqrq_prev, 0, sizeof(mq->mqrq_prev));
mq->mqrq_cur = mqrq_cur;
mq->mqrq_prev = mqrq_prev;
mq->queue->queuedata = mq;
blk_queue_prep_rq(mq->queue, mmc_prep_request);
queue_flag_set_unlocked(QUEUE_FLAG_NONROT, mq->queue);
if (mmc_can_erase(card))
mmc_queue_setup_discard(mq->queue, card);
#ifdef CONFIG_MMC_BLOCK_BOUNCE
if (host->max_segs == 1) {
unsigned int bouncesz;
if(!mmc_card_sd(card))
bouncesz = MMC_QUEUE_BOUNCESZ;
else
bouncesz = MMC_QUEUE_SD_BOUNCESZ;
if (bouncesz > host->max_req_size)
bouncesz = host->max_req_size;
if (bouncesz > host->max_seg_size)
bouncesz = host->max_seg_size;
if (bouncesz > (host->max_blk_count * 512))
bouncesz = host->max_blk_count * 512;
if (bouncesz > 512) {
if(!mmc_card_sd(card))
mqrq_cur->bounce_buf = kmalloc(bouncesz, GFP_KERNEL);
else
mqrq_cur->bounce_buf = mmc_queue_cur_bounce_buf;
if (!mqrq_cur->bounce_buf) {
printk(KERN_WARNING "%s: unable to "
"allocate bounce cur buffer\n",
mmc_card_name(card));
}
if(!mmc_card_sd(card))
mqrq_prev->bounce_buf = kmalloc(bouncesz, GFP_KERNEL);
else
mqrq_prev->bounce_buf = mmc_queue_prev_bounce_buf;
if (!mqrq_prev->bounce_buf) {
printk(KERN_WARNING "%s: unable to "
"allocate bounce prev buffer\n",
mmc_card_name(card));
kfree(mqrq_cur->bounce_buf);
mqrq_cur->bounce_buf = NULL;
}
}
if (mqrq_cur->bounce_buf && mqrq_prev->bounce_buf) {
blk_queue_bounce_limit(mq->queue, BLK_BOUNCE_ANY);
blk_queue_max_hw_sectors(mq->queue, bouncesz / 512);
blk_queue_max_segments(mq->queue, bouncesz / 512);
blk_queue_max_segment_size(mq->queue, bouncesz);
mqrq_cur->sg = mmc_alloc_sg(1, &ret);
if (ret)
goto cleanup_queue;
mqrq_cur->bounce_sg =
mmc_alloc_sg(bouncesz / 512, &ret);
if (ret)
goto cleanup_queue;
mqrq_prev->sg = mmc_alloc_sg(1, &ret);
if (ret)
goto cleanup_queue;
mqrq_prev->bounce_sg =
mmc_alloc_sg(bouncesz / 512, &ret);
if (ret)
goto cleanup_queue;
}
}
#endif
if (!mqrq_cur->bounce_buf && !mqrq_prev->bounce_buf) {
blk_queue_bounce_limit(mq->queue, limit);
blk_queue_max_hw_sectors(mq->queue,
min(host->max_blk_count, host->max_req_size / 512));
blk_queue_max_segments(mq->queue, host->max_segs);
blk_queue_max_segment_size(mq->queue, host->max_seg_size);
mqrq_cur->sg = mmc_alloc_sg(host->max_segs, &ret);
if (ret)
goto cleanup_queue;
mqrq_prev->sg = mmc_alloc_sg(host->max_segs, &ret);
if (ret)
goto cleanup_queue;
}
sema_init(&mq->thread_sem, 1);
mq->thread = kthread_run(mmc_queue_thread, mq, "mmcqd/%d%s",
host->index, subname ? subname : "");
if (IS_ERR(mq->thread)) {
ret = PTR_ERR(mq->thread);
goto free_bounce_sg;
}
return 0;
free_bounce_sg:
kfree(mqrq_cur->bounce_sg);
mqrq_cur->bounce_sg = NULL;
kfree(mqrq_prev->bounce_sg);
mqrq_prev->bounce_sg = NULL;
cleanup_queue:
kfree(mqrq_cur->sg);
mqrq_cur->sg = NULL;
if(!mmc_card_sd(card))
kfree(mqrq_cur->bounce_buf);
mqrq_cur->bounce_buf = NULL;
kfree(mqrq_prev->sg);
mqrq_prev->sg = NULL;
if(!mmc_card_sd(card))
kfree(mqrq_prev->bounce_buf);
mqrq_prev->bounce_buf = NULL;
blk_cleanup_queue(mq->queue);
return ret;
}
/* Create disk on demand. So we won't create lots of disk for un-used devices. */
static struct kobject *tzmem_blk_probe(dev_t dev, int *part, void *data)
{
uint32_t len;
struct gendisk *disk;
struct kobject *kobj;
struct request_queue *queue;
struct tzmem_diskinfo_s *diskInfo;
int ret;
KREE_SESSION_HANDLE session;
#ifdef MTEE_TZMEM_DBG
pr_warn("====> tzmem_blk_probe\n");
#endif
mutex_lock(&tzmem_probe_mutex);
diskInfo = (struct tzmem_diskinfo_s *) &_tzmem_diskInfo[tzmem_poolIndex];
if (diskInfo->disk == NULL) {
disk = alloc_disk(1);
if (!disk)
goto out_info;
queue = blk_init_queue(do_tzmem_blk_request, &tzmem_blk_lock);
if (!queue)
goto out_queue;
blk_queue_max_hw_sectors(queue, 1024);
blk_queue_bounce_limit(queue, BLK_BOUNCE_ANY);
if (_tzmem_get_poolsize(&len))
goto out_init;
disk->major = IO_NODE_MAJOR_TZMEM;
disk->first_minor = MINOR(dev);
disk->fops = &tzmem_blk_fops;
disk->private_data = &_tzmem_diskInfo;
snprintf(disk->disk_name, sizeof(disk->disk_name), "tzmem%d", MINOR(dev));
disk->queue = queue;
set_capacity(disk, len / 512);
add_disk(disk);
ret = KREE_CreateSession(TZ_TA_MEM_UUID, &session);
if (ret != TZ_RESULT_SUCCESS) {
pr_debug(MTEE_TZMEM_TAG
"[%s] _tzmem_get_poolsize: KREE_CreateSession Error = 0x%x\n",
MODULE_NAME, ret);
goto out_init;
}
diskInfo->session = session;
diskInfo->pool_size = len;
diskInfo->disk = disk;
diskInfo->size = len;
}
*part = 0;
kobj = diskInfo ? get_disk(diskInfo->disk) : ERR_PTR(-ENOMEM);
mutex_unlock(&tzmem_probe_mutex);
return kobj;
out_init:
blk_cleanup_queue(queue);
out_queue:
put_disk(disk);
out_info:
mutex_unlock(&tzmem_probe_mutex);
return ERR_PTR(-ENOMEM);
}
/**
* mmc_init_queue - initialise a queue structure.
* @mq: mmc queue
* @card: mmc card to attach this queue
* @lock: queue lock
*
* Initialise a MMC card request queue.
*/
int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card, spinlock_t *lock)
{
struct mmc_host *host = card->host;
u64 limit = BLK_BOUNCE_HIGH;
int ret;
if (mmc_dev(host)->dma_mask && *mmc_dev(host)->dma_mask)
limit = *mmc_dev(host)->dma_mask;
mq->card = card;
mq->queue = blk_init_queue(mmc_request, lock);
if (!mq->queue)
return -ENOMEM;
mq->queue->queuedata = mq;
mq->req = NULL;
blk_queue_prep_rq(mq->queue, mmc_prep_request);
blk_queue_ordered(mq->queue, QUEUE_ORDERED_DRAIN, NULL);
queue_flag_set_unlocked(QUEUE_FLAG_NONROT, mq->queue);
#ifdef CONFIG_MMC_BLOCK_BOUNCE
if (host->max_hw_segs == 1) {
unsigned int bouncesz;
bouncesz = MMC_QUEUE_BOUNCESZ;
if (bouncesz > host->max_req_size)
bouncesz = host->max_req_size;
if (bouncesz > host->max_seg_size)
bouncesz = host->max_seg_size;
if (bouncesz > (host->max_blk_count * 512))
bouncesz = host->max_blk_count * 512;
if (bouncesz > 512) {
mq->bounce_buf = kmalloc(bouncesz, GFP_KERNEL);
if (!mq->bounce_buf) {
printk(KERN_WARNING "%s: unable to "
"allocate bounce buffer\n",
mmc_card_name(card));
}
}
if (mq->bounce_buf) {
blk_queue_bounce_limit(mq->queue, BLK_BOUNCE_ANY);
blk_queue_max_sectors(mq->queue, bouncesz / 512);
blk_queue_max_phys_segments(mq->queue, bouncesz / 512);
blk_queue_max_hw_segments(mq->queue, bouncesz / 512);
blk_queue_max_segment_size(mq->queue, bouncesz);
mq->sg = kmalloc(sizeof(struct scatterlist),
GFP_KERNEL);
if (!mq->sg) {
ret = -ENOMEM;
goto cleanup_queue;
}
sg_init_table(mq->sg, 1);
mq->bounce_sg = kmalloc(sizeof(struct scatterlist) *
bouncesz / 512, GFP_KERNEL);
if (!mq->bounce_sg) {
ret = -ENOMEM;
goto cleanup_queue;
}
sg_init_table(mq->bounce_sg, bouncesz / 512);
}
}
#endif
if (!mq->bounce_buf) {
blk_queue_bounce_limit(mq->queue, limit);
blk_queue_max_sectors(mq->queue,
min(host->max_blk_count, host->max_req_size / 512));
blk_queue_max_phys_segments(mq->queue, host->max_phys_segs);
blk_queue_max_hw_segments(mq->queue, host->max_hw_segs);
blk_queue_max_segment_size(mq->queue, host->max_seg_size);
mq->sg = kmalloc(sizeof(struct scatterlist) *
host->max_phys_segs, GFP_KERNEL);
if (!mq->sg) {
ret = -ENOMEM;
goto cleanup_queue;
}
sg_init_table(mq->sg, host->max_phys_segs);
}
init_MUTEX(&mq->thread_sem);
mq->thread = kthread_run(mmc_queue_thread, mq, "mmcqd");
if (IS_ERR(mq->thread)) {
ret = PTR_ERR(mq->thread);
goto free_bounce_sg;
}
return 0;
free_bounce_sg:
if (mq->bounce_sg)
kfree(mq->bounce_sg);
mq->bounce_sg = NULL;
cleanup_queue:
if (mq->sg)
kfree(mq->sg);
mq->sg = NULL;
if (mq->bounce_buf)
kfree(mq->bounce_buf);
mq->bounce_buf = NULL;
blk_cleanup_queue(mq->queue);
return ret;
}
int td_linux_block_create(struct td_osdev *dev)
{
int rc;
struct request_queue *queue;
unsigned bio_sector_size = dev->block_params.bio_sector_size;
unsigned hw_sector_size = dev->block_params.hw_sector_size;
/* very simple sector size support */
if (!bio_sector_size || bio_sector_size & 511 || bio_sector_size > 4096) {
td_os_err(dev, "bio sector size of %u is not supported\n", bio_sector_size);
return -EINVAL;
}
/* MetaData is reported here */
if (hw_sector_size == 520)
hw_sector_size = 512;
if (!hw_sector_size || hw_sector_size & 511 || hw_sector_size > 4096) {
td_os_err(dev, "hw sector size of %u is not supported\n", hw_sector_size);
return -EINVAL;
}
td_os_notice(dev, " - Set capacity to %llu (%u bytes/sector)\n",
dev->block_params.capacity, dev->block_params.hw_sector_size);
/* create a new bio queue */
queue = blk_alloc_queue(GFP_KERNEL);
if (!queue) {
td_os_err(dev, "Error allocating disk queue.\n");
rc = -ENOMEM;
goto error_alloc_queue;
}
#ifdef QUEUE_FLAG_NONROT
queue_flag_set_unlocked(QUEUE_FLAG_NONROT, queue);
#endif
switch (dev->type) {
case TD_OSDEV_DEVICE:
blk_queue_make_request(queue, td_device_make_request);
dev->_bio_error = td_device_bio_error;
break;
case TD_OSDEV_RAID:
blk_queue_make_request(queue, td_raid_make_request);
dev->_bio_error = td_raid_bio_error;
break;
default:
td_os_err(dev, "Unkonwn OS Type, cannot register block request handler\n");
goto error_config_queue;
}
queue->queuedata = dev;
#if defined QUEUE_FLAG_PLUGGED
queue->unplug_fn = td_device_queue_unplug;
#endif
/* configure queue ordering */
/* in QUEUE_ORDERED_DRAIN we will get BARRIERS after the queue has
* been drained. */
#if defined KABI__blk_queue_ordered
#if KABI__blk_queue_ordered == 2
blk_queue_ordered(queue, QUEUE_ORDERED_DRAIN);
#elif KABI__blk_queue_ordered == 3
blk_queue_ordered(queue, QUEUE_ORDERED_DRAIN, NULL);
#else
#error unhandled value of KABI__blk_queue_ordered
#endif
#elif defined KABI__blk_queue_flush
/*
* blk_queue_ordered was replaced with blk_queue_flush
* The default implementation is QUEUE_ORDERED_DRAIN
*/
blk_queue_flush(queue, 0);
#else
#error undefined KABI__blk_queue_flush or KABI__blk_queue_ordered
#endif
/* max out the throttling */
#ifdef KABI__blk_queue_max_hw_sectors
blk_queue_max_hw_sectors(queue, dev->block_params.bio_max_bytes/512);
#elif defined KABI__blk_queue_max_sectors
blk_queue_max_sectors(queue, dev->block_params.bio_max_bytes/512);
#else
td_os_err(dev, "No kernel API for maximum sectors\n");
#endif
#if defined KABI__blk_queue_max_segments
blk_queue_max_segments(queue, BLK_MAX_SEGMENTS);
#elif defined KABI__blk_queue_max_phys_segments
blk_queue_max_phys_segments(queue, MAX_SEGMENT_SIZE);
blk_queue_max_hw_segments(queue, MAX_SEGMENT_SIZE);
#else
td_os_err(dev, "No kernel API for maximum segments\n");
#endif
blk_queue_max_segment_size(queue, dev->block_params.bio_max_bytes);
blk_queue_bounce_limit(queue, BLK_BOUNCE_ANY);
/* setup paged based access */
td_os_info(dev, "Set queue physical block size to %u\n", hw_sector_size);
#ifdef KABI__blk_queue_physical_block_size
blk_queue_physical_block_size(queue, hw_sector_size);
#elif defined KABI__blk_queue_hardsect_size
blk_queue_hardsect_size(queue, hw_sector_size);
#else
td_os_err(dev, "No kernel API for physical sector size\n");
#endif
#ifdef KABI__blk_queue_logical_block_size
td_os_info(dev, "Set queue logical block size to %u\n", bio_sector_size);
blk_queue_logical_block_size(queue, bio_sector_size);
#else
td_os_err(dev, "No kernel API for logical block size\n");
#endif
#ifdef KABI__blk_queue_io_min
td_os_info(dev, "Set queue io_min to %u\n", bio_sector_size);
blk_queue_io_min(queue, bio_sector_size);
#else
td_os_err(dev, "No kernel API for minimum IO size\n");
#endif
#ifdef KABI__blk_queue_io_opt
td_os_info(dev, "Set queue io_opt to %u\n", dev->block_params.bio_max_bytes);
blk_queue_io_opt(queue, dev->block_params.bio_max_bytes);
#else
td_os_err(dev, "No kernel API for optimal IO size\n");
#endif
#if 0
if (dev->block_params.discard)
{
int did_something = 0;
#if defined KABI__blk_queue_discard_granularity
queue->limits.discard_granularity = bio_sector_size;
did_something++;
#endif
#ifdef KABI__blk_queue_max_discard_sectors
/* 0xFFFF (max sector size of chunk on trim) * 64 * # SSD */
blk_queue_max_discard_sectors(queue, TD_MAX_DISCARD_LBA_COUNT * 2);
did_something++;
#endif
#ifdef KABI__blk_queue_discard_zeroes_data
queue->limits.discard_zeroes_data = 1;
did_something++;
#endif
#ifdef KABI__queue_flag_set_unlocked
queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, queue);
did_something++;
#endif
/* Maybe some day.. But not today.
queue_flag_set_unlocked(QUEUE_FLAG_SECDISCARD, queue);
*/
if (did_something)
td_os_info(dev, "Enabling discard support\n");
else
td_os_notice(dev, "No kernel API for discard support\n");
} else {
td_os_info(dev, "No DISCARD support enabled\n");
}
#else
/* bug 7444 */
if (dev->block_params.discard)
td_os_info(dev, "Device supports DISCARD but is currently being forced disabled\n");
#endif
/* assign */
dev->queue = queue;
return 0;
error_config_queue:
blk_cleanup_queue(dev->queue);
dev->queue = NULL;
error_alloc_queue:
return rc;
}
