blk_rq_bytes
ikega@DESKTOP-ESV85JT MINGW64 ~/OneDrive/Documents/tokunori/linux (master)
$ git grep blk_rq_bytes
arch/um/drivers/ubd_kern.c: io_req->io_desc[0].length = blk_rq_bytes(req);
block/bfq-cgroup.c: blkg_rwstat_add(&bfqg->stats.bytes, rq->cmd_flags, blk_rq_bytes(rq));
block/blk-crypto-internal.h: return bio_crypt_ctx_mergeable(req->crypt_ctx, blk_rq_bytes(req),
block/blk-crypto-internal.h: return bio_crypt_ctx_mergeable(req->crypt_ctx, blk_rq_bytes(req),
block/blk-merge.c: req->__data_len += blk_rq_bytes(next);
block/blk-mq.c: rq->bio, rq->biotail, blk_rq_bytes(rq));
block/blk-mq.c: int total_bytes = blk_rq_bytes(req);
block/blk-mq.c: * Passing the result of blk_rq_bytes() as @nr_bytes guarantees
block/blk-mq.c: if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
block/blk-mq.c: if (blk_update_request(rq, error, blk_rq_bytes(rq)))
block/blk-mq.c: blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) {
block/blk-mq.c: if (q->disk && should_fail_request(q->disk->part0, blk_rq_bytes(rq)))
block/blk-mq.c: rq->__data_len = blk_rq_bytes(rq_src);
block/bsg-lib.c: buf->payload_len = blk_rq_bytes(req);
drivers/ata/libata-scsi.c: req_blocks = blk_rq_bytes(rq) / scmd->device->sector_size;
drivers/block/ataflop.c: blk_rq_bytes(fd_request)));
drivers/block/floppy.c: if (remaining > blk_rq_bytes(current_req) && CT(raw_cmd->cmd[COMMAND]) == FD_WRITE) {
drivers/block/loop.c: ret = file->f_op->fallocate(file, mode, pos, blk_rq_bytes(rq));
drivers/block/loop.c: if (!cmd->use_aio || cmd->ret < 0 || cmd->ret == blk_rq_bytes(rq) ||
drivers/block/loop.c: iov_iter_bvec(&iter, rw, bvec, nr_bvec, blk_rq_bytes(rq));
drivers/block/nbd.c: blk_rq_bytes(req), (req->timeout / HZ) * cmd->retries);
drivers/block/nbd.c: unsigned long size = blk_rq_bytes(req);
drivers/block/nbd.c: (unsigned long long)blk_rq_pos(req) << 9, blk_rq_bytes(req));
drivers/block/null_blk/main.c: if (atomic_long_sub_return(blk_rq_bytes(rq), &nullb->cur_bytes) < 0) {
drivers/block/rbd.c: u64 length = blk_rq_bytes(rq);
drivers/block/rnbd/rnbd-clt.c: msg.bi_size = cpu_to_le32(blk_rq_bytes(rq));
drivers/block/ublk_drv.c: const unsigned int rq_bytes = blk_rq_bytes(req);
drivers/block/ublk_drv.c: const unsigned int rq_bytes = blk_rq_bytes(req);
drivers/block/ublk_drv.c: if (unlikely(mapped_bytes != blk_rq_bytes(req))) {
drivers/md/dm-mpath.c: size_t nr_bytes = blk_rq_bytes(rq);
drivers/memstick/core/ms_block.c: blk_rq_bytes(req), &len);
drivers/memstick/core/ms_block.c: blk_rq_bytes(req), &len);
drivers/memstick/core/mspro_block.c: count = blk_rq_bytes(msb->block_req);
drivers/memstick/core/mspro_block.c: t_len = blk_rq_bytes(msb->block_req);
drivers/mmc/core/block.c: } else if (!blk_rq_bytes(req)) {
drivers/mtd/ubi/block.c: to_read = blk_rq_bytes(req);
drivers/nvme/host/core.c: (unsigned long long)blk_rq_bytes(req) >> ns->lba_shift,
drivers/nvme/host/core.c: cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
drivers/nvme/host/core.c: cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
drivers/s390/block/dasd_eckd.c: data_size = blk_rq_bytes(req);
drivers/s390/block/scm_blk.c: aidaw = scm_aidaw_fetch(scmrq, blk_rq_bytes(req));
drivers/scsi/scsi_lib.c: return blk_rq_bytes(rq);
drivers/scsi/scsi_lib.c: BUG_ON(blk_rq_bytes(rq) && !bytes);
drivers/scsi/scsi_lib.c: } else if (blk_rq_bytes(req) == 0 && sense_current) {
drivers/scsi/scsi_lib.c: * good_bytes != blk_rq_bytes(req) as the signal for an error.
drivers/scsi/scsi_lib.c: if (likely(blk_rq_bytes(req) > 0 || blk_stat == BLK_STS_OK)) {
drivers/scsi/scsi_lib.c: if (scsi_end_request(req, blk_stat, blk_rq_bytes(req)))
drivers/scsi/scsi_lib.c: if (blk_rq_bytes(rq) & rq->q->dma_pad_mask) {
drivers/scsi/scsi_lib.c: (rq->q->dma_pad_mask & ~blk_rq_bytes(rq)) + 1;
drivers/scsi/scsi_lib.c: BUG_ON(blk_rq_bytes(req));
drivers/scsi/scsi_lib.c: cmd->transfersize = blk_rq_bytes(req);
drivers/scsi/scsi_lib.c: if (blk_rq_bytes(req))
drivers/scsi/sd.c: good_bytes = blk_rq_bytes(req);
drivers/scsi/sd.c: scsi_set_resid(SCpnt, blk_rq_bytes(req));
drivers/scsi/sd_zbc.c: scsi_set_resid(cmd, blk_rq_bytes(rq));
drivers/ufs/core/ufshcd.c: transfer_len = blk_rq_bytes(rq);
include/linux/blk-mq.h: * blk_rq_bytes() : bytes left in the entire request
include/linux/blk-mq.h:static inline unsigned int blk_rq_bytes(const struct request *rq)
include/linux/blk-mq.h: return blk_rq_bytes(rq) >> SECTOR_SHIFT;
include/linux/blk-mq.h: return blk_rq_bytes(rq);
include/scsi/scsi_cmnd.h: return blk_rq_bytes(scsi_cmd_to_rq(scmd)) >> shift;
include/trace/events/block.h: __entry->bytes = blk_rq_bytes(rq);
kernel/trace/blktrace.c: blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_INSERT,
kernel/trace/blktrace.c: blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_ISSUE,
kernel/trace/blktrace.c: blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_BACKMERGE,
kernel/trace/blktrace.c: blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_REQUEUE,
kernel/trace/blktrace.c: __blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq),
kernel/trace/blktrace.c: __blk_add_trace(bt, blk_rq_trace_sector(rq), blk_rq_bytes(rq), 0,
ikega@DESKTOP-ESV85JT MINGW64 ~/OneDrive/Documents/tokunori/linux (master)
$ git grep __data_len
block/blk-map.c: rq->__data_len += (bio)->bi_iter.bi_size;
block/blk-merge.c: req->__data_len += blk_rq_bytes(next);
block/blk-merge.c: req->__data_len += bio->bi_iter.bi_size;
block/blk-merge.c: req->__data_len += bio->bi_iter.bi_size;
block/blk-merge.c: req->__data_len += bio->bi_iter.bi_size;
block/blk-mq.c: rq->__data_len = 0;
block/blk-mq.c: req->__data_len = 0;
block/blk-mq.c: req->__data_len = 0;
block/blk-mq.c: req->__data_len -= total_bytes;
block/blk-mq.c: req->__data_len = blk_rq_cur_bytes(req);
block/blk-mq.c: rq->__data_len = blk_rq_bytes(rq_src);
block/blk-mq.c: rq->__data_len = 0;
include/linux/blk-mq.h: unsigned int __data_len; /* total data len */
include/linux/blk-mq.h: rq->__data_len = bio->bi_iter.bi_size;
include/linux/blk-mq.h: return rq->__data_len;
ikega@DESKTOP-ESV85JT MINGW64 ~/OneDrive/Documents/tokunori/linux (master)
$ git grep "bi_size = "
arch/x86/kernel/fpu/core.c: gfpu->uabi_size = fpu_user_cfg.default_size;
arch/x86/kernel/fpu/xstate.c: guest_fpu->uabi_size = usize;
block/bio-integrity.c: bip->bip_iter.bi_size = len;
block/bio-integrity.c: bip->bip_iter.bi_size = bio_integrity_bytes(bi, bio_sectors(bio));
block/bio.c: bio->bi_iter.bi_size = 0;
block/bio.c: bio->bi_iter.bi_size = new_size;
block/bio.c: bio->bi_iter.bi_size = size;
block/bio.c: split->bi_iter.bi_size = sectors << 9;
block/bio.c: bio->bi_iter.bi_size = size;
block/blk-lib.c: bio->bi_iter.bi_size = req_sects << 9;
block/blk-lib.c: bio->bi_iter.bi_size = max_write_zeroes_sectors << 9;
block/blk-lib.c: bio->bi_iter.bi_size = nr_sects << 9;
block/blk-lib.c: int bi_size = 0;
block/blk-lib.c: bi_size = bio_add_page(bio, ZERO_PAGE(0), sz, 0);
block/blk-lib.c: bio->bi_iter.bi_size = len;
drivers/block/aoe/aoecmd.c: f->iter.bi_size = min_t(unsigned long,
drivers/block/aoe/aoecmd.c: f->iter.bi_size = t->d->maxbcnt ? t->d->maxbcnt : DEFAULTBCNT;
drivers/block/aoe/aoecmd.c: iter.bi_size = cnt;
drivers/block/aoe/aoecmd.c: buf->iter.bi_size = 0;
drivers/block/rbd.c: .iter = { .bi_size = ceph_file_extents_bytes(img_extents,
drivers/block/rbd.c: .iter = { .bi_size = bytes },
drivers/block/rnbd/rnbd-srv.c: bio->bi_iter.bi_size = le32_to_cpu(msg->bi_size);
drivers/i3c/master/mipi-i3c-hci/dma.c: ibi_size = 0;
drivers/md/bcache/btree.c: bio->bi_iter.bi_size = KEY_SIZE(&b->key) << 9;
drivers/md/bcache/debug.c: check->bi_iter.bi_size = bio->bi_iter.bi_size;
drivers/md/bcache/debug.c: citer.bi_size = UINT_MAX;
drivers/md/bcache/journal.c: bio->bi_iter.bi_size = sectors << 9;
drivers/md/bcache/request.c: s->iop.bio->bi_iter.bi_size = s->insert_bio_sectors << 9;
drivers/md/bcache/super.c: bio->bi_iter.bi_size = KEY_SIZE(k) << 9;
drivers/md/dm-clone-target.c: bio->bi_iter.bi_size = to_bytes(len);
drivers/md/dm-crypt.c: bip->bip_iter.bi_size = tag_len;
drivers/md/dm-integrity.c: unsigned bi_size = dio->bio_details.bi_iter.bi_size;
drivers/md/dm-integrity.c: bio->bi_iter.bi_size = dio->range.n_sectors << SECTOR_SHIFT;
drivers/md/dm-io.c: bio->bi_iter.bi_size = num_sectors << SECTOR_SHIFT;
drivers/md/dm-log-writes.c: bio->bi_iter.bi_size = 0;
drivers/md/dm-log-writes.c: bio->bi_iter.bi_size = 0;
drivers/md/dm-log-writes.c: bio->bi_iter.bi_size = 0;
drivers/md/dm-log-writes.c: bio->bi_iter.bi_size = 0;
drivers/md/dm-stripe.c: bio->bi_iter.bi_size = to_bytes(end - begin);
drivers/md/dm-thin.c: bio->bi_iter.bi_size = (tc->origin_size - bio->bi_iter.bi_sector) << SECTOR_SHIFT;
drivers/md/dm-zoned-target.c: clone->bi_iter.bi_size = dmz_blk2sect(nr_blocks) << SECTOR_SHIFT;
drivers/md/dm.c: clone->bi_iter.bi_size = to_bytes(*len);
drivers/md/dm.c: bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT;
drivers/md/dm.c: ci->io->tio.clone.bi_iter.bi_size = 0;
drivers/md/md-faulty.c: b->bi_iter.bi_size = bio->bi_iter.bi_size;
drivers/md/raid1.c: behind_bio->bi_iter.bi_size = size;
drivers/md/raid1.c: wbio->bi_iter.bi_size = r1_bio->sectors << 9;
drivers/md/raid10.c: fbio->bi_iter.bi_size = r10_bio->sectors << 9;
drivers/md/raid5.c: bi->bi_iter.bi_size = RAID5_STRIPE_SIZE(conf);
drivers/md/raid5.c: rbi->bi_iter.bi_size = RAID5_STRIPE_SIZE(conf);
drivers/media/platform/amphion/vpu_malone.c: info->mbi_size = (info->sizeimage[0] + info->sizeimage[1]) >> 2;
drivers/media/platform/amphion/vpu_malone.c: info->mbi_size = ALIGN(info->mbi_size, MALONE_ALIGN_MBI);
drivers/media/usb/em28xx/em28xx-video.c: int vbi_size = v4l2->vbi_width * v4l2->vbi_height;
drivers/net/ethernet/intel/i40e/i40e_txrx.c: bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
drivers/net/ethernet/intel/i40e/i40e_txrx.c: bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
drivers/net/ethernet/intel/iavf/iavf_txrx.c: bi_size = sizeof(struct iavf_tx_buffer) * tx_ring->count;
drivers/net/ethernet/intel/iavf/iavf_txrx.c: bi_size = sizeof(struct iavf_tx_buffer) * tx_ring->count;
drivers/net/ethernet/intel/iavf/iavf_txrx.c: bi_size = sizeof(struct iavf_rx_buffer) * rx_ring->count;
drivers/net/ethernet/intel/iavf/iavf_txrx.c: bi_size = sizeof(struct iavf_rx_buffer) * rx_ring->count;
drivers/nvme/host/ioctl.c: bip->bip_iter.bi_size = len;
drivers/nvme/target/io-cmd-bdev.c: bip->bip_iter.bi_size = bio_integrity_bytes(bi, bio_sectors(bio));
drivers/target/target_core_iblock.c: bip->bip_iter.bi_size = bio_integrity_bytes(bi, bio_sectors(bio));
drivers/usb/misc/idmouse.c: dev->orig_bi_size = usb_endpoint_maxp(endpoint);
fs/jfs/jfs_logmgr.c: bio->bi_iter.bi_size = 0;
fs/jfs/jfs_logmgr.c: bio->bi_iter.bi_size = 0;
include/linux/bio.h: bio->bi_iter.bi_size = 0;
include/linux/bvec.h: iter->bi_size = 0;
include/linux/ceph/messenger.h: __cur_iter.bi_size = __cur_n; \
include/linux/ceph/messenger.h: __cur_iter.bi_size = (n); \
include/linux/ceph/messenger.h: (it)->iter.bi_size = (n); \
io_uring/net.c: bi.bi_size = min(from->count, length);
kernel/ptrace.c: .rseq_abi_size = sizeof(*task->rseq),
net/ceph/messenger.c: it->iter.bi_size = cursor->resid;
net/ceph/messenger.c: it->iter.bi_size = cursor->resid;
net/ceph/messenger.c: cursor->bvec_iter.bi_size = cursor->resid;
net/ceph/osd_client.c: .iter = { .bi_size = bytes },
net/ceph/osd_client.c: .iter = { .bi_size = bytes },
net/sunrpc/xprtsock.c: .bi_size = count,
ikega@DESKTOP-ESV85JT MINGW64 ~/OneDrive/Documents/tokunori/linux (master)
$ git grep "bi_size = " | grep max
block/blk-lib.c: bio->bi_iter.bi_size = max_write_zeroes_sectors << 9;
drivers/block/aoe/aoecmd.c: f->iter.bi_size = t->d->maxbcnt ? t->d->maxbcnt : DEFAULTBCNT;
drivers/usb/misc/idmouse.c: dev->orig_bi_size = usb_endpoint_maxp(endpoint);
ikega@DESKTOP-ESV85JT MINGW64 ~/OneDrive/Documents/tokunori/linux (master)
$ git grep bio_add_hw_page
block/bio.c: * bio_add_hw_page - attempt to add a page to a bio with hw constraints
block/bio.c:int bio_add_hw_page(struct request_queue *q, struct bio *bio,
block/bio.c: return bio_add_hw_page(q, bio, page, len, offset,
block/bio.c: return bio_add_hw_page(q, bio, page, len, offset,
block/bio.c: if (bio_add_hw_page(q, bio, page, len, offset,
block/blk-map.c: if (!bio_add_hw_page(rq->q, bio, page, n, offs,
block/blk.h:int bio_add_hw_page(struct request_queue *q, struct bio *bio,
/*
* blk_rq_pos() : the current sector
* blk_rq_bytes() : bytes left in the entire request
* blk_rq_cur_bytes() : bytes left in the current segment
* blk_rq_sectors() : sectors left in the entire request
* blk_rq_cur_sectors() : sectors left in the current segment
* blk_rq_stats_sectors() : sectors of the entire request used for stats
*/
static inline sector_t blk_rq_pos(const struct request *rq)
{
return rq->__sector;
}
static inline unsigned int blk_rq_bytes(const struct request *rq)
{
return rq->__data_len;
}
static inline int blk_rq_cur_bytes(const struct request *rq)
{
if (!rq->bio)
return 0;
if (!bio_has_data(rq->bio)) /* dataless requests such as discard */
return rq->bio->bi_iter.bi_size;
return bio_iovec(rq->bio).bv_len;
}
static inline unsigned int blk_rq_sectors(const struct request *rq)
{
return blk_rq_bytes(rq) >> SECTOR_SHIFT;
}
static inline unsigned int blk_rq_cur_sectors(const struct request *rq)
{
return blk_rq_cur_bytes(rq) >> SECTOR_SHIFT;
}
static inline unsigned int blk_rq_stats_sectors(const struct request *rq)
{
return rq->stats_sectors;
}
/*
* Some commands like WRITE SAME have a payload or data transfer size which
* is different from the size of the request. Any driver that supports such
* commands using the RQF_SPECIAL_PAYLOAD flag needs to use this helper to
* calculate the data transfer size.
*/
static inline unsigned int blk_rq_payload_bytes(struct request *rq)
{
if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
return rq->special_vec.bv_len;
return blk_rq_bytes(rq);
}
static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
struct request *req, struct nvme_command *cmnd,
enum nvme_opcode op)
{
u16 control = 0;
u32 dsmgmt = 0;
if (req->cmd_flags & REQ_FUA)
control |= NVME_RW_FUA;
if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
control |= NVME_RW_LR;
if (req->cmd_flags & REQ_RAHEAD)
dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
cmnd->rw.opcode = op;
cmnd->rw.flags = 0;
cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id);
cmnd->rw.cdw2 = 0;
cmnd->rw.cdw3 = 0;
cmnd->rw.metadata = 0;
cmnd->rw.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
static inline blk_status_t nvme_setup_write_zeroes(struct nvme_ns *ns,
struct request *req, struct nvme_command *cmnd)
{
memset(cmnd, 0, sizeof(*cmnd));
if (ns->ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
return nvme_setup_discard(ns, req, cmnd);
cmnd->write_zeroes.opcode = nvme_cmd_write_zeroes;
cmnd->write_zeroes.nsid = cpu_to_le32(ns->head->ns_id);
cmnd->write_zeroes.slba =
cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
cmnd->write_zeroes.length =
cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
static inline void blk_rq_bio_prep(struct request *rq, struct bio *bio,
unsigned int nr_segs)
{
rq->nr_phys_segments = nr_segs;
rq->__data_len = bio->bi_iter.bi_size;
rq->bio = rq->biotail = bio;
rq->ioprio = bio_prio(bio);
}
static int __blkdev_issue_write_zeroes(struct block_device *bdev,
sector_t sector, sector_t nr_sects, gfp_t gfp_mask,
struct bio **biop, unsigned flags)
{
struct bio *bio = *biop;
unsigned int max_write_zeroes_sectors;
if (bdev_read_only(bdev))
return -EPERM;
/* Ensure that max_write_zeroes_sectors doesn't overflow bi_size */
max_write_zeroes_sectors = bdev_write_zeroes_sectors(bdev);
if (max_write_zeroes_sectors == 0)
return -EOPNOTSUPP;
while (nr_sects) {
bio = blk_next_bio(bio, bdev, 0, REQ_OP_WRITE_ZEROES, gfp_mask);
bio->bi_iter.bi_sector = sector;
if (flags & BLKDEV_ZERO_NOUNMAP)
bio->bi_opf |= REQ_NOUNMAP;
if (nr_sects > max_write_zeroes_sectors) {
bio->bi_iter.bi_size = max_write_zeroes_sectors << 9;
nr_sects -= max_write_zeroes_sectors;
sector += max_write_zeroes_sectors;
} else {
bio->bi_iter.bi_size = nr_sects << 9;
/**
* bio_add_hw_page - attempt to add a page to a bio with hw constraints
* @q: the target queue
* @bio: destination bio
* @page: page to add
* @len: vec entry length
* @offset: vec entry offset
* @max_sectors: maximum number of sectors that can be added
* @same_page: return if the segment has been merged inside the same page
*
* Add a page to a bio while respecting the hardware max_sectors, max_segment
* and gap limitations.
*/
int bio_add_hw_page(struct request_queue *q, struct bio *bio,
struct page *page, unsigned int len, unsigned int offset,
unsigned int max_sectors, bool *same_page)
{
struct bio_vec *bvec;
if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
return 0;
if (((bio->bi_iter.bi_size + len) >> 9) > max_sectors)
return 0;
if (bio->bi_vcnt > 0) {
if (bio_try_merge_hw_seg(q, bio, page, len, offset, same_page))
return len;
/*
* If the queue doesn't support SG gaps and adding this segment
* would create a gap, disallow it.
*/
bvec = &bio->bi_io_vec[bio->bi_vcnt - 1];
if (bvec_gap_to_prev(&q->limits, bvec, offset))
return 0;
}
if (bio_full(bio, len))
return 0;
if (bio->bi_vcnt >= queue_max_segments(q))
return 0;
bvec = &bio->bi_io_vec[bio->bi_vcnt];
bvec->bv_page = page;
bvec->bv_len = len;
bvec->bv_offset = offset;
bio->bi_vcnt++;
bio->bi_iter.bi_size += len;
return len;
}
static int bio_map_user_iov(struct request *rq, struct iov_iter *iter,
gfp_t gfp_mask)
{
unsigned int max_sectors = queue_max_hw_sectors(rq->q);
unsigned int nr_vecs = iov_iter_npages(iter, BIO_MAX_VECS);
struct bio *bio;
int ret;
int j;
if (!iov_iter_count(iter))
return -EINVAL;
bio = bio_kmalloc(nr_vecs, gfp_mask);
if (!bio)
return -ENOMEM;
bio_init(bio, NULL, bio->bi_inline_vecs, nr_vecs, req_op(rq));
while (iov_iter_count(iter)) {
struct page **pages;
ssize_t bytes;
size_t offs, added = 0;
int npages;
bytes = iov_iter_get_pages_alloc2(iter, &pages, LONG_MAX, &offs);
if (unlikely(bytes <= 0)) {
ret = bytes ? bytes : -EFAULT;
goto out_unmap;
}
npages = DIV_ROUND_UP(offs + bytes, PAGE_SIZE);
if (unlikely(offs & queue_dma_alignment(rq->q)))
j = 0;
else {
for (j = 0; j < npages; j++) {
struct page *page = pages[j];
unsigned int n = PAGE_SIZE - offs;
bool same_page = false;
if (n > bytes)
n = bytes;
if (!bio_add_hw_page(rq->q, bio, page, n, offs,
max_sectors, &same_page)) {
max hardware sectors
/*
* These can be higher, but we need to ensure that any command doesn't
* require an sg allocation that needs more than a page of data.
*/
#define NVME_MAX_KB_SZ 4096
#define NVME_MAX_SEGS 127
/*
* Will slightly overestimate the number of pages needed. This is OK
* as it only leads to a small amount of wasted memory for the lifetime of
* the I/O.
*/
static int nvme_pci_npages_prp(void)
{
unsigned nprps = DIV_ROUND_UP(NVME_MAX_KB_SZ + NVME_CTRL_PAGE_SIZE,
NVME_CTRL_PAGE_SIZE);
return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
}
/*
* Calculates the number of pages needed for the SGL segments. For example a 4k
* page can accommodate 256 SGL descriptors.
*/
static int nvme_pci_npages_sgl(void)
{
return DIV_ROUND_UP(NVME_MAX_SEGS * sizeof(struct nvme_sgl_desc),
PAGE_SIZE);
}
static size_t nvme_pci_iod_alloc_size(void)
{
size_t npages = max(nvme_pci_npages_prp(), nvme_pci_npages_sgl());
return sizeof(__le64 *) * npages +
sizeof(struct scatterlist) * NVME_MAX_SEGS;
}
static void nvme_reset_work(struct work_struct *work)
{
...
/*
* Limit the max command size to prevent iod->sg allocations going
* over a single page.
*/
dev->ctrl.max_hw_sectors = min_t(u32,
NVME_MAX_KB_SZ << 1, dma_max_mapping_size(dev->dev) >> 9);
dev->ctrl.max_segments = NVME_MAX_SEGS;
static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
struct request_queue *q)
{
bool vwc = ctrl->vwc & NVME_CTRL_VWC_PRESENT;
if (ctrl->max_hw_sectors) {
u32 max_segments =
(ctrl->max_hw_sectors / (NVME_CTRL_PAGE_SIZE >> 9)) + 1;
max_segments = min_not_zero(max_segments, ctrl->max_segments);
blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
static int nvme_init_non_mdts_limits(struct nvme_ctrl *ctrl)
{
...
/*
* Even though NVMe spec explicitly states that MDTS is not applicable
* to the write-zeroes, we are cautious and limit the size to the
* controllers max_hw_sectors value, which is based on the MDTS field
* and possibly other limiting factors.
*/
if ((ctrl->oncs & NVME_CTRL_ONCS_WRITE_ZEROES) &&
!(ctrl->quirks & NVME_QUIRK_DISABLE_WRITE_ZEROES))
ctrl->max_zeroes_sectors = ctrl->max_hw_sectors;
else
ctrl->max_zeroes_sectors = 0;
static int nvme_init_identify(struct nvme_ctrl *ctrl)
{
...
u32 max_hw_sectors;
...
if (id->mdts)
max_hw_sectors = nvme_mps_to_sectors(ctrl, id->mdts);
else
max_hw_sectors = UINT_MAX;
ctrl->max_hw_sectors =
min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
nlb
struct nvme_rw_command {
__u8 opcode;
__u8 flags;
__u16 command_id;
__le32 nsid;
__le32 cdw2;
__le32 cdw3;
__le64 metadata;
union nvme_data_ptr dptr;
__le64 slba;
__le16 length;
__le16 control;
__le32 dsmgmt;
__le32 reftag;
__le16 apptag;
__le16 appmask;
};
struct nvme_command {
union {
struct nvme_common_command common;
struct nvme_rw_command rw;
static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
{
...
memset(&c, 0, sizeof(c));
c.rw.opcode = io.opcode;
c.rw.flags = io.flags;
c.rw.nsid = cpu_to_le32(ns->head->ns_id);
c.rw.slba = cpu_to_le64(io.slba);
c.rw.length = cpu_to_le16(io.nblocks);
static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
struct request *req, struct nvme_command *cmnd,
enum nvme_opcode op)
{
u16 control = 0;
u32 dsmgmt = 0;
if (req->cmd_flags & REQ_FUA)
control |= NVME_RW_FUA;
if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
control |= NVME_RW_LR;
if (req->cmd_flags & REQ_RAHEAD)
dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
cmnd->rw.opcode = op;
cmnd->rw.flags = 0;
cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id);
cmnd->rw.cdw2 = 0;
cmnd->rw.cdw3 = 0;
cmnd->rw.metadata = 0;
cmnd->rw.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
MDTS
static int nvme_init_identify(struct nvme_ctrl *ctrl)
{
...
u32 max_hw_sectors;
...
if (id->mdts)
max_hw_sectors = nvme_mps_to_sectors(ctrl, id->mdts);
else
max_hw_sectors = UINT_MAX;
ctrl->max_hw_sectors =
min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
static inline u32 nvme_mps_to_sectors(struct nvme_ctrl *ctrl, u32 units)
{
u32 page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12, val;
if (check_shl_overflow(1U, units + page_shift - 9, &val))
return UINT_MAX;
return val;
}
static int nvme_init_non_mdts_limits(struct nvme_ctrl *ctrl)
{
...
/*
* Even though NVMe spec explicitly states that MDTS is not applicable
* to the write-zeroes, we are cautious and limit the size to the
* controllers max_hw_sectors value, which is based on the MDTS field
* and possibly other limiting factors.
*/
if ((ctrl->oncs & NVME_CTRL_ONCS_WRITE_ZEROES) &&
!(ctrl->quirks & NVME_QUIRK_DISABLE_WRITE_ZEROES))
ctrl->max_zeroes_sectors = ctrl->max_hw_sectors;
else
ctrl->max_zeroes_sectors = 0;
static void nvme_set_chunk_sectors(struct nvme_ns *ns, struct nvme_id_ns *id)
{
struct nvme_ctrl *ctrl = ns->ctrl;
u32 iob;
if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) &&
is_power_of_2(ctrl->max_hw_sectors))
iob = ctrl->max_hw_sectors;
else
iob = nvme_lba_to_sect(ns, le16_to_cpu(id->noiob));
static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
struct request_queue *q)
{
bool vwc = ctrl->vwc & NVME_CTRL_VWC_PRESENT;
if (ctrl->max_hw_sectors) {
u32 max_segments =
(ctrl->max_hw_sectors / (NVME_CTRL_PAGE_SIZE >> 9)) + 1;
max_segments = min_not_zero(max_segments, ctrl->max_segments);
blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);