Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : //#define DEBUG
3 : #include <linux/spinlock.h>
4 : #include <linux/slab.h>
5 : #include <linux/blkdev.h>
6 : #include <linux/hdreg.h>
7 : #include <linux/module.h>
8 : #include <linux/mutex.h>
9 : #include <linux/interrupt.h>
10 : #include <linux/virtio.h>
11 : #include <linux/virtio_blk.h>
12 : #include <linux/scatterlist.h>
13 : #include <linux/string_helpers.h>
14 : #include <linux/idr.h>
15 : #include <linux/blk-mq.h>
16 : #include <linux/blk-mq-virtio.h>
17 : #include <linux/numa.h>
18 : #include <uapi/linux/virtio_ring.h>
19 :
20 : #define PART_BITS 4
21 : #define VQ_NAME_LEN 16
22 : #define MAX_DISCARD_SEGMENTS 256u
23 :
24 : static int major;
25 : static DEFINE_IDA(vd_index_ida);
26 :
27 : static struct workqueue_struct *virtblk_wq;
28 :
29 : struct virtio_blk_vq {
30 : struct virtqueue *vq;
31 : spinlock_t lock;
32 : char name[VQ_NAME_LEN];
33 : } ____cacheline_aligned_in_smp;
34 :
35 : struct virtio_blk {
36 : /*
37 : * This mutex must be held by anything that may run after
38 : * virtblk_remove() sets vblk->vdev to NULL.
39 : *
40 : * blk-mq, virtqueue processing, and sysfs attribute code paths are
41 : * shut down before vblk->vdev is set to NULL and therefore do not need
42 : * to hold this mutex.
43 : */
44 : struct mutex vdev_mutex;
45 : struct virtio_device *vdev;
46 :
47 : /* The disk structure for the kernel. */
48 : struct gendisk *disk;
49 :
50 : /* Block layer tags. */
51 : struct blk_mq_tag_set tag_set;
52 :
53 : /* Process context for config space updates */
54 : struct work_struct config_work;
55 :
56 : /*
57 : * Tracks references from block_device_operations open/release and
58 : * virtio_driver probe/remove so this object can be freed once no
59 : * longer in use.
60 : */
61 : refcount_t refs;
62 :
63 : /* What host tells us, plus 2 for header & tailer. */
64 : unsigned int sg_elems;
65 :
66 : /* Ida index - used to track minor number allocations. */
67 : int index;
68 :
69 : /* num of vqs */
70 : int num_vqs;
71 : struct virtio_blk_vq *vqs;
72 : };
73 :
74 : struct virtblk_req {
75 : struct virtio_blk_outhdr out_hdr;
76 : u8 status;
77 : struct scatterlist sg[];
78 : };
79 :
80 3437 : static inline blk_status_t virtblk_result(struct virtblk_req *vbr)
81 : {
82 2 : switch (vbr->status) {
83 : case VIRTIO_BLK_S_OK:
84 : return BLK_STS_OK;
85 0 : case VIRTIO_BLK_S_UNSUPP:
86 0 : return BLK_STS_NOTSUPP;
87 0 : default:
88 0 : return BLK_STS_IOERR;
89 : }
90 : }
91 :
92 3435 : static int virtblk_add_req(struct virtqueue *vq, struct virtblk_req *vbr,
93 : struct scatterlist *data_sg, bool have_data)
94 : {
95 3435 : struct scatterlist hdr, status, *sgs[3];
96 3435 : unsigned int num_out = 0, num_in = 0;
97 :
98 3435 : sg_init_one(&hdr, &vbr->out_hdr, sizeof(vbr->out_hdr));
99 3435 : sgs[num_out++] = &hdr;
100 :
101 3435 : if (have_data) {
102 3242 : if (vbr->out_hdr.type & cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_OUT))
103 679 : sgs[num_out++] = data_sg;
104 : else
105 2563 : sgs[num_out + num_in++] = data_sg;
106 : }
107 :
108 3435 : sg_init_one(&status, &vbr->status, sizeof(vbr->status));
109 3435 : sgs[num_out + num_in++] = &status;
110 :
111 3435 : return virtqueue_add_sgs(vq, sgs, num_out, num_in, vbr, GFP_ATOMIC);
112 : }
113 :
114 0 : static int virtblk_setup_discard_write_zeroes(struct request *req, bool unmap)
115 : {
116 0 : unsigned short segments = blk_rq_nr_discard_segments(req);
117 0 : unsigned short n = 0;
118 0 : struct virtio_blk_discard_write_zeroes *range;
119 0 : struct bio *bio;
120 0 : u32 flags = 0;
121 :
122 0 : if (unmap)
123 0 : flags |= VIRTIO_BLK_WRITE_ZEROES_FLAG_UNMAP;
124 :
125 0 : range = kmalloc_array(segments, sizeof(*range), GFP_ATOMIC);
126 0 : if (!range)
127 : return -ENOMEM;
128 :
129 : /*
130 : * Single max discard segment means multi-range discard isn't
131 : * supported, and block layer only runs contiguity merge like
132 : * normal RW request. So we can't reply on bio for retrieving
133 : * each range info.
134 : */
135 0 : if (queue_max_discard_segments(req->q) == 1) {
136 0 : range[0].flags = cpu_to_le32(flags);
137 0 : range[0].num_sectors = cpu_to_le32(blk_rq_sectors(req));
138 0 : range[0].sector = cpu_to_le64(blk_rq_pos(req));
139 0 : n = 1;
140 : } else {
141 0 : __rq_for_each_bio(bio, req) {
142 0 : u64 sector = bio->bi_iter.bi_sector;
143 0 : u32 num_sectors = bio->bi_iter.bi_size >> SECTOR_SHIFT;
144 :
145 0 : range[n].flags = cpu_to_le32(flags);
146 0 : range[n].num_sectors = cpu_to_le32(num_sectors);
147 0 : range[n].sector = cpu_to_le64(sector);
148 0 : n++;
149 : }
150 : }
151 :
152 0 : WARN_ON_ONCE(n != segments);
153 :
154 0 : req->special_vec.bv_page = virt_to_page(range);
155 0 : req->special_vec.bv_offset = offset_in_page(range);
156 0 : req->special_vec.bv_len = sizeof(*range) * segments;
157 0 : req->rq_flags |= RQF_SPECIAL_PAYLOAD;
158 :
159 0 : return 0;
160 : }
161 :
162 3435 : static inline void virtblk_request_done(struct request *req)
163 : {
164 3435 : struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);
165 :
166 3435 : if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
167 0 : kfree(page_address(req->special_vec.bv_page) +
168 0 : req->special_vec.bv_offset);
169 : }
170 :
171 3435 : blk_mq_end_request(req, virtblk_result(vbr));
172 3435 : }
173 :
174 2690 : static void virtblk_done(struct virtqueue *vq)
175 : {
176 2690 : struct virtio_blk *vblk = vq->vdev->priv;
177 2690 : bool req_done = false;
178 2690 : int qid = vq->index;
179 2690 : struct virtblk_req *vbr;
180 2690 : unsigned long flags;
181 2690 : unsigned int len;
182 :
183 2690 : spin_lock_irqsave(&vblk->vqs[qid].lock, flags);
184 2690 : do {
185 2690 : virtqueue_disable_cb(vq);
186 6125 : while ((vbr = virtqueue_get_buf(vblk->vqs[qid].vq, &len)) != NULL) {
187 3435 : struct request *req = blk_mq_rq_from_pdu(vbr);
188 :
189 3435 : if (likely(!blk_should_fake_timeout(req->q)))
190 3435 : blk_mq_complete_request(req);
191 3435 : req_done = true;
192 : }
193 2690 : if (unlikely(virtqueue_is_broken(vq)))
194 : break;
195 2690 : } while (!virtqueue_enable_cb(vq));
196 :
197 : /* In case queue is stopped waiting for more buffers. */
198 2690 : if (req_done)
199 2690 : blk_mq_start_stopped_hw_queues(vblk->disk->queue, true);
200 2690 : spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);
201 2690 : }
202 :
203 0 : static void virtio_commit_rqs(struct blk_mq_hw_ctx *hctx)
204 : {
205 0 : struct virtio_blk *vblk = hctx->queue->queuedata;
206 0 : struct virtio_blk_vq *vq = &vblk->vqs[hctx->queue_num];
207 0 : bool kick;
208 :
209 0 : spin_lock_irq(&vq->lock);
210 0 : kick = virtqueue_kick_prepare(vq->vq);
211 0 : spin_unlock_irq(&vq->lock);
212 :
213 0 : if (kick)
214 0 : virtqueue_notify(vq->vq);
215 0 : }
216 :
217 3435 : static blk_status_t virtio_queue_rq(struct blk_mq_hw_ctx *hctx,
218 : const struct blk_mq_queue_data *bd)
219 : {
220 3435 : struct virtio_blk *vblk = hctx->queue->queuedata;
221 3435 : struct request *req = bd->rq;
222 3435 : struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);
223 3435 : unsigned long flags;
224 3435 : unsigned int num;
225 3435 : int qid = hctx->queue_num;
226 3435 : int err;
227 3435 : bool notify = false;
228 3435 : bool unmap = false;
229 3435 : u32 type;
230 :
231 3435 : BUG_ON(req->nr_phys_segments + 2 > vblk->sg_elems);
232 :
233 3435 : switch (req_op(req)) {
234 : case REQ_OP_READ:
235 : case REQ_OP_WRITE:
236 : type = 0;
237 : break;
238 193 : case REQ_OP_FLUSH:
239 193 : type = VIRTIO_BLK_T_FLUSH;
240 193 : break;
241 0 : case REQ_OP_DISCARD:
242 0 : type = VIRTIO_BLK_T_DISCARD;
243 0 : break;
244 0 : case REQ_OP_WRITE_ZEROES:
245 0 : type = VIRTIO_BLK_T_WRITE_ZEROES;
246 0 : unmap = !(req->cmd_flags & REQ_NOUNMAP);
247 0 : break;
248 2 : case REQ_OP_DRV_IN:
249 2 : type = VIRTIO_BLK_T_GET_ID;
250 2 : break;
251 : default:
252 0 : WARN_ON_ONCE(1);
253 0 : return BLK_STS_IOERR;
254 : }
255 :
256 3435 : vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, type);
257 6870 : vbr->out_hdr.sector = type ?
258 3435 : 0 : cpu_to_virtio64(vblk->vdev, blk_rq_pos(req));
259 3435 : vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(req));
260 :
261 3435 : blk_mq_start_request(req);
262 :
263 3434 : if (type == VIRTIO_BLK_T_DISCARD || type == VIRTIO_BLK_T_WRITE_ZEROES) {
264 0 : err = virtblk_setup_discard_write_zeroes(req, unmap);
265 0 : if (err)
266 : return BLK_STS_RESOURCE;
267 : }
268 :
269 3434 : num = blk_rq_map_sg(hctx->queue, req, vbr->sg);
270 3435 : if (num) {
271 3241 : if (rq_data_dir(req) == WRITE)
272 679 : vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_OUT);
273 : else
274 2562 : vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_IN);
275 : }
276 :
277 3435 : spin_lock_irqsave(&vblk->vqs[qid].lock, flags);
278 3435 : err = virtblk_add_req(vblk->vqs[qid].vq, vbr, vbr->sg, num);
279 3435 : if (err) {
280 0 : virtqueue_kick(vblk->vqs[qid].vq);
281 : /* Don't stop the queue if -ENOMEM: we may have failed to
282 : * bounce the buffer due to global resource outage.
283 : */
284 0 : if (err == -ENOSPC)
285 0 : blk_mq_stop_hw_queue(hctx);
286 0 : spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);
287 0 : switch (err) {
288 : case -ENOSPC:
289 : return BLK_STS_DEV_RESOURCE;
290 0 : case -ENOMEM:
291 0 : return BLK_STS_RESOURCE;
292 0 : default:
293 0 : return BLK_STS_IOERR;
294 : }
295 : }
296 :
297 3435 : if (bd->last && virtqueue_kick_prepare(vblk->vqs[qid].vq))
298 2743 : notify = true;
299 3435 : spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);
300 :
301 3435 : if (notify)
302 2743 : virtqueue_notify(vblk->vqs[qid].vq);
303 : return BLK_STS_OK;
304 : }
305 :
306 : /* return id (s/n) string for *disk to *id_str
307 : */
308 2 : static int virtblk_get_id(struct gendisk *disk, char *id_str)
309 : {
310 2 : struct virtio_blk *vblk = disk->private_data;
311 2 : struct request_queue *q = vblk->disk->queue;
312 2 : struct request *req;
313 2 : int err;
314 :
315 2 : req = blk_get_request(q, REQ_OP_DRV_IN, 0);
316 2 : if (IS_ERR(req))
317 0 : return PTR_ERR(req);
318 :
319 2 : err = blk_rq_map_kern(q, req, id_str, VIRTIO_BLK_ID_BYTES, GFP_KERNEL);
320 2 : if (err)
321 0 : goto out;
322 :
323 2 : blk_execute_rq(vblk->disk, req, false);
324 2 : err = blk_status_to_errno(virtblk_result(blk_mq_rq_to_pdu(req)));
325 2 : out:
326 2 : blk_put_request(req);
327 2 : return err;
328 : }
329 :
330 10 : static void virtblk_get(struct virtio_blk *vblk)
331 : {
332 10 : refcount_inc(&vblk->refs);
333 10 : }
334 :
335 9 : static void virtblk_put(struct virtio_blk *vblk)
336 : {
337 9 : if (refcount_dec_and_test(&vblk->refs)) {
338 0 : ida_simple_remove(&vd_index_ida, vblk->index);
339 0 : mutex_destroy(&vblk->vdev_mutex);
340 0 : kfree(vblk);
341 : }
342 9 : }
343 :
344 10 : static int virtblk_open(struct block_device *bd, fmode_t mode)
345 : {
346 10 : struct virtio_blk *vblk = bd->bd_disk->private_data;
347 10 : int ret = 0;
348 :
349 10 : mutex_lock(&vblk->vdev_mutex);
350 :
351 10 : if (vblk->vdev)
352 10 : virtblk_get(vblk);
353 : else
354 : ret = -ENXIO;
355 :
356 10 : mutex_unlock(&vblk->vdev_mutex);
357 10 : return ret;
358 : }
359 :
360 9 : static void virtblk_release(struct gendisk *disk, fmode_t mode)
361 : {
362 9 : struct virtio_blk *vblk = disk->private_data;
363 :
364 9 : virtblk_put(vblk);
365 9 : }
366 :
367 : /* We provide getgeo only to please some old bootloader/partitioning tools */
368 0 : static int virtblk_getgeo(struct block_device *bd, struct hd_geometry *geo)
369 : {
370 0 : struct virtio_blk *vblk = bd->bd_disk->private_data;
371 0 : int ret = 0;
372 :
373 0 : mutex_lock(&vblk->vdev_mutex);
374 :
375 0 : if (!vblk->vdev) {
376 0 : ret = -ENXIO;
377 0 : goto out;
378 : }
379 :
380 : /* see if the host passed in geometry config */
381 0 : if (virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_GEOMETRY)) {
382 0 : virtio_cread(vblk->vdev, struct virtio_blk_config,
383 : geometry.cylinders, &geo->cylinders);
384 0 : virtio_cread(vblk->vdev, struct virtio_blk_config,
385 : geometry.heads, &geo->heads);
386 0 : virtio_cread(vblk->vdev, struct virtio_blk_config,
387 : geometry.sectors, &geo->sectors);
388 : } else {
389 : /* some standard values, similar to sd */
390 0 : geo->heads = 1 << 6;
391 0 : geo->sectors = 1 << 5;
392 0 : geo->cylinders = get_capacity(bd->bd_disk) >> 11;
393 : }
394 0 : out:
395 0 : mutex_unlock(&vblk->vdev_mutex);
396 0 : return ret;
397 : }
398 :
399 : static const struct block_device_operations virtblk_fops = {
400 : .owner = THIS_MODULE,
401 : .open = virtblk_open,
402 : .release = virtblk_release,
403 : .getgeo = virtblk_getgeo,
404 : };
405 :
406 1 : static int index_to_minor(int index)
407 : {
408 1 : return index << PART_BITS;
409 : }
410 :
411 1 : static int minor_to_index(int minor)
412 : {
413 1 : return minor >> PART_BITS;
414 : }
415 :
416 2 : static ssize_t serial_show(struct device *dev,
417 : struct device_attribute *attr, char *buf)
418 : {
419 2 : struct gendisk *disk = dev_to_disk(dev);
420 2 : int err;
421 :
422 : /* sysfs gives us a PAGE_SIZE buffer */
423 2 : BUILD_BUG_ON(PAGE_SIZE < VIRTIO_BLK_ID_BYTES);
424 :
425 2 : buf[VIRTIO_BLK_ID_BYTES] = '\0';
426 2 : err = virtblk_get_id(disk, buf);
427 2 : if (!err)
428 2 : return strlen(buf);
429 :
430 0 : if (err == -EIO) /* Unsupported? Make it empty. */
431 : return 0;
432 :
433 0 : return err;
434 : }
435 :
436 : static DEVICE_ATTR_RO(serial);
437 :
438 : /* The queue's logical block size must be set before calling this */
439 1 : static void virtblk_update_capacity(struct virtio_blk *vblk, bool resize)
440 : {
441 1 : struct virtio_device *vdev = vblk->vdev;
442 1 : struct request_queue *q = vblk->disk->queue;
443 1 : char cap_str_2[10], cap_str_10[10];
444 1 : unsigned long long nblocks;
445 1 : u64 capacity;
446 :
447 : /* Host must always specify the capacity. */
448 1 : virtio_cread(vdev, struct virtio_blk_config, capacity, &capacity);
449 :
450 : /* If capacity is too big, truncate with warning. */
451 1 : if ((sector_t)capacity != capacity) {
452 : dev_warn(&vdev->dev, "Capacity %llu too large: truncating\n",
453 : (unsigned long long)capacity);
454 : capacity = (sector_t)-1;
455 : }
456 :
457 2 : nblocks = DIV_ROUND_UP_ULL(capacity, queue_logical_block_size(q) >> 9);
458 :
459 2 : string_get_size(nblocks, queue_logical_block_size(q),
460 : STRING_UNITS_2, cap_str_2, sizeof(cap_str_2));
461 2 : string_get_size(nblocks, queue_logical_block_size(q),
462 : STRING_UNITS_10, cap_str_10, sizeof(cap_str_10));
463 :
464 3 : dev_notice(&vdev->dev,
465 : "[%s] %s%llu %d-byte logical blocks (%s/%s)\n",
466 : vblk->disk->disk_name,
467 : resize ? "new size: " : "",
468 : nblocks,
469 : queue_logical_block_size(q),
470 : cap_str_10,
471 : cap_str_2);
472 :
473 1 : set_capacity_and_notify(vblk->disk, capacity);
474 1 : }
475 :
476 0 : static void virtblk_config_changed_work(struct work_struct *work)
477 : {
478 0 : struct virtio_blk *vblk =
479 0 : container_of(work, struct virtio_blk, config_work);
480 :
481 0 : virtblk_update_capacity(vblk, true);
482 0 : }
483 :
484 0 : static void virtblk_config_changed(struct virtio_device *vdev)
485 : {
486 0 : struct virtio_blk *vblk = vdev->priv;
487 :
488 0 : queue_work(virtblk_wq, &vblk->config_work);
489 0 : }
490 :
491 1 : static int init_vq(struct virtio_blk *vblk)
492 : {
493 1 : int err;
494 1 : int i;
495 1 : vq_callback_t **callbacks;
496 1 : const char **names;
497 1 : struct virtqueue **vqs;
498 1 : unsigned short num_vqs;
499 1 : struct virtio_device *vdev = vblk->vdev;
500 1 : struct irq_affinity desc = { 0, };
501 :
502 1 : err = virtio_cread_feature(vdev, VIRTIO_BLK_F_MQ,
503 : struct virtio_blk_config, num_queues,
504 : &num_vqs);
505 0 : if (err)
506 : num_vqs = 1;
507 :
508 1 : num_vqs = min_t(unsigned int, nr_cpu_ids, num_vqs);
509 :
510 1 : vblk->vqs = kmalloc_array(num_vqs, sizeof(*vblk->vqs), GFP_KERNEL);
511 1 : if (!vblk->vqs)
512 : return -ENOMEM;
513 :
514 1 : names = kmalloc_array(num_vqs, sizeof(*names), GFP_KERNEL);
515 1 : callbacks = kmalloc_array(num_vqs, sizeof(*callbacks), GFP_KERNEL);
516 1 : vqs = kmalloc_array(num_vqs, sizeof(*vqs), GFP_KERNEL);
517 1 : if (!names || !callbacks || !vqs) {
518 0 : err = -ENOMEM;
519 0 : goto out;
520 : }
521 :
522 2 : for (i = 0; i < num_vqs; i++) {
523 1 : callbacks[i] = virtblk_done;
524 1 : snprintf(vblk->vqs[i].name, VQ_NAME_LEN, "req.%d", i);
525 1 : names[i] = vblk->vqs[i].name;
526 : }
527 :
528 : /* Discover virtqueues and write information to configuration. */
529 1 : err = virtio_find_vqs(vdev, num_vqs, vqs, callbacks, names, &desc);
530 1 : if (err)
531 0 : goto out;
532 :
533 2 : for (i = 0; i < num_vqs; i++) {
534 1 : spin_lock_init(&vblk->vqs[i].lock);
535 1 : vblk->vqs[i].vq = vqs[i];
536 : }
537 1 : vblk->num_vqs = num_vqs;
538 :
539 1 : out:
540 1 : kfree(vqs);
541 1 : kfree(callbacks);
542 1 : kfree(names);
543 1 : if (err)
544 0 : kfree(vblk->vqs);
545 : return err;
546 : }
547 :
548 : /*
549 : * Legacy naming scheme used for virtio devices. We are stuck with it for
550 : * virtio blk but don't ever use it for any new driver.
551 : */
552 1 : static int virtblk_name_format(char *prefix, int index, char *buf, int buflen)
553 : {
554 1 : const int base = 'z' - 'a' + 1;
555 1 : char *begin = buf + strlen(prefix);
556 1 : char *end = buf + buflen;
557 1 : char *p;
558 1 : int unit;
559 :
560 1 : p = end - 1;
561 1 : *p = '\0';
562 1 : unit = base;
563 1 : do {
564 1 : if (p == begin)
565 : return -EINVAL;
566 1 : *--p = 'a' + (index % unit);
567 1 : index = (index / unit) - 1;
568 1 : } while (index >= 0);
569 :
570 1 : memmove(begin, p, end - p);
571 1 : memcpy(buf, prefix, strlen(prefix));
572 :
573 1 : return 0;
574 : }
575 :
576 1 : static int virtblk_get_cache_mode(struct virtio_device *vdev)
577 : {
578 1 : u8 writeback;
579 1 : int err;
580 :
581 1 : err = virtio_cread_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE,
582 : struct virtio_blk_config, wce,
583 : &writeback);
584 :
585 : /*
586 : * If WCE is not configurable and flush is not available,
587 : * assume no writeback cache is in use.
588 : */
589 1 : if (err)
590 0 : writeback = virtio_has_feature(vdev, VIRTIO_BLK_F_FLUSH);
591 :
592 1 : return writeback;
593 : }
594 :
595 1 : static void virtblk_update_cache_mode(struct virtio_device *vdev)
596 : {
597 1 : u8 writeback = virtblk_get_cache_mode(vdev);
598 1 : struct virtio_blk *vblk = vdev->priv;
599 :
600 1 : blk_queue_write_cache(vblk->disk->queue, writeback, false);
601 1 : }
602 :
603 : static const char *const virtblk_cache_types[] = {
604 : "write through", "write back"
605 : };
606 :
607 : static ssize_t
608 0 : cache_type_store(struct device *dev, struct device_attribute *attr,
609 : const char *buf, size_t count)
610 : {
611 0 : struct gendisk *disk = dev_to_disk(dev);
612 0 : struct virtio_blk *vblk = disk->private_data;
613 0 : struct virtio_device *vdev = vblk->vdev;
614 0 : int i;
615 :
616 0 : BUG_ON(!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_CONFIG_WCE));
617 0 : i = sysfs_match_string(virtblk_cache_types, buf);
618 0 : if (i < 0)
619 0 : return i;
620 :
621 0 : virtio_cwrite8(vdev, offsetof(struct virtio_blk_config, wce), i);
622 0 : virtblk_update_cache_mode(vdev);
623 0 : return count;
624 : }
625 :
626 : static ssize_t
627 0 : cache_type_show(struct device *dev, struct device_attribute *attr, char *buf)
628 : {
629 0 : struct gendisk *disk = dev_to_disk(dev);
630 0 : struct virtio_blk *vblk = disk->private_data;
631 0 : u8 writeback = virtblk_get_cache_mode(vblk->vdev);
632 :
633 0 : BUG_ON(writeback >= ARRAY_SIZE(virtblk_cache_types));
634 0 : return snprintf(buf, 40, "%s\n", virtblk_cache_types[writeback]);
635 : }
636 :
637 : static DEVICE_ATTR_RW(cache_type);
638 :
639 : static struct attribute *virtblk_attrs[] = {
640 : &dev_attr_serial.attr,
641 : &dev_attr_cache_type.attr,
642 : NULL,
643 : };
644 :
645 2 : static umode_t virtblk_attrs_are_visible(struct kobject *kobj,
646 : struct attribute *a, int n)
647 : {
648 2 : struct device *dev = kobj_to_dev(kobj);
649 2 : struct gendisk *disk = dev_to_disk(dev);
650 2 : struct virtio_blk *vblk = disk->private_data;
651 2 : struct virtio_device *vdev = vblk->vdev;
652 :
653 3 : if (a == &dev_attr_cache_type.attr &&
654 1 : !virtio_has_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE))
655 : return S_IRUGO;
656 :
657 2 : return a->mode;
658 : }
659 :
660 : static const struct attribute_group virtblk_attr_group = {
661 : .attrs = virtblk_attrs,
662 : .is_visible = virtblk_attrs_are_visible,
663 : };
664 :
665 : static const struct attribute_group *virtblk_attr_groups[] = {
666 : &virtblk_attr_group,
667 : NULL,
668 : };
669 :
670 1025 : static int virtblk_init_request(struct blk_mq_tag_set *set, struct request *rq,
671 : unsigned int hctx_idx, unsigned int numa_node)
672 : {
673 1025 : struct virtio_blk *vblk = set->driver_data;
674 1025 : struct virtblk_req *vbr = blk_mq_rq_to_pdu(rq);
675 :
676 1025 : sg_init_table(vbr->sg, vblk->sg_elems);
677 1025 : return 0;
678 : }
679 :
680 1 : static int virtblk_map_queues(struct blk_mq_tag_set *set)
681 : {
682 1 : struct virtio_blk *vblk = set->driver_data;
683 :
684 1 : return blk_mq_virtio_map_queues(&set->map[HCTX_TYPE_DEFAULT],
685 : vblk->vdev, 0);
686 : }
687 :
688 : static const struct blk_mq_ops virtio_mq_ops = {
689 : .queue_rq = virtio_queue_rq,
690 : .commit_rqs = virtio_commit_rqs,
691 : .complete = virtblk_request_done,
692 : .init_request = virtblk_init_request,
693 : .map_queues = virtblk_map_queues,
694 : };
695 :
696 : static unsigned int virtblk_queue_depth;
697 : module_param_named(queue_depth, virtblk_queue_depth, uint, 0444);
698 :
699 1 : static int virtblk_probe(struct virtio_device *vdev)
700 : {
701 1 : struct virtio_blk *vblk;
702 1 : struct request_queue *q;
703 1 : int err, index;
704 :
705 1 : u32 v, blk_size, max_size, sg_elems, opt_io_size;
706 1 : u16 min_io_size;
707 1 : u8 physical_block_exp, alignment_offset;
708 1 : unsigned int queue_depth;
709 :
710 1 : if (!vdev->config->get) {
711 0 : dev_err(&vdev->dev, "%s failure: config access disabled\n",
712 : __func__);
713 0 : return -EINVAL;
714 : }
715 :
716 1 : err = ida_simple_get(&vd_index_ida, 0, minor_to_index(1 << MINORBITS),
717 : GFP_KERNEL);
718 1 : if (err < 0)
719 0 : goto out;
720 1 : index = err;
721 :
722 : /* We need to know how many segments before we allocate. */
723 1 : err = virtio_cread_feature(vdev, VIRTIO_BLK_F_SEG_MAX,
724 : struct virtio_blk_config, seg_max,
725 : &sg_elems);
726 :
727 : /* We need at least one SG element, whatever they say. */
728 1 : if (err || !sg_elems)
729 : sg_elems = 1;
730 :
731 : /* We need an extra sg elements at head and tail. */
732 1 : sg_elems += 2;
733 1 : vdev->priv = vblk = kmalloc(sizeof(*vblk), GFP_KERNEL);
734 1 : if (!vblk) {
735 0 : err = -ENOMEM;
736 0 : goto out_free_index;
737 : }
738 :
739 : /* This reference is dropped in virtblk_remove(). */
740 1 : refcount_set(&vblk->refs, 1);
741 1 : mutex_init(&vblk->vdev_mutex);
742 :
743 1 : vblk->vdev = vdev;
744 1 : vblk->sg_elems = sg_elems;
745 :
746 1 : INIT_WORK(&vblk->config_work, virtblk_config_changed_work);
747 :
748 1 : err = init_vq(vblk);
749 1 : if (err)
750 0 : goto out_free_vblk;
751 :
752 : /* FIXME: How many partitions? How long is a piece of string? */
753 1 : vblk->disk = alloc_disk(1 << PART_BITS);
754 1 : if (!vblk->disk) {
755 0 : err = -ENOMEM;
756 0 : goto out_free_vq;
757 : }
758 :
759 : /* Default queue sizing is to fill the ring. */
760 1 : if (likely(!virtblk_queue_depth)) {
761 1 : queue_depth = vblk->vqs[0].vq->num_free;
762 : /* ... but without indirect descs, we use 2 descs per req */
763 1 : if (!virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC))
764 0 : queue_depth /= 2;
765 : } else {
766 : queue_depth = virtblk_queue_depth;
767 : }
768 :
769 1 : memset(&vblk->tag_set, 0, sizeof(vblk->tag_set));
770 1 : vblk->tag_set.ops = &virtio_mq_ops;
771 1 : vblk->tag_set.queue_depth = queue_depth;
772 1 : vblk->tag_set.numa_node = NUMA_NO_NODE;
773 1 : vblk->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
774 1 : vblk->tag_set.cmd_size =
775 1 : sizeof(struct virtblk_req) +
776 1 : sizeof(struct scatterlist) * sg_elems;
777 1 : vblk->tag_set.driver_data = vblk;
778 1 : vblk->tag_set.nr_hw_queues = vblk->num_vqs;
779 :
780 1 : err = blk_mq_alloc_tag_set(&vblk->tag_set);
781 1 : if (err)
782 0 : goto out_put_disk;
783 :
784 1 : q = blk_mq_init_queue(&vblk->tag_set);
785 1 : if (IS_ERR(q)) {
786 0 : err = -ENOMEM;
787 0 : goto out_free_tags;
788 : }
789 1 : vblk->disk->queue = q;
790 :
791 1 : q->queuedata = vblk;
792 :
793 1 : virtblk_name_format("vd", index, vblk->disk->disk_name, DISK_NAME_LEN);
794 :
795 1 : vblk->disk->major = major;
796 1 : vblk->disk->first_minor = index_to_minor(index);
797 1 : vblk->disk->private_data = vblk;
798 1 : vblk->disk->fops = &virtblk_fops;
799 1 : vblk->disk->flags |= GENHD_FL_EXT_DEVT;
800 1 : vblk->index = index;
801 :
802 : /* configure queue flush support */
803 1 : virtblk_update_cache_mode(vdev);
804 :
805 : /* If disk is read-only in the host, the guest should obey */
806 1 : if (virtio_has_feature(vdev, VIRTIO_BLK_F_RO))
807 0 : set_disk_ro(vblk->disk, 1);
808 :
809 : /* We can handle whatever the host told us to handle. */
810 1 : blk_queue_max_segments(q, vblk->sg_elems-2);
811 :
812 : /* No real sector limit. */
813 1 : blk_queue_max_hw_sectors(q, -1U);
814 :
815 1 : max_size = virtio_max_dma_size(vdev);
816 :
817 : /* Host can optionally specify maximum segment size and number of
818 : * segments. */
819 1 : err = virtio_cread_feature(vdev, VIRTIO_BLK_F_SIZE_MAX,
820 : struct virtio_blk_config, size_max, &v);
821 0 : if (!err)
822 0 : max_size = min(max_size, v);
823 :
824 1 : blk_queue_max_segment_size(q, max_size);
825 :
826 : /* Host can optionally specify the block size of the device */
827 1 : err = virtio_cread_feature(vdev, VIRTIO_BLK_F_BLK_SIZE,
828 : struct virtio_blk_config, blk_size,
829 : &blk_size);
830 1 : if (!err)
831 1 : blk_queue_logical_block_size(q, blk_size);
832 : else
833 0 : blk_size = queue_logical_block_size(q);
834 :
835 : /* Use topology information if available */
836 1 : err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY,
837 : struct virtio_blk_config, physical_block_exp,
838 : &physical_block_exp);
839 1 : if (!err && physical_block_exp)
840 0 : blk_queue_physical_block_size(q,
841 : blk_size * (1 << physical_block_exp));
842 :
843 1 : err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY,
844 : struct virtio_blk_config, alignment_offset,
845 : &alignment_offset);
846 1 : if (!err && alignment_offset)
847 0 : blk_queue_alignment_offset(q, blk_size * alignment_offset);
848 :
849 1 : err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY,
850 : struct virtio_blk_config, min_io_size,
851 : &min_io_size);
852 1 : if (!err && min_io_size)
853 0 : blk_queue_io_min(q, blk_size * min_io_size);
854 :
855 1 : err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY,
856 : struct virtio_blk_config, opt_io_size,
857 : &opt_io_size);
858 1 : if (!err && opt_io_size)
859 0 : blk_queue_io_opt(q, blk_size * opt_io_size);
860 :
861 1 : if (virtio_has_feature(vdev, VIRTIO_BLK_F_DISCARD)) {
862 1 : q->limits.discard_granularity = blk_size;
863 :
864 1 : virtio_cread(vdev, struct virtio_blk_config,
865 : discard_sector_alignment, &v);
866 1 : q->limits.discard_alignment = v ? v << SECTOR_SHIFT : 0;
867 :
868 1 : virtio_cread(vdev, struct virtio_blk_config,
869 : max_discard_sectors, &v);
870 1 : blk_queue_max_discard_sectors(q, v ? v : UINT_MAX);
871 :
872 1 : virtio_cread(vdev, struct virtio_blk_config, max_discard_seg,
873 : &v);
874 1 : blk_queue_max_discard_segments(q,
875 1 : min_not_zero(v,
876 : MAX_DISCARD_SEGMENTS));
877 :
878 1 : blk_queue_flag_set(QUEUE_FLAG_DISCARD, q);
879 : }
880 :
881 1 : if (virtio_has_feature(vdev, VIRTIO_BLK_F_WRITE_ZEROES)) {
882 1 : virtio_cread(vdev, struct virtio_blk_config,
883 : max_write_zeroes_sectors, &v);
884 1 : blk_queue_max_write_zeroes_sectors(q, v ? v : UINT_MAX);
885 : }
886 :
887 1 : virtblk_update_capacity(vblk, false);
888 1 : virtio_device_ready(vdev);
889 :
890 1 : device_add_disk(&vdev->dev, vblk->disk, virtblk_attr_groups);
891 1 : return 0;
892 :
893 0 : out_free_tags:
894 0 : blk_mq_free_tag_set(&vblk->tag_set);
895 0 : out_put_disk:
896 0 : put_disk(vblk->disk);
897 0 : out_free_vq:
898 0 : vdev->config->del_vqs(vdev);
899 0 : kfree(vblk->vqs);
900 0 : out_free_vblk:
901 0 : kfree(vblk);
902 0 : out_free_index:
903 0 : ida_simple_remove(&vd_index_ida, index);
904 : out:
905 : return err;
906 : }
907 :
908 0 : static void virtblk_remove(struct virtio_device *vdev)
909 : {
910 0 : struct virtio_blk *vblk = vdev->priv;
911 :
912 : /* Make sure no work handler is accessing the device. */
913 0 : flush_work(&vblk->config_work);
914 :
915 0 : del_gendisk(vblk->disk);
916 0 : blk_cleanup_queue(vblk->disk->queue);
917 :
918 0 : blk_mq_free_tag_set(&vblk->tag_set);
919 :
920 0 : mutex_lock(&vblk->vdev_mutex);
921 :
922 : /* Stop all the virtqueues. */
923 0 : vdev->config->reset(vdev);
924 :
925 : /* Virtqueues are stopped, nothing can use vblk->vdev anymore. */
926 0 : vblk->vdev = NULL;
927 :
928 0 : put_disk(vblk->disk);
929 0 : vdev->config->del_vqs(vdev);
930 0 : kfree(vblk->vqs);
931 :
932 0 : mutex_unlock(&vblk->vdev_mutex);
933 :
934 0 : virtblk_put(vblk);
935 0 : }
936 :
937 : #ifdef CONFIG_PM_SLEEP
938 : static int virtblk_freeze(struct virtio_device *vdev)
939 : {
940 : struct virtio_blk *vblk = vdev->priv;
941 :
942 : /* Ensure we don't receive any more interrupts */
943 : vdev->config->reset(vdev);
944 :
945 : /* Make sure no work handler is accessing the device. */
946 : flush_work(&vblk->config_work);
947 :
948 : blk_mq_quiesce_queue(vblk->disk->queue);
949 :
950 : vdev->config->del_vqs(vdev);
951 : return 0;
952 : }
953 :
954 : static int virtblk_restore(struct virtio_device *vdev)
955 : {
956 : struct virtio_blk *vblk = vdev->priv;
957 : int ret;
958 :
959 : ret = init_vq(vdev->priv);
960 : if (ret)
961 : return ret;
962 :
963 : virtio_device_ready(vdev);
964 :
965 : blk_mq_unquiesce_queue(vblk->disk->queue);
966 : return 0;
967 : }
968 : #endif
969 :
970 : static const struct virtio_device_id id_table[] = {
971 : { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID },
972 : { 0 },
973 : };
974 :
975 : static unsigned int features_legacy[] = {
976 : VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY,
977 : VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE,
978 : VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE,
979 : VIRTIO_BLK_F_MQ, VIRTIO_BLK_F_DISCARD, VIRTIO_BLK_F_WRITE_ZEROES,
980 : }
981 : ;
982 : static unsigned int features[] = {
983 : VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY,
984 : VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE,
985 : VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE,
986 : VIRTIO_BLK_F_MQ, VIRTIO_BLK_F_DISCARD, VIRTIO_BLK_F_WRITE_ZEROES,
987 : };
988 :
989 : static struct virtio_driver virtio_blk = {
990 : .feature_table = features,
991 : .feature_table_size = ARRAY_SIZE(features),
992 : .feature_table_legacy = features_legacy,
993 : .feature_table_size_legacy = ARRAY_SIZE(features_legacy),
994 : .driver.name = KBUILD_MODNAME,
995 : .driver.owner = THIS_MODULE,
996 : .id_table = id_table,
997 : .probe = virtblk_probe,
998 : .remove = virtblk_remove,
999 : .config_changed = virtblk_config_changed,
1000 : #ifdef CONFIG_PM_SLEEP
1001 : .freeze = virtblk_freeze,
1002 : .restore = virtblk_restore,
1003 : #endif
1004 : };
1005 :
1006 1 : static int __init init(void)
1007 : {
1008 1 : int error;
1009 :
1010 1 : virtblk_wq = alloc_workqueue("virtio-blk", 0, 0);
1011 1 : if (!virtblk_wq)
1012 : return -ENOMEM;
1013 :
1014 1 : major = register_blkdev(0, "virtblk");
1015 1 : if (major < 0) {
1016 0 : error = major;
1017 0 : goto out_destroy_workqueue;
1018 : }
1019 :
1020 1 : error = register_virtio_driver(&virtio_blk);
1021 1 : if (error)
1022 0 : goto out_unregister_blkdev;
1023 : return 0;
1024 :
1025 0 : out_unregister_blkdev:
1026 0 : unregister_blkdev(major, "virtblk");
1027 0 : out_destroy_workqueue:
1028 0 : destroy_workqueue(virtblk_wq);
1029 0 : return error;
1030 : }
1031 :
1032 0 : static void __exit fini(void)
1033 : {
1034 0 : unregister_virtio_driver(&virtio_blk);
1035 0 : unregister_blkdev(major, "virtblk");
1036 0 : destroy_workqueue(virtblk_wq);
1037 0 : }
1038 : module_init(init);
1039 : module_exit(fini);
1040 :
1041 : MODULE_DEVICE_TABLE(virtio, id_table);
1042 : MODULE_DESCRIPTION("Virtio block driver");
1043 : MODULE_LICENSE("GPL");
|
