Line data Source code
1 : /*
2 : * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3 : * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
4 : *
5 : * This file is released under the GPL.
6 : */
7 :
8 : #include "dm-core.h"
9 : #include "dm-rq.h"
10 : #include "dm-uevent.h"
11 :
12 : #include <linux/init.h>
13 : #include <linux/module.h>
14 : #include <linux/mutex.h>
15 : #include <linux/sched/mm.h>
16 : #include <linux/sched/signal.h>
17 : #include <linux/blkpg.h>
18 : #include <linux/bio.h>
19 : #include <linux/mempool.h>
20 : #include <linux/dax.h>
21 : #include <linux/slab.h>
22 : #include <linux/idr.h>
23 : #include <linux/uio.h>
24 : #include <linux/hdreg.h>
25 : #include <linux/delay.h>
26 : #include <linux/wait.h>
27 : #include <linux/pr.h>
28 : #include <linux/refcount.h>
29 : #include <linux/part_stat.h>
30 : #include <linux/blk-crypto.h>
31 : #include <linux/keyslot-manager.h>
32 :
33 : #define DM_MSG_PREFIX "core"
34 :
35 : /*
36 : * Cookies are numeric values sent with CHANGE and REMOVE
37 : * uevents while resuming, removing or renaming the device.
38 : */
39 : #define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE"
40 : #define DM_COOKIE_LENGTH 24
41 :
42 : static const char *_name = DM_NAME;
43 :
44 : static unsigned int major = 0;
45 : static unsigned int _major = 0;
46 :
47 : static DEFINE_IDR(_minor_idr);
48 :
49 : static DEFINE_SPINLOCK(_minor_lock);
50 :
51 : static void do_deferred_remove(struct work_struct *w);
52 :
53 : static DECLARE_WORK(deferred_remove_work, do_deferred_remove);
54 :
55 : static struct workqueue_struct *deferred_remove_workqueue;
56 :
57 : atomic_t dm_global_event_nr = ATOMIC_INIT(0);
58 : DECLARE_WAIT_QUEUE_HEAD(dm_global_eventq);
59 :
60 0 : void dm_issue_global_event(void)
61 : {
62 0 : atomic_inc(&dm_global_event_nr);
63 0 : wake_up(&dm_global_eventq);
64 0 : }
65 :
66 : /*
67 : * One of these is allocated (on-stack) per original bio.
68 : */
69 : struct clone_info {
70 : struct dm_table *map;
71 : struct bio *bio;
72 : struct dm_io *io;
73 : sector_t sector;
74 : unsigned sector_count;
75 : };
76 :
77 : /*
78 : * One of these is allocated per clone bio.
79 : */
80 : #define DM_TIO_MAGIC 7282014
81 : struct dm_target_io {
82 : unsigned magic;
83 : struct dm_io *io;
84 : struct dm_target *ti;
85 : unsigned target_bio_nr;
86 : unsigned *len_ptr;
87 : bool inside_dm_io;
88 : struct bio clone;
89 : };
90 :
91 : /*
92 : * One of these is allocated per original bio.
93 : * It contains the first clone used for that original.
94 : */
95 : #define DM_IO_MAGIC 5191977
96 : struct dm_io {
97 : unsigned magic;
98 : struct mapped_device *md;
99 : blk_status_t status;
100 : atomic_t io_count;
101 : struct bio *orig_bio;
102 : unsigned long start_time;
103 : spinlock_t endio_lock;
104 : struct dm_stats_aux stats_aux;
105 : /* last member of dm_target_io is 'struct bio' */
106 : struct dm_target_io tio;
107 : };
108 :
109 : #define DM_TARGET_IO_BIO_OFFSET (offsetof(struct dm_target_io, clone))
110 : #define DM_IO_BIO_OFFSET \
111 : (offsetof(struct dm_target_io, clone) + offsetof(struct dm_io, tio))
112 :
113 0 : void *dm_per_bio_data(struct bio *bio, size_t data_size)
114 : {
115 0 : struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
116 0 : if (!tio->inside_dm_io)
117 0 : return (char *)bio - DM_TARGET_IO_BIO_OFFSET - data_size;
118 0 : return (char *)bio - DM_IO_BIO_OFFSET - data_size;
119 : }
120 : EXPORT_SYMBOL_GPL(dm_per_bio_data);
121 :
122 0 : struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size)
123 : {
124 0 : struct dm_io *io = (struct dm_io *)((char *)data + data_size);
125 0 : if (io->magic == DM_IO_MAGIC)
126 0 : return (struct bio *)((char *)io + DM_IO_BIO_OFFSET);
127 0 : BUG_ON(io->magic != DM_TIO_MAGIC);
128 0 : return (struct bio *)((char *)io + DM_TARGET_IO_BIO_OFFSET);
129 : }
130 : EXPORT_SYMBOL_GPL(dm_bio_from_per_bio_data);
131 :
132 0 : unsigned dm_bio_get_target_bio_nr(const struct bio *bio)
133 : {
134 0 : return container_of(bio, struct dm_target_io, clone)->target_bio_nr;
135 : }
136 : EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr);
137 :
138 : #define MINOR_ALLOCED ((void *)-1)
139 :
140 : /*
141 : * Bits for the md->flags field.
142 : */
143 : #define DMF_BLOCK_IO_FOR_SUSPEND 0
144 : #define DMF_SUSPENDED 1
145 : #define DMF_FROZEN 2
146 : #define DMF_FREEING 3
147 : #define DMF_DELETING 4
148 : #define DMF_NOFLUSH_SUSPENDING 5
149 : #define DMF_DEFERRED_REMOVE 6
150 : #define DMF_SUSPENDED_INTERNALLY 7
151 : #define DMF_POST_SUSPENDING 8
152 :
153 : #define DM_NUMA_NODE NUMA_NO_NODE
154 : static int dm_numa_node = DM_NUMA_NODE;
155 :
156 : #define DEFAULT_SWAP_BIOS (8 * 1048576 / PAGE_SIZE)
157 : static int swap_bios = DEFAULT_SWAP_BIOS;
158 0 : static int get_swap_bios(void)
159 : {
160 0 : int latch = READ_ONCE(swap_bios);
161 0 : if (unlikely(latch <= 0))
162 0 : latch = DEFAULT_SWAP_BIOS;
163 0 : return latch;
164 : }
165 :
166 : /*
167 : * For mempools pre-allocation at the table loading time.
168 : */
169 : struct dm_md_mempools {
170 : struct bio_set bs;
171 : struct bio_set io_bs;
172 : };
173 :
174 : struct table_device {
175 : struct list_head list;
176 : refcount_t count;
177 : struct dm_dev dm_dev;
178 : };
179 :
180 : /*
181 : * Bio-based DM's mempools' reserved IOs set by the user.
182 : */
183 : #define RESERVED_BIO_BASED_IOS 16
184 : static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS;
185 :
186 0 : static int __dm_get_module_param_int(int *module_param, int min, int max)
187 : {
188 0 : int param = READ_ONCE(*module_param);
189 0 : int modified_param = 0;
190 0 : bool modified = true;
191 :
192 0 : if (param < min)
193 : modified_param = min;
194 0 : else if (param > max)
195 : modified_param = max;
196 : else
197 : modified = false;
198 :
199 : if (modified) {
200 0 : (void)cmpxchg(module_param, param, modified_param);
201 0 : param = modified_param;
202 : }
203 :
204 0 : return param;
205 : }
206 :
207 0 : unsigned __dm_get_module_param(unsigned *module_param,
208 : unsigned def, unsigned max)
209 : {
210 0 : unsigned param = READ_ONCE(*module_param);
211 0 : unsigned modified_param = 0;
212 :
213 0 : if (!param)
214 : modified_param = def;
215 0 : else if (param > max)
216 : modified_param = max;
217 :
218 0 : if (modified_param) {
219 0 : (void)cmpxchg(module_param, param, modified_param);
220 0 : param = modified_param;
221 : }
222 :
223 0 : return param;
224 : }
225 :
226 0 : unsigned dm_get_reserved_bio_based_ios(void)
227 : {
228 0 : return __dm_get_module_param(&reserved_bio_based_ios,
229 : RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS);
230 : }
231 : EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios);
232 :
233 0 : static unsigned dm_get_numa_node(void)
234 : {
235 0 : return __dm_get_module_param_int(&dm_numa_node,
236 0 : DM_NUMA_NODE, num_online_nodes() - 1);
237 : }
238 :
239 1 : static int __init local_init(void)
240 : {
241 1 : int r;
242 :
243 1 : r = dm_uevent_init();
244 1 : if (r)
245 : return r;
246 :
247 1 : deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1);
248 1 : if (!deferred_remove_workqueue) {
249 0 : r = -ENOMEM;
250 0 : goto out_uevent_exit;
251 : }
252 :
253 1 : _major = major;
254 1 : r = register_blkdev(_major, _name);
255 1 : if (r < 0)
256 0 : goto out_free_workqueue;
257 :
258 1 : if (!_major)
259 1 : _major = r;
260 :
261 : return 0;
262 :
263 0 : out_free_workqueue:
264 0 : destroy_workqueue(deferred_remove_workqueue);
265 : out_uevent_exit:
266 : dm_uevent_exit();
267 :
268 : return r;
269 : }
270 :
271 0 : static void local_exit(void)
272 : {
273 0 : flush_scheduled_work();
274 0 : destroy_workqueue(deferred_remove_workqueue);
275 :
276 0 : unregister_blkdev(_major, _name);
277 0 : dm_uevent_exit();
278 :
279 0 : _major = 0;
280 :
281 0 : DMINFO("cleaned up");
282 0 : }
283 :
284 : static int (*_inits[])(void) __initdata = {
285 : local_init,
286 : dm_target_init,
287 : dm_linear_init,
288 : dm_stripe_init,
289 : dm_io_init,
290 : dm_kcopyd_init,
291 : dm_interface_init,
292 : dm_statistics_init,
293 : };
294 :
295 : static void (*_exits[])(void) = {
296 : local_exit,
297 : dm_target_exit,
298 : dm_linear_exit,
299 : dm_stripe_exit,
300 : dm_io_exit,
301 : dm_kcopyd_exit,
302 : dm_interface_exit,
303 : dm_statistics_exit,
304 : };
305 :
306 1 : static int __init dm_init(void)
307 : {
308 1 : const int count = ARRAY_SIZE(_inits);
309 :
310 1 : int r, i;
311 :
312 9 : for (i = 0; i < count; i++) {
313 8 : r = _inits[i]();
314 8 : if (r)
315 0 : goto bad;
316 : }
317 :
318 : return 0;
319 :
320 0 : bad:
321 0 : while (i--)
322 0 : _exits[i]();
323 :
324 : return r;
325 : }
326 :
327 0 : static void __exit dm_exit(void)
328 : {
329 0 : int i = ARRAY_SIZE(_exits);
330 :
331 0 : while (i--)
332 0 : _exits[i]();
333 :
334 : /*
335 : * Should be empty by this point.
336 : */
337 0 : idr_destroy(&_minor_idr);
338 0 : }
339 :
340 : /*
341 : * Block device functions
342 : */
343 0 : int dm_deleting_md(struct mapped_device *md)
344 : {
345 0 : return test_bit(DMF_DELETING, &md->flags);
346 : }
347 :
348 0 : static int dm_blk_open(struct block_device *bdev, fmode_t mode)
349 : {
350 0 : struct mapped_device *md;
351 :
352 0 : spin_lock(&_minor_lock);
353 :
354 0 : md = bdev->bd_disk->private_data;
355 0 : if (!md)
356 0 : goto out;
357 :
358 0 : if (test_bit(DMF_FREEING, &md->flags) ||
359 0 : dm_deleting_md(md)) {
360 0 : md = NULL;
361 0 : goto out;
362 : }
363 :
364 0 : dm_get(md);
365 0 : atomic_inc(&md->open_count);
366 0 : out:
367 0 : spin_unlock(&_minor_lock);
368 :
369 0 : return md ? 0 : -ENXIO;
370 : }
371 :
372 0 : static void dm_blk_close(struct gendisk *disk, fmode_t mode)
373 : {
374 0 : struct mapped_device *md;
375 :
376 0 : spin_lock(&_minor_lock);
377 :
378 0 : md = disk->private_data;
379 0 : if (WARN_ON(!md))
380 0 : goto out;
381 :
382 0 : if (atomic_dec_and_test(&md->open_count) &&
383 0 : (test_bit(DMF_DEFERRED_REMOVE, &md->flags)))
384 0 : queue_work(deferred_remove_workqueue, &deferred_remove_work);
385 :
386 0 : dm_put(md);
387 0 : out:
388 0 : spin_unlock(&_minor_lock);
389 0 : }
390 :
391 0 : int dm_open_count(struct mapped_device *md)
392 : {
393 0 : return atomic_read(&md->open_count);
394 : }
395 :
396 : /*
397 : * Guarantees nothing is using the device before it's deleted.
398 : */
399 0 : int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred)
400 : {
401 0 : int r = 0;
402 :
403 0 : spin_lock(&_minor_lock);
404 :
405 0 : if (dm_open_count(md)) {
406 0 : r = -EBUSY;
407 0 : if (mark_deferred)
408 0 : set_bit(DMF_DEFERRED_REMOVE, &md->flags);
409 0 : } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags))
410 : r = -EEXIST;
411 : else
412 0 : set_bit(DMF_DELETING, &md->flags);
413 :
414 0 : spin_unlock(&_minor_lock);
415 :
416 0 : return r;
417 : }
418 :
419 0 : int dm_cancel_deferred_remove(struct mapped_device *md)
420 : {
421 0 : int r = 0;
422 :
423 0 : spin_lock(&_minor_lock);
424 :
425 0 : if (test_bit(DMF_DELETING, &md->flags))
426 : r = -EBUSY;
427 : else
428 0 : clear_bit(DMF_DEFERRED_REMOVE, &md->flags);
429 :
430 0 : spin_unlock(&_minor_lock);
431 :
432 0 : return r;
433 : }
434 :
435 0 : static void do_deferred_remove(struct work_struct *w)
436 : {
437 0 : dm_deferred_remove();
438 0 : }
439 :
440 0 : static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
441 : {
442 0 : struct mapped_device *md = bdev->bd_disk->private_data;
443 :
444 0 : return dm_get_geometry(md, geo);
445 : }
446 :
447 : #ifdef CONFIG_BLK_DEV_ZONED
448 : int dm_report_zones_cb(struct blk_zone *zone, unsigned int idx, void *data)
449 : {
450 : struct dm_report_zones_args *args = data;
451 : sector_t sector_diff = args->tgt->begin - args->start;
452 :
453 : /*
454 : * Ignore zones beyond the target range.
455 : */
456 : if (zone->start >= args->start + args->tgt->len)
457 : return 0;
458 :
459 : /*
460 : * Remap the start sector and write pointer position of the zone
461 : * to match its position in the target range.
462 : */
463 : zone->start += sector_diff;
464 : if (zone->type != BLK_ZONE_TYPE_CONVENTIONAL) {
465 : if (zone->cond == BLK_ZONE_COND_FULL)
466 : zone->wp = zone->start + zone->len;
467 : else if (zone->cond == BLK_ZONE_COND_EMPTY)
468 : zone->wp = zone->start;
469 : else
470 : zone->wp += sector_diff;
471 : }
472 :
473 : args->next_sector = zone->start + zone->len;
474 : return args->orig_cb(zone, args->zone_idx++, args->orig_data);
475 : }
476 : EXPORT_SYMBOL_GPL(dm_report_zones_cb);
477 :
478 : static int dm_blk_report_zones(struct gendisk *disk, sector_t sector,
479 : unsigned int nr_zones, report_zones_cb cb, void *data)
480 : {
481 : struct mapped_device *md = disk->private_data;
482 : struct dm_table *map;
483 : int srcu_idx, ret;
484 : struct dm_report_zones_args args = {
485 : .next_sector = sector,
486 : .orig_data = data,
487 : .orig_cb = cb,
488 : };
489 :
490 : if (dm_suspended_md(md))
491 : return -EAGAIN;
492 :
493 : map = dm_get_live_table(md, &srcu_idx);
494 : if (!map) {
495 : ret = -EIO;
496 : goto out;
497 : }
498 :
499 : do {
500 : struct dm_target *tgt;
501 :
502 : tgt = dm_table_find_target(map, args.next_sector);
503 : if (WARN_ON_ONCE(!tgt->type->report_zones)) {
504 : ret = -EIO;
505 : goto out;
506 : }
507 :
508 : args.tgt = tgt;
509 : ret = tgt->type->report_zones(tgt, &args,
510 : nr_zones - args.zone_idx);
511 : if (ret < 0)
512 : goto out;
513 : } while (args.zone_idx < nr_zones &&
514 : args.next_sector < get_capacity(disk));
515 :
516 : ret = args.zone_idx;
517 : out:
518 : dm_put_live_table(md, srcu_idx);
519 : return ret;
520 : }
521 : #else
522 : #define dm_blk_report_zones NULL
523 : #endif /* CONFIG_BLK_DEV_ZONED */
524 :
525 0 : static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx,
526 : struct block_device **bdev)
527 : {
528 0 : struct dm_target *tgt;
529 0 : struct dm_table *map;
530 0 : int r;
531 :
532 0 : retry:
533 0 : r = -ENOTTY;
534 0 : map = dm_get_live_table(md, srcu_idx);
535 0 : if (!map || !dm_table_get_size(map))
536 0 : return r;
537 :
538 : /* We only support devices that have a single target */
539 0 : if (dm_table_get_num_targets(map) != 1)
540 : return r;
541 :
542 0 : tgt = dm_table_get_target(map, 0);
543 0 : if (!tgt->type->prepare_ioctl)
544 : return r;
545 :
546 0 : if (dm_suspended_md(md))
547 : return -EAGAIN;
548 :
549 0 : r = tgt->type->prepare_ioctl(tgt, bdev);
550 0 : if (r == -ENOTCONN && !fatal_signal_pending(current)) {
551 0 : dm_put_live_table(md, *srcu_idx);
552 0 : msleep(10);
553 0 : goto retry;
554 : }
555 :
556 : return r;
557 : }
558 :
559 0 : static void dm_unprepare_ioctl(struct mapped_device *md, int srcu_idx)
560 : {
561 0 : dm_put_live_table(md, srcu_idx);
562 : }
563 :
564 0 : static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
565 : unsigned int cmd, unsigned long arg)
566 : {
567 0 : struct mapped_device *md = bdev->bd_disk->private_data;
568 0 : int r, srcu_idx;
569 :
570 0 : r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
571 0 : if (r < 0)
572 0 : goto out;
573 :
574 0 : if (r > 0) {
575 : /*
576 : * Target determined this ioctl is being issued against a
577 : * subset of the parent bdev; require extra privileges.
578 : */
579 0 : if (!capable(CAP_SYS_RAWIO)) {
580 0 : DMDEBUG_LIMIT(
581 : "%s: sending ioctl %x to DM device without required privilege.",
582 : current->comm, cmd);
583 0 : r = -ENOIOCTLCMD;
584 0 : goto out;
585 : }
586 : }
587 :
588 0 : if (!bdev->bd_disk->fops->ioctl)
589 : r = -ENOTTY;
590 : else
591 0 : r = bdev->bd_disk->fops->ioctl(bdev, mode, cmd, arg);
592 0 : out:
593 0 : dm_unprepare_ioctl(md, srcu_idx);
594 0 : return r;
595 : }
596 :
597 0 : u64 dm_start_time_ns_from_clone(struct bio *bio)
598 : {
599 0 : struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
600 0 : struct dm_io *io = tio->io;
601 :
602 0 : return jiffies_to_nsecs(io->start_time);
603 : }
604 : EXPORT_SYMBOL_GPL(dm_start_time_ns_from_clone);
605 :
606 0 : static void start_io_acct(struct dm_io *io)
607 : {
608 0 : struct mapped_device *md = io->md;
609 0 : struct bio *bio = io->orig_bio;
610 :
611 0 : io->start_time = bio_start_io_acct(bio);
612 0 : if (unlikely(dm_stats_used(&md->stats)))
613 0 : dm_stats_account_io(&md->stats, bio_data_dir(bio),
614 0 : bio->bi_iter.bi_sector, bio_sectors(bio),
615 : false, 0, &io->stats_aux);
616 0 : }
617 :
618 0 : static void end_io_acct(struct dm_io *io)
619 : {
620 0 : struct mapped_device *md = io->md;
621 0 : struct bio *bio = io->orig_bio;
622 0 : unsigned long duration = jiffies - io->start_time;
623 :
624 0 : bio_end_io_acct(bio, io->start_time);
625 :
626 0 : if (unlikely(dm_stats_used(&md->stats)))
627 0 : dm_stats_account_io(&md->stats, bio_data_dir(bio),
628 0 : bio->bi_iter.bi_sector, bio_sectors(bio),
629 : true, duration, &io->stats_aux);
630 :
631 : /* nudge anyone waiting on suspend queue */
632 0 : if (unlikely(wq_has_sleeper(&md->wait)))
633 0 : wake_up(&md->wait);
634 0 : }
635 :
636 0 : static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio)
637 : {
638 0 : struct dm_io *io;
639 0 : struct dm_target_io *tio;
640 0 : struct bio *clone;
641 :
642 0 : clone = bio_alloc_bioset(GFP_NOIO, 0, &md->io_bs);
643 0 : if (!clone)
644 : return NULL;
645 :
646 0 : tio = container_of(clone, struct dm_target_io, clone);
647 0 : tio->inside_dm_io = true;
648 0 : tio->io = NULL;
649 :
650 0 : io = container_of(tio, struct dm_io, tio);
651 0 : io->magic = DM_IO_MAGIC;
652 0 : io->status = 0;
653 0 : atomic_set(&io->io_count, 1);
654 0 : io->orig_bio = bio;
655 0 : io->md = md;
656 0 : spin_lock_init(&io->endio_lock);
657 :
658 0 : start_io_acct(io);
659 :
660 0 : return io;
661 : }
662 :
663 0 : static void free_io(struct mapped_device *md, struct dm_io *io)
664 : {
665 0 : bio_put(&io->tio.clone);
666 : }
667 :
668 0 : static struct dm_target_io *alloc_tio(struct clone_info *ci, struct dm_target *ti,
669 : unsigned target_bio_nr, gfp_t gfp_mask)
670 : {
671 0 : struct dm_target_io *tio;
672 :
673 0 : if (!ci->io->tio.io) {
674 : /* the dm_target_io embedded in ci->io is available */
675 0 : tio = &ci->io->tio;
676 : } else {
677 0 : struct bio *clone = bio_alloc_bioset(gfp_mask, 0, &ci->io->md->bs);
678 0 : if (!clone)
679 : return NULL;
680 :
681 0 : tio = container_of(clone, struct dm_target_io, clone);
682 0 : tio->inside_dm_io = false;
683 : }
684 :
685 0 : tio->magic = DM_TIO_MAGIC;
686 0 : tio->io = ci->io;
687 0 : tio->ti = ti;
688 0 : tio->target_bio_nr = target_bio_nr;
689 :
690 0 : return tio;
691 : }
692 :
693 0 : static void free_tio(struct dm_target_io *tio)
694 : {
695 0 : if (tio->inside_dm_io)
696 : return;
697 0 : bio_put(&tio->clone);
698 : }
699 :
700 : /*
701 : * Add the bio to the list of deferred io.
702 : */
703 0 : static void queue_io(struct mapped_device *md, struct bio *bio)
704 : {
705 0 : unsigned long flags;
706 :
707 0 : spin_lock_irqsave(&md->deferred_lock, flags);
708 0 : bio_list_add(&md->deferred, bio);
709 0 : spin_unlock_irqrestore(&md->deferred_lock, flags);
710 0 : queue_work(md->wq, &md->work);
711 0 : }
712 :
713 : /*
714 : * Everyone (including functions in this file), should use this
715 : * function to access the md->map field, and make sure they call
716 : * dm_put_live_table() when finished.
717 : */
718 0 : struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier)
719 : {
720 0 : *srcu_idx = srcu_read_lock(&md->io_barrier);
721 :
722 0 : return srcu_dereference(md->map, &md->io_barrier);
723 : }
724 :
725 0 : void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier)
726 : {
727 0 : srcu_read_unlock(&md->io_barrier, srcu_idx);
728 0 : }
729 :
730 0 : void dm_sync_table(struct mapped_device *md)
731 : {
732 0 : synchronize_srcu(&md->io_barrier);
733 0 : synchronize_rcu_expedited();
734 0 : }
735 :
736 : /*
737 : * A fast alternative to dm_get_live_table/dm_put_live_table.
738 : * The caller must not block between these two functions.
739 : */
740 0 : static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU)
741 : {
742 0 : rcu_read_lock();
743 0 : return rcu_dereference(md->map);
744 : }
745 :
746 0 : static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU)
747 : {
748 0 : rcu_read_unlock();
749 : }
750 :
751 : static char *_dm_claim_ptr = "I belong to device-mapper";
752 :
753 : /*
754 : * Open a table device so we can use it as a map destination.
755 : */
756 0 : static int open_table_device(struct table_device *td, dev_t dev,
757 : struct mapped_device *md)
758 : {
759 0 : struct block_device *bdev;
760 :
761 0 : int r;
762 :
763 0 : BUG_ON(td->dm_dev.bdev);
764 :
765 0 : bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _dm_claim_ptr);
766 0 : if (IS_ERR(bdev))
767 0 : return PTR_ERR(bdev);
768 :
769 0 : r = bd_link_disk_holder(bdev, dm_disk(md));
770 0 : if (r) {
771 0 : blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL);
772 0 : return r;
773 : }
774 :
775 0 : td->dm_dev.bdev = bdev;
776 0 : td->dm_dev.dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
777 0 : return 0;
778 : }
779 :
780 : /*
781 : * Close a table device that we've been using.
782 : */
783 0 : static void close_table_device(struct table_device *td, struct mapped_device *md)
784 : {
785 0 : if (!td->dm_dev.bdev)
786 : return;
787 :
788 0 : bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md));
789 0 : blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL);
790 0 : put_dax(td->dm_dev.dax_dev);
791 0 : td->dm_dev.bdev = NULL;
792 0 : td->dm_dev.dax_dev = NULL;
793 : }
794 :
795 0 : static struct table_device *find_table_device(struct list_head *l, dev_t dev,
796 : fmode_t mode)
797 : {
798 0 : struct table_device *td;
799 :
800 0 : list_for_each_entry(td, l, list)
801 0 : if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode)
802 : return td;
803 :
804 : return NULL;
805 : }
806 :
807 0 : int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode,
808 : struct dm_dev **result)
809 : {
810 0 : int r;
811 0 : struct table_device *td;
812 :
813 0 : mutex_lock(&md->table_devices_lock);
814 0 : td = find_table_device(&md->table_devices, dev, mode);
815 0 : if (!td) {
816 0 : td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id);
817 0 : if (!td) {
818 0 : mutex_unlock(&md->table_devices_lock);
819 0 : return -ENOMEM;
820 : }
821 :
822 0 : td->dm_dev.mode = mode;
823 0 : td->dm_dev.bdev = NULL;
824 :
825 0 : if ((r = open_table_device(td, dev, md))) {
826 0 : mutex_unlock(&md->table_devices_lock);
827 0 : kfree(td);
828 0 : return r;
829 : }
830 :
831 0 : format_dev_t(td->dm_dev.name, dev);
832 :
833 0 : refcount_set(&td->count, 1);
834 0 : list_add(&td->list, &md->table_devices);
835 : } else {
836 0 : refcount_inc(&td->count);
837 : }
838 0 : mutex_unlock(&md->table_devices_lock);
839 :
840 0 : *result = &td->dm_dev;
841 0 : return 0;
842 : }
843 : EXPORT_SYMBOL_GPL(dm_get_table_device);
844 :
845 0 : void dm_put_table_device(struct mapped_device *md, struct dm_dev *d)
846 : {
847 0 : struct table_device *td = container_of(d, struct table_device, dm_dev);
848 :
849 0 : mutex_lock(&md->table_devices_lock);
850 0 : if (refcount_dec_and_test(&td->count)) {
851 0 : close_table_device(td, md);
852 0 : list_del(&td->list);
853 0 : kfree(td);
854 : }
855 0 : mutex_unlock(&md->table_devices_lock);
856 0 : }
857 : EXPORT_SYMBOL(dm_put_table_device);
858 :
859 0 : static void free_table_devices(struct list_head *devices)
860 : {
861 0 : struct list_head *tmp, *next;
862 :
863 0 : list_for_each_safe(tmp, next, devices) {
864 0 : struct table_device *td = list_entry(tmp, struct table_device, list);
865 :
866 0 : DMWARN("dm_destroy: %s still exists with %d references",
867 : td->dm_dev.name, refcount_read(&td->count));
868 0 : kfree(td);
869 : }
870 0 : }
871 :
872 : /*
873 : * Get the geometry associated with a dm device
874 : */
875 0 : int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
876 : {
877 0 : *geo = md->geometry;
878 :
879 0 : return 0;
880 : }
881 :
882 : /*
883 : * Set the geometry of a device.
884 : */
885 0 : int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
886 : {
887 0 : sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
888 :
889 0 : if (geo->start > sz) {
890 0 : DMWARN("Start sector is beyond the geometry limits.");
891 0 : return -EINVAL;
892 : }
893 :
894 0 : md->geometry = *geo;
895 :
896 0 : return 0;
897 : }
898 :
899 0 : static int __noflush_suspending(struct mapped_device *md)
900 : {
901 0 : return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
902 : }
903 :
904 : /*
905 : * Decrements the number of outstanding ios that a bio has been
906 : * cloned into, completing the original io if necc.
907 : */
908 0 : static void dec_pending(struct dm_io *io, blk_status_t error)
909 : {
910 0 : unsigned long flags;
911 0 : blk_status_t io_error;
912 0 : struct bio *bio;
913 0 : struct mapped_device *md = io->md;
914 :
915 : /* Push-back supersedes any I/O errors */
916 0 : if (unlikely(error)) {
917 0 : spin_lock_irqsave(&io->endio_lock, flags);
918 0 : if (!(io->status == BLK_STS_DM_REQUEUE && __noflush_suspending(md)))
919 0 : io->status = error;
920 0 : spin_unlock_irqrestore(&io->endio_lock, flags);
921 : }
922 :
923 0 : if (atomic_dec_and_test(&io->io_count)) {
924 0 : if (io->status == BLK_STS_DM_REQUEUE) {
925 : /*
926 : * Target requested pushing back the I/O.
927 : */
928 0 : spin_lock_irqsave(&md->deferred_lock, flags);
929 0 : if (__noflush_suspending(md))
930 : /* NOTE early return due to BLK_STS_DM_REQUEUE below */
931 0 : bio_list_add_head(&md->deferred, io->orig_bio);
932 : else
933 : /* noflush suspend was interrupted. */
934 0 : io->status = BLK_STS_IOERR;
935 0 : spin_unlock_irqrestore(&md->deferred_lock, flags);
936 : }
937 :
938 0 : io_error = io->status;
939 0 : bio = io->orig_bio;
940 0 : end_io_acct(io);
941 0 : free_io(md, io);
942 :
943 0 : if (io_error == BLK_STS_DM_REQUEUE)
944 : return;
945 :
946 0 : if ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size) {
947 : /*
948 : * Preflush done for flush with data, reissue
949 : * without REQ_PREFLUSH.
950 : */
951 0 : bio->bi_opf &= ~REQ_PREFLUSH;
952 0 : queue_io(md, bio);
953 : } else {
954 : /* done with normal IO or empty flush */
955 0 : if (io_error)
956 0 : bio->bi_status = io_error;
957 0 : bio_endio(bio);
958 : }
959 : }
960 : }
961 :
962 0 : void disable_discard(struct mapped_device *md)
963 : {
964 0 : struct queue_limits *limits = dm_get_queue_limits(md);
965 :
966 : /* device doesn't really support DISCARD, disable it */
967 0 : limits->max_discard_sectors = 0;
968 0 : blk_queue_flag_clear(QUEUE_FLAG_DISCARD, md->queue);
969 0 : }
970 :
971 0 : void disable_write_same(struct mapped_device *md)
972 : {
973 0 : struct queue_limits *limits = dm_get_queue_limits(md);
974 :
975 : /* device doesn't really support WRITE SAME, disable it */
976 0 : limits->max_write_same_sectors = 0;
977 0 : }
978 :
979 0 : void disable_write_zeroes(struct mapped_device *md)
980 : {
981 0 : struct queue_limits *limits = dm_get_queue_limits(md);
982 :
983 : /* device doesn't really support WRITE ZEROES, disable it */
984 0 : limits->max_write_zeroes_sectors = 0;
985 0 : }
986 :
987 0 : static bool swap_bios_limit(struct dm_target *ti, struct bio *bio)
988 : {
989 0 : return unlikely((bio->bi_opf & REQ_SWAP) != 0) && unlikely(ti->limit_swap_bios);
990 : }
991 :
992 0 : static void clone_endio(struct bio *bio)
993 : {
994 0 : blk_status_t error = bio->bi_status;
995 0 : struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
996 0 : struct dm_io *io = tio->io;
997 0 : struct mapped_device *md = tio->io->md;
998 0 : dm_endio_fn endio = tio->ti->type->end_io;
999 0 : struct bio *orig_bio = io->orig_bio;
1000 0 : struct request_queue *q = bio->bi_bdev->bd_disk->queue;
1001 :
1002 0 : if (unlikely(error == BLK_STS_TARGET)) {
1003 0 : if (bio_op(bio) == REQ_OP_DISCARD &&
1004 0 : !q->limits.max_discard_sectors)
1005 0 : disable_discard(md);
1006 0 : else if (bio_op(bio) == REQ_OP_WRITE_SAME &&
1007 0 : !q->limits.max_write_same_sectors)
1008 0 : disable_write_same(md);
1009 0 : else if (bio_op(bio) == REQ_OP_WRITE_ZEROES &&
1010 0 : !q->limits.max_write_zeroes_sectors)
1011 0 : disable_write_zeroes(md);
1012 : }
1013 :
1014 : /*
1015 : * For zone-append bios get offset in zone of the written
1016 : * sector and add that to the original bio sector pos.
1017 : */
1018 0 : if (bio_op(orig_bio) == REQ_OP_ZONE_APPEND) {
1019 0 : sector_t written_sector = bio->bi_iter.bi_sector;
1020 0 : struct request_queue *q = orig_bio->bi_bdev->bd_disk->queue;
1021 0 : u64 mask = (u64)blk_queue_zone_sectors(q) - 1;
1022 :
1023 0 : orig_bio->bi_iter.bi_sector += written_sector & mask;
1024 : }
1025 :
1026 0 : if (endio) {
1027 0 : int r = endio(tio->ti, bio, &error);
1028 0 : switch (r) {
1029 0 : case DM_ENDIO_REQUEUE:
1030 0 : error = BLK_STS_DM_REQUEUE;
1031 : fallthrough;
1032 : case DM_ENDIO_DONE:
1033 : break;
1034 0 : case DM_ENDIO_INCOMPLETE:
1035 : /* The target will handle the io */
1036 0 : return;
1037 0 : default:
1038 0 : DMWARN("unimplemented target endio return value: %d", r);
1039 0 : BUG();
1040 : }
1041 0 : }
1042 :
1043 0 : if (unlikely(swap_bios_limit(tio->ti, bio))) {
1044 0 : struct mapped_device *md = io->md;
1045 0 : up(&md->swap_bios_semaphore);
1046 : }
1047 :
1048 0 : free_tio(tio);
1049 0 : dec_pending(io, error);
1050 : }
1051 :
1052 : /*
1053 : * Return maximum size of I/O possible at the supplied sector up to the current
1054 : * target boundary.
1055 : */
1056 0 : static inline sector_t max_io_len_target_boundary(struct dm_target *ti,
1057 : sector_t target_offset)
1058 : {
1059 0 : return ti->len - target_offset;
1060 : }
1061 :
1062 0 : static sector_t max_io_len(struct dm_target *ti, sector_t sector)
1063 : {
1064 0 : sector_t target_offset = dm_target_offset(ti, sector);
1065 0 : sector_t len = max_io_len_target_boundary(ti, target_offset);
1066 0 : sector_t max_len;
1067 :
1068 : /*
1069 : * Does the target need to split IO even further?
1070 : * - varied (per target) IO splitting is a tenet of DM; this
1071 : * explains why stacked chunk_sectors based splitting via
1072 : * blk_max_size_offset() isn't possible here. So pass in
1073 : * ti->max_io_len to override stacked chunk_sectors.
1074 : */
1075 0 : if (ti->max_io_len) {
1076 0 : max_len = blk_max_size_offset(ti->table->md->queue,
1077 : target_offset, ti->max_io_len);
1078 0 : if (len > max_len)
1079 : len = max_len;
1080 : }
1081 :
1082 0 : return len;
1083 : }
1084 :
1085 0 : int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
1086 : {
1087 0 : if (len > UINT_MAX) {
1088 0 : DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)",
1089 : (unsigned long long)len, UINT_MAX);
1090 0 : ti->error = "Maximum size of target IO is too large";
1091 0 : return -EINVAL;
1092 : }
1093 :
1094 0 : ti->max_io_len = (uint32_t) len;
1095 :
1096 0 : return 0;
1097 : }
1098 : EXPORT_SYMBOL_GPL(dm_set_target_max_io_len);
1099 :
1100 : static struct dm_target *dm_dax_get_live_target(struct mapped_device *md,
1101 : sector_t sector, int *srcu_idx)
1102 : __acquires(md->io_barrier)
1103 : {
1104 : struct dm_table *map;
1105 : struct dm_target *ti;
1106 :
1107 : map = dm_get_live_table(md, srcu_idx);
1108 : if (!map)
1109 : return NULL;
1110 :
1111 : ti = dm_table_find_target(map, sector);
1112 : if (!ti)
1113 : return NULL;
1114 :
1115 : return ti;
1116 : }
1117 :
1118 : static long dm_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
1119 : long nr_pages, void **kaddr, pfn_t *pfn)
1120 : {
1121 : struct mapped_device *md = dax_get_private(dax_dev);
1122 : sector_t sector = pgoff * PAGE_SECTORS;
1123 : struct dm_target *ti;
1124 : long len, ret = -EIO;
1125 : int srcu_idx;
1126 :
1127 : ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1128 :
1129 : if (!ti)
1130 : goto out;
1131 : if (!ti->type->direct_access)
1132 : goto out;
1133 : len = max_io_len(ti, sector) / PAGE_SECTORS;
1134 : if (len < 1)
1135 : goto out;
1136 : nr_pages = min(len, nr_pages);
1137 : ret = ti->type->direct_access(ti, pgoff, nr_pages, kaddr, pfn);
1138 :
1139 : out:
1140 : dm_put_live_table(md, srcu_idx);
1141 :
1142 : return ret;
1143 : }
1144 :
1145 : static bool dm_dax_supported(struct dax_device *dax_dev, struct block_device *bdev,
1146 : int blocksize, sector_t start, sector_t len)
1147 : {
1148 : struct mapped_device *md = dax_get_private(dax_dev);
1149 : struct dm_table *map;
1150 : bool ret = false;
1151 : int srcu_idx;
1152 :
1153 : map = dm_get_live_table(md, &srcu_idx);
1154 : if (!map)
1155 : goto out;
1156 :
1157 : ret = dm_table_supports_dax(map, device_not_dax_capable, &blocksize);
1158 :
1159 : out:
1160 : dm_put_live_table(md, srcu_idx);
1161 :
1162 : return ret;
1163 : }
1164 :
1165 : static size_t dm_dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff,
1166 : void *addr, size_t bytes, struct iov_iter *i)
1167 : {
1168 : struct mapped_device *md = dax_get_private(dax_dev);
1169 : sector_t sector = pgoff * PAGE_SECTORS;
1170 : struct dm_target *ti;
1171 : long ret = 0;
1172 : int srcu_idx;
1173 :
1174 : ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1175 :
1176 : if (!ti)
1177 : goto out;
1178 : if (!ti->type->dax_copy_from_iter) {
1179 : ret = copy_from_iter(addr, bytes, i);
1180 : goto out;
1181 : }
1182 : ret = ti->type->dax_copy_from_iter(ti, pgoff, addr, bytes, i);
1183 : out:
1184 : dm_put_live_table(md, srcu_idx);
1185 :
1186 : return ret;
1187 : }
1188 :
1189 : static size_t dm_dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff,
1190 : void *addr, size_t bytes, struct iov_iter *i)
1191 : {
1192 : struct mapped_device *md = dax_get_private(dax_dev);
1193 : sector_t sector = pgoff * PAGE_SECTORS;
1194 : struct dm_target *ti;
1195 : long ret = 0;
1196 : int srcu_idx;
1197 :
1198 : ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1199 :
1200 : if (!ti)
1201 : goto out;
1202 : if (!ti->type->dax_copy_to_iter) {
1203 : ret = copy_to_iter(addr, bytes, i);
1204 : goto out;
1205 : }
1206 : ret = ti->type->dax_copy_to_iter(ti, pgoff, addr, bytes, i);
1207 : out:
1208 : dm_put_live_table(md, srcu_idx);
1209 :
1210 : return ret;
1211 : }
1212 :
1213 : static int dm_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
1214 : size_t nr_pages)
1215 : {
1216 : struct mapped_device *md = dax_get_private(dax_dev);
1217 : sector_t sector = pgoff * PAGE_SECTORS;
1218 : struct dm_target *ti;
1219 : int ret = -EIO;
1220 : int srcu_idx;
1221 :
1222 : ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1223 :
1224 : if (!ti)
1225 : goto out;
1226 : if (WARN_ON(!ti->type->dax_zero_page_range)) {
1227 : /*
1228 : * ->zero_page_range() is mandatory dax operation. If we are
1229 : * here, something is wrong.
1230 : */
1231 : goto out;
1232 : }
1233 : ret = ti->type->dax_zero_page_range(ti, pgoff, nr_pages);
1234 : out:
1235 : dm_put_live_table(md, srcu_idx);
1236 :
1237 : return ret;
1238 : }
1239 :
1240 : /*
1241 : * A target may call dm_accept_partial_bio only from the map routine. It is
1242 : * allowed for all bio types except REQ_PREFLUSH, REQ_OP_ZONE_RESET,
1243 : * REQ_OP_ZONE_OPEN, REQ_OP_ZONE_CLOSE and REQ_OP_ZONE_FINISH.
1244 : *
1245 : * dm_accept_partial_bio informs the dm that the target only wants to process
1246 : * additional n_sectors sectors of the bio and the rest of the data should be
1247 : * sent in a next bio.
1248 : *
1249 : * A diagram that explains the arithmetics:
1250 : * +--------------------+---------------+-------+
1251 : * | 1 | 2 | 3 |
1252 : * +--------------------+---------------+-------+
1253 : *
1254 : * <-------------- *tio->len_ptr --------------->
1255 : * <------- bi_size ------->
1256 : * <-- n_sectors -->
1257 : *
1258 : * Region 1 was already iterated over with bio_advance or similar function.
1259 : * (it may be empty if the target doesn't use bio_advance)
1260 : * Region 2 is the remaining bio size that the target wants to process.
1261 : * (it may be empty if region 1 is non-empty, although there is no reason
1262 : * to make it empty)
1263 : * The target requires that region 3 is to be sent in the next bio.
1264 : *
1265 : * If the target wants to receive multiple copies of the bio (via num_*bios, etc),
1266 : * the partially processed part (the sum of regions 1+2) must be the same for all
1267 : * copies of the bio.
1268 : */
1269 0 : void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors)
1270 : {
1271 0 : struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
1272 0 : unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT;
1273 0 : BUG_ON(bio->bi_opf & REQ_PREFLUSH);
1274 0 : BUG_ON(bi_size > *tio->len_ptr);
1275 0 : BUG_ON(n_sectors > bi_size);
1276 0 : *tio->len_ptr -= bi_size - n_sectors;
1277 0 : bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT;
1278 0 : }
1279 : EXPORT_SYMBOL_GPL(dm_accept_partial_bio);
1280 :
1281 0 : static noinline void __set_swap_bios_limit(struct mapped_device *md, int latch)
1282 : {
1283 0 : mutex_lock(&md->swap_bios_lock);
1284 0 : while (latch < md->swap_bios) {
1285 0 : cond_resched();
1286 0 : down(&md->swap_bios_semaphore);
1287 0 : md->swap_bios--;
1288 : }
1289 0 : while (latch > md->swap_bios) {
1290 0 : cond_resched();
1291 0 : up(&md->swap_bios_semaphore);
1292 0 : md->swap_bios++;
1293 : }
1294 0 : mutex_unlock(&md->swap_bios_lock);
1295 0 : }
1296 :
1297 0 : static blk_qc_t __map_bio(struct dm_target_io *tio)
1298 : {
1299 0 : int r;
1300 0 : sector_t sector;
1301 0 : struct bio *clone = &tio->clone;
1302 0 : struct dm_io *io = tio->io;
1303 0 : struct dm_target *ti = tio->ti;
1304 0 : blk_qc_t ret = BLK_QC_T_NONE;
1305 :
1306 0 : clone->bi_end_io = clone_endio;
1307 :
1308 : /*
1309 : * Map the clone. If r == 0 we don't need to do
1310 : * anything, the target has assumed ownership of
1311 : * this io.
1312 : */
1313 0 : atomic_inc(&io->io_count);
1314 0 : sector = clone->bi_iter.bi_sector;
1315 :
1316 0 : if (unlikely(swap_bios_limit(ti, clone))) {
1317 0 : struct mapped_device *md = io->md;
1318 0 : int latch = get_swap_bios();
1319 0 : if (unlikely(latch != md->swap_bios))
1320 0 : __set_swap_bios_limit(md, latch);
1321 0 : down(&md->swap_bios_semaphore);
1322 : }
1323 :
1324 0 : r = ti->type->map(ti, clone);
1325 0 : switch (r) {
1326 : case DM_MAPIO_SUBMITTED:
1327 : break;
1328 0 : case DM_MAPIO_REMAPPED:
1329 : /* the bio has been remapped so dispatch it */
1330 0 : trace_block_bio_remap(clone, bio_dev(io->orig_bio), sector);
1331 0 : ret = submit_bio_noacct(clone);
1332 0 : break;
1333 : case DM_MAPIO_KILL:
1334 0 : if (unlikely(swap_bios_limit(ti, clone))) {
1335 0 : struct mapped_device *md = io->md;
1336 0 : up(&md->swap_bios_semaphore);
1337 : }
1338 0 : free_tio(tio);
1339 0 : dec_pending(io, BLK_STS_IOERR);
1340 0 : break;
1341 : case DM_MAPIO_REQUEUE:
1342 0 : if (unlikely(swap_bios_limit(ti, clone))) {
1343 0 : struct mapped_device *md = io->md;
1344 0 : up(&md->swap_bios_semaphore);
1345 : }
1346 0 : free_tio(tio);
1347 0 : dec_pending(io, BLK_STS_DM_REQUEUE);
1348 0 : break;
1349 0 : default:
1350 0 : DMWARN("unimplemented target map return value: %d", r);
1351 0 : BUG();
1352 : }
1353 :
1354 0 : return ret;
1355 : }
1356 :
1357 0 : static void bio_setup_sector(struct bio *bio, sector_t sector, unsigned len)
1358 : {
1359 0 : bio->bi_iter.bi_sector = sector;
1360 0 : bio->bi_iter.bi_size = to_bytes(len);
1361 0 : }
1362 :
1363 : /*
1364 : * Creates a bio that consists of range of complete bvecs.
1365 : */
1366 0 : static int clone_bio(struct dm_target_io *tio, struct bio *bio,
1367 : sector_t sector, unsigned len)
1368 : {
1369 0 : struct bio *clone = &tio->clone;
1370 0 : int r;
1371 :
1372 0 : __bio_clone_fast(clone, bio);
1373 :
1374 0 : r = bio_crypt_clone(clone, bio, GFP_NOIO);
1375 0 : if (r < 0)
1376 : return r;
1377 :
1378 0 : if (bio_integrity(bio)) {
1379 : if (unlikely(!dm_target_has_integrity(tio->ti->type) &&
1380 : !dm_target_passes_integrity(tio->ti->type))) {
1381 : DMWARN("%s: the target %s doesn't support integrity data.",
1382 : dm_device_name(tio->io->md),
1383 : tio->ti->type->name);
1384 : return -EIO;
1385 : }
1386 :
1387 : r = bio_integrity_clone(clone, bio, GFP_NOIO);
1388 : if (r < 0)
1389 : return r;
1390 : }
1391 :
1392 0 : bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector));
1393 0 : clone->bi_iter.bi_size = to_bytes(len);
1394 :
1395 0 : if (bio_integrity(bio))
1396 0 : bio_integrity_trim(clone);
1397 :
1398 0 : return 0;
1399 : }
1400 :
1401 0 : static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci,
1402 : struct dm_target *ti, unsigned num_bios)
1403 : {
1404 0 : struct dm_target_io *tio;
1405 0 : int try;
1406 :
1407 0 : if (!num_bios)
1408 : return;
1409 :
1410 0 : if (num_bios == 1) {
1411 0 : tio = alloc_tio(ci, ti, 0, GFP_NOIO);
1412 0 : bio_list_add(blist, &tio->clone);
1413 0 : return;
1414 : }
1415 :
1416 0 : for (try = 0; try < 2; try++) {
1417 0 : int bio_nr;
1418 0 : struct bio *bio;
1419 :
1420 0 : if (try)
1421 0 : mutex_lock(&ci->io->md->table_devices_lock);
1422 0 : for (bio_nr = 0; bio_nr < num_bios; bio_nr++) {
1423 0 : tio = alloc_tio(ci, ti, bio_nr, try ? GFP_NOIO : GFP_NOWAIT);
1424 0 : if (!tio)
1425 : break;
1426 :
1427 0 : bio_list_add(blist, &tio->clone);
1428 : }
1429 0 : if (try)
1430 0 : mutex_unlock(&ci->io->md->table_devices_lock);
1431 0 : if (bio_nr == num_bios)
1432 : return;
1433 :
1434 0 : while ((bio = bio_list_pop(blist))) {
1435 0 : tio = container_of(bio, struct dm_target_io, clone);
1436 0 : free_tio(tio);
1437 : }
1438 : }
1439 : }
1440 :
1441 0 : static blk_qc_t __clone_and_map_simple_bio(struct clone_info *ci,
1442 : struct dm_target_io *tio, unsigned *len)
1443 : {
1444 0 : struct bio *clone = &tio->clone;
1445 :
1446 0 : tio->len_ptr = len;
1447 :
1448 0 : __bio_clone_fast(clone, ci->bio);
1449 0 : if (len)
1450 0 : bio_setup_sector(clone, ci->sector, *len);
1451 :
1452 0 : return __map_bio(tio);
1453 : }
1454 :
1455 0 : static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti,
1456 : unsigned num_bios, unsigned *len)
1457 : {
1458 0 : struct bio_list blist = BIO_EMPTY_LIST;
1459 0 : struct bio *bio;
1460 0 : struct dm_target_io *tio;
1461 :
1462 0 : alloc_multiple_bios(&blist, ci, ti, num_bios);
1463 :
1464 0 : while ((bio = bio_list_pop(&blist))) {
1465 0 : tio = container_of(bio, struct dm_target_io, clone);
1466 0 : (void) __clone_and_map_simple_bio(ci, tio, len);
1467 : }
1468 0 : }
1469 :
1470 0 : static int __send_empty_flush(struct clone_info *ci)
1471 : {
1472 0 : unsigned target_nr = 0;
1473 0 : struct dm_target *ti;
1474 0 : struct bio flush_bio;
1475 :
1476 : /*
1477 : * Use an on-stack bio for this, it's safe since we don't
1478 : * need to reference it after submit. It's just used as
1479 : * the basis for the clone(s).
1480 : */
1481 0 : bio_init(&flush_bio, NULL, 0);
1482 0 : flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC;
1483 0 : bio_set_dev(&flush_bio, ci->io->md->disk->part0);
1484 :
1485 0 : ci->bio = &flush_bio;
1486 0 : ci->sector_count = 0;
1487 :
1488 0 : BUG_ON(bio_has_data(ci->bio));
1489 0 : while ((ti = dm_table_get_target(ci->map, target_nr++)))
1490 0 : __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL);
1491 :
1492 0 : bio_uninit(ci->bio);
1493 0 : return 0;
1494 : }
1495 :
1496 0 : static int __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti,
1497 : sector_t sector, unsigned *len)
1498 : {
1499 0 : struct bio *bio = ci->bio;
1500 0 : struct dm_target_io *tio;
1501 0 : int r;
1502 :
1503 0 : tio = alloc_tio(ci, ti, 0, GFP_NOIO);
1504 0 : tio->len_ptr = len;
1505 0 : r = clone_bio(tio, bio, sector, *len);
1506 0 : if (r < 0) {
1507 0 : free_tio(tio);
1508 0 : return r;
1509 : }
1510 0 : (void) __map_bio(tio);
1511 :
1512 0 : return 0;
1513 : }
1514 :
1515 0 : static int __send_changing_extent_only(struct clone_info *ci, struct dm_target *ti,
1516 : unsigned num_bios)
1517 : {
1518 0 : unsigned len;
1519 :
1520 : /*
1521 : * Even though the device advertised support for this type of
1522 : * request, that does not mean every target supports it, and
1523 : * reconfiguration might also have changed that since the
1524 : * check was performed.
1525 : */
1526 0 : if (!num_bios)
1527 : return -EOPNOTSUPP;
1528 :
1529 0 : len = min_t(sector_t, ci->sector_count,
1530 : max_io_len_target_boundary(ti, dm_target_offset(ti, ci->sector)));
1531 :
1532 0 : __send_duplicate_bios(ci, ti, num_bios, &len);
1533 :
1534 0 : ci->sector += len;
1535 0 : ci->sector_count -= len;
1536 :
1537 0 : return 0;
1538 : }
1539 :
1540 0 : static bool is_abnormal_io(struct bio *bio)
1541 : {
1542 0 : bool r = false;
1543 :
1544 0 : switch (bio_op(bio)) {
1545 0 : case REQ_OP_DISCARD:
1546 : case REQ_OP_SECURE_ERASE:
1547 : case REQ_OP_WRITE_SAME:
1548 : case REQ_OP_WRITE_ZEROES:
1549 0 : r = true;
1550 0 : break;
1551 : }
1552 :
1553 0 : return r;
1554 : }
1555 :
1556 0 : static bool __process_abnormal_io(struct clone_info *ci, struct dm_target *ti,
1557 : int *result)
1558 : {
1559 0 : struct bio *bio = ci->bio;
1560 0 : unsigned num_bios = 0;
1561 :
1562 0 : switch (bio_op(bio)) {
1563 0 : case REQ_OP_DISCARD:
1564 0 : num_bios = ti->num_discard_bios;
1565 0 : break;
1566 0 : case REQ_OP_SECURE_ERASE:
1567 0 : num_bios = ti->num_secure_erase_bios;
1568 0 : break;
1569 0 : case REQ_OP_WRITE_SAME:
1570 0 : num_bios = ti->num_write_same_bios;
1571 0 : break;
1572 0 : case REQ_OP_WRITE_ZEROES:
1573 0 : num_bios = ti->num_write_zeroes_bios;
1574 0 : break;
1575 : default:
1576 : return false;
1577 : }
1578 :
1579 0 : *result = __send_changing_extent_only(ci, ti, num_bios);
1580 0 : return true;
1581 : }
1582 :
1583 : /*
1584 : * Select the correct strategy for processing a non-flush bio.
1585 : */
1586 0 : static int __split_and_process_non_flush(struct clone_info *ci)
1587 : {
1588 0 : struct dm_target *ti;
1589 0 : unsigned len;
1590 0 : int r;
1591 :
1592 0 : ti = dm_table_find_target(ci->map, ci->sector);
1593 0 : if (!ti)
1594 : return -EIO;
1595 :
1596 0 : if (__process_abnormal_io(ci, ti, &r))
1597 0 : return r;
1598 :
1599 0 : len = min_t(sector_t, max_io_len(ti, ci->sector), ci->sector_count);
1600 :
1601 0 : r = __clone_and_map_data_bio(ci, ti, ci->sector, &len);
1602 0 : if (r < 0)
1603 : return r;
1604 :
1605 0 : ci->sector += len;
1606 0 : ci->sector_count -= len;
1607 :
1608 0 : return 0;
1609 : }
1610 :
1611 0 : static void init_clone_info(struct clone_info *ci, struct mapped_device *md,
1612 : struct dm_table *map, struct bio *bio)
1613 : {
1614 0 : ci->map = map;
1615 0 : ci->io = alloc_io(md, bio);
1616 0 : ci->sector = bio->bi_iter.bi_sector;
1617 : }
1618 :
1619 : #define __dm_part_stat_sub(part, field, subnd) \
1620 : (part_stat_get(part, field) -= (subnd))
1621 :
1622 : /*
1623 : * Entry point to split a bio into clones and submit them to the targets.
1624 : */
1625 0 : static blk_qc_t __split_and_process_bio(struct mapped_device *md,
1626 : struct dm_table *map, struct bio *bio)
1627 : {
1628 0 : struct clone_info ci;
1629 0 : blk_qc_t ret = BLK_QC_T_NONE;
1630 0 : int error = 0;
1631 :
1632 0 : init_clone_info(&ci, md, map, bio);
1633 :
1634 0 : if (bio->bi_opf & REQ_PREFLUSH) {
1635 0 : error = __send_empty_flush(&ci);
1636 : /* dec_pending submits any data associated with flush */
1637 0 : } else if (op_is_zone_mgmt(bio_op(bio))) {
1638 0 : ci.bio = bio;
1639 0 : ci.sector_count = 0;
1640 0 : error = __split_and_process_non_flush(&ci);
1641 : } else {
1642 0 : ci.bio = bio;
1643 0 : ci.sector_count = bio_sectors(bio);
1644 0 : while (ci.sector_count && !error) {
1645 0 : error = __split_and_process_non_flush(&ci);
1646 0 : if (ci.sector_count && !error) {
1647 : /*
1648 : * Remainder must be passed to submit_bio_noacct()
1649 : * so that it gets handled *after* bios already submitted
1650 : * have been completely processed.
1651 : * We take a clone of the original to store in
1652 : * ci.io->orig_bio to be used by end_io_acct() and
1653 : * for dec_pending to use for completion handling.
1654 : */
1655 0 : struct bio *b = bio_split(bio, bio_sectors(bio) - ci.sector_count,
1656 0 : GFP_NOIO, &md->queue->bio_split);
1657 0 : ci.io->orig_bio = b;
1658 :
1659 : /*
1660 : * Adjust IO stats for each split, otherwise upon queue
1661 : * reentry there will be redundant IO accounting.
1662 : * NOTE: this is a stop-gap fix, a proper fix involves
1663 : * significant refactoring of DM core's bio splitting
1664 : * (by eliminating DM's splitting and just using bio_split)
1665 : */
1666 0 : part_stat_lock();
1667 0 : __dm_part_stat_sub(dm_disk(md)->part0,
1668 : sectors[op_stat_group(bio_op(bio))], ci.sector_count);
1669 0 : part_stat_unlock();
1670 :
1671 0 : bio_chain(b, bio);
1672 0 : trace_block_split(b, bio->bi_iter.bi_sector);
1673 0 : ret = submit_bio_noacct(bio);
1674 0 : break;
1675 : }
1676 : }
1677 : }
1678 :
1679 : /* drop the extra reference count */
1680 0 : dec_pending(ci.io, errno_to_blk_status(error));
1681 0 : return ret;
1682 : }
1683 :
1684 0 : static blk_qc_t dm_submit_bio(struct bio *bio)
1685 : {
1686 0 : struct mapped_device *md = bio->bi_bdev->bd_disk->private_data;
1687 0 : blk_qc_t ret = BLK_QC_T_NONE;
1688 0 : int srcu_idx;
1689 0 : struct dm_table *map;
1690 :
1691 0 : map = dm_get_live_table(md, &srcu_idx);
1692 0 : if (unlikely(!map)) {
1693 0 : DMERR_LIMIT("%s: mapping table unavailable, erroring io",
1694 : dm_device_name(md));
1695 0 : bio_io_error(bio);
1696 0 : goto out;
1697 : }
1698 :
1699 : /* If suspended, queue this IO for later */
1700 0 : if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {
1701 0 : if (bio->bi_opf & REQ_NOWAIT)
1702 0 : bio_wouldblock_error(bio);
1703 0 : else if (bio->bi_opf & REQ_RAHEAD)
1704 0 : bio_io_error(bio);
1705 : else
1706 0 : queue_io(md, bio);
1707 0 : goto out;
1708 : }
1709 :
1710 : /*
1711 : * Use blk_queue_split() for abnormal IO (e.g. discard, writesame, etc)
1712 : * otherwise associated queue_limits won't be imposed.
1713 : */
1714 0 : if (is_abnormal_io(bio))
1715 0 : blk_queue_split(&bio);
1716 :
1717 0 : ret = __split_and_process_bio(md, map, bio);
1718 0 : out:
1719 0 : dm_put_live_table(md, srcu_idx);
1720 0 : return ret;
1721 : }
1722 :
1723 : /*-----------------------------------------------------------------
1724 : * An IDR is used to keep track of allocated minor numbers.
1725 : *---------------------------------------------------------------*/
1726 0 : static void free_minor(int minor)
1727 : {
1728 0 : spin_lock(&_minor_lock);
1729 0 : idr_remove(&_minor_idr, minor);
1730 0 : spin_unlock(&_minor_lock);
1731 0 : }
1732 :
1733 : /*
1734 : * See if the device with a specific minor # is free.
1735 : */
1736 0 : static int specific_minor(int minor)
1737 : {
1738 0 : int r;
1739 :
1740 0 : if (minor >= (1 << MINORBITS))
1741 : return -EINVAL;
1742 :
1743 0 : idr_preload(GFP_KERNEL);
1744 0 : spin_lock(&_minor_lock);
1745 :
1746 0 : r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT);
1747 :
1748 0 : spin_unlock(&_minor_lock);
1749 0 : idr_preload_end();
1750 0 : if (r < 0)
1751 0 : return r == -ENOSPC ? -EBUSY : r;
1752 : return 0;
1753 : }
1754 :
1755 0 : static int next_free_minor(int *minor)
1756 : {
1757 0 : int r;
1758 :
1759 0 : idr_preload(GFP_KERNEL);
1760 0 : spin_lock(&_minor_lock);
1761 :
1762 0 : r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT);
1763 :
1764 0 : spin_unlock(&_minor_lock);
1765 0 : idr_preload_end();
1766 0 : if (r < 0)
1767 : return r;
1768 0 : *minor = r;
1769 0 : return 0;
1770 : }
1771 :
1772 : static const struct block_device_operations dm_blk_dops;
1773 : static const struct block_device_operations dm_rq_blk_dops;
1774 : static const struct dax_operations dm_dax_ops;
1775 :
1776 : static void dm_wq_work(struct work_struct *work);
1777 :
1778 : #ifdef CONFIG_BLK_INLINE_ENCRYPTION
1779 : static void dm_queue_destroy_keyslot_manager(struct request_queue *q)
1780 : {
1781 : dm_destroy_keyslot_manager(q->ksm);
1782 : }
1783 :
1784 : #else /* CONFIG_BLK_INLINE_ENCRYPTION */
1785 :
1786 0 : static inline void dm_queue_destroy_keyslot_manager(struct request_queue *q)
1787 : {
1788 0 : }
1789 : #endif /* !CONFIG_BLK_INLINE_ENCRYPTION */
1790 :
1791 0 : static void cleanup_mapped_device(struct mapped_device *md)
1792 : {
1793 0 : if (md->wq)
1794 0 : destroy_workqueue(md->wq);
1795 0 : bioset_exit(&md->bs);
1796 0 : bioset_exit(&md->io_bs);
1797 :
1798 0 : if (md->dax_dev) {
1799 0 : kill_dax(md->dax_dev);
1800 0 : put_dax(md->dax_dev);
1801 0 : md->dax_dev = NULL;
1802 : }
1803 :
1804 0 : if (md->disk) {
1805 0 : spin_lock(&_minor_lock);
1806 0 : md->disk->private_data = NULL;
1807 0 : spin_unlock(&_minor_lock);
1808 0 : del_gendisk(md->disk);
1809 0 : put_disk(md->disk);
1810 : }
1811 :
1812 0 : if (md->queue) {
1813 0 : dm_queue_destroy_keyslot_manager(md->queue);
1814 0 : blk_cleanup_queue(md->queue);
1815 : }
1816 :
1817 0 : cleanup_srcu_struct(&md->io_barrier);
1818 :
1819 0 : mutex_destroy(&md->suspend_lock);
1820 0 : mutex_destroy(&md->type_lock);
1821 0 : mutex_destroy(&md->table_devices_lock);
1822 0 : mutex_destroy(&md->swap_bios_lock);
1823 :
1824 0 : dm_mq_cleanup_mapped_device(md);
1825 0 : }
1826 :
1827 : /*
1828 : * Allocate and initialise a blank device with a given minor.
1829 : */
1830 0 : static struct mapped_device *alloc_dev(int minor)
1831 : {
1832 0 : int r, numa_node_id = dm_get_numa_node();
1833 0 : struct mapped_device *md;
1834 0 : void *old_md;
1835 :
1836 0 : md = kvzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id);
1837 0 : if (!md) {
1838 0 : DMWARN("unable to allocate device, out of memory.");
1839 0 : return NULL;
1840 : }
1841 :
1842 0 : if (!try_module_get(THIS_MODULE))
1843 : goto bad_module_get;
1844 :
1845 : /* get a minor number for the dev */
1846 0 : if (minor == DM_ANY_MINOR)
1847 0 : r = next_free_minor(&minor);
1848 : else
1849 0 : r = specific_minor(minor);
1850 0 : if (r < 0)
1851 0 : goto bad_minor;
1852 :
1853 0 : r = init_srcu_struct(&md->io_barrier);
1854 0 : if (r < 0)
1855 0 : goto bad_io_barrier;
1856 :
1857 0 : md->numa_node_id = numa_node_id;
1858 0 : md->init_tio_pdu = false;
1859 0 : md->type = DM_TYPE_NONE;
1860 0 : mutex_init(&md->suspend_lock);
1861 0 : mutex_init(&md->type_lock);
1862 0 : mutex_init(&md->table_devices_lock);
1863 0 : spin_lock_init(&md->deferred_lock);
1864 0 : atomic_set(&md->holders, 1);
1865 0 : atomic_set(&md->open_count, 0);
1866 0 : atomic_set(&md->event_nr, 0);
1867 0 : atomic_set(&md->uevent_seq, 0);
1868 0 : INIT_LIST_HEAD(&md->uevent_list);
1869 0 : INIT_LIST_HEAD(&md->table_devices);
1870 0 : spin_lock_init(&md->uevent_lock);
1871 :
1872 : /*
1873 : * default to bio-based until DM table is loaded and md->type
1874 : * established. If request-based table is loaded: blk-mq will
1875 : * override accordingly.
1876 : */
1877 0 : md->queue = blk_alloc_queue(numa_node_id);
1878 0 : if (!md->queue)
1879 0 : goto bad;
1880 :
1881 0 : md->disk = alloc_disk_node(1, md->numa_node_id);
1882 0 : if (!md->disk)
1883 0 : goto bad;
1884 :
1885 0 : init_waitqueue_head(&md->wait);
1886 0 : INIT_WORK(&md->work, dm_wq_work);
1887 0 : init_waitqueue_head(&md->eventq);
1888 0 : init_completion(&md->kobj_holder.completion);
1889 :
1890 0 : md->swap_bios = get_swap_bios();
1891 0 : sema_init(&md->swap_bios_semaphore, md->swap_bios);
1892 0 : mutex_init(&md->swap_bios_lock);
1893 :
1894 0 : md->disk->major = _major;
1895 0 : md->disk->first_minor = minor;
1896 0 : md->disk->fops = &dm_blk_dops;
1897 0 : md->disk->queue = md->queue;
1898 0 : md->disk->private_data = md;
1899 0 : sprintf(md->disk->disk_name, "dm-%d", minor);
1900 :
1901 0 : if (IS_ENABLED(CONFIG_DAX_DRIVER)) {
1902 : md->dax_dev = alloc_dax(md, md->disk->disk_name,
1903 : &dm_dax_ops, 0);
1904 : if (IS_ERR(md->dax_dev))
1905 : goto bad;
1906 : }
1907 :
1908 0 : add_disk_no_queue_reg(md->disk);
1909 0 : format_dev_t(md->name, MKDEV(_major, minor));
1910 :
1911 0 : md->wq = alloc_workqueue("kdmflush", WQ_MEM_RECLAIM, 0);
1912 0 : if (!md->wq)
1913 0 : goto bad;
1914 :
1915 0 : dm_stats_init(&md->stats);
1916 :
1917 : /* Populate the mapping, nobody knows we exist yet */
1918 0 : spin_lock(&_minor_lock);
1919 0 : old_md = idr_replace(&_minor_idr, md, minor);
1920 0 : spin_unlock(&_minor_lock);
1921 :
1922 0 : BUG_ON(old_md != MINOR_ALLOCED);
1923 :
1924 : return md;
1925 :
1926 0 : bad:
1927 0 : cleanup_mapped_device(md);
1928 0 : bad_io_barrier:
1929 0 : free_minor(minor);
1930 0 : bad_minor:
1931 0 : module_put(THIS_MODULE);
1932 0 : bad_module_get:
1933 0 : kvfree(md);
1934 0 : return NULL;
1935 : }
1936 :
1937 : static void unlock_fs(struct mapped_device *md);
1938 :
1939 0 : static void free_dev(struct mapped_device *md)
1940 : {
1941 0 : int minor = MINOR(disk_devt(md->disk));
1942 :
1943 0 : unlock_fs(md);
1944 :
1945 0 : cleanup_mapped_device(md);
1946 :
1947 0 : free_table_devices(&md->table_devices);
1948 0 : dm_stats_cleanup(&md->stats);
1949 0 : free_minor(minor);
1950 :
1951 0 : module_put(THIS_MODULE);
1952 0 : kvfree(md);
1953 0 : }
1954 :
1955 0 : static int __bind_mempools(struct mapped_device *md, struct dm_table *t)
1956 : {
1957 0 : struct dm_md_mempools *p = dm_table_get_md_mempools(t);
1958 0 : int ret = 0;
1959 :
1960 0 : if (dm_table_bio_based(t)) {
1961 : /*
1962 : * The md may already have mempools that need changing.
1963 : * If so, reload bioset because front_pad may have changed
1964 : * because a different table was loaded.
1965 : */
1966 0 : bioset_exit(&md->bs);
1967 0 : bioset_exit(&md->io_bs);
1968 :
1969 0 : } else if (bioset_initialized(&md->bs)) {
1970 : /*
1971 : * There's no need to reload with request-based dm
1972 : * because the size of front_pad doesn't change.
1973 : * Note for future: If you are to reload bioset,
1974 : * prep-ed requests in the queue may refer
1975 : * to bio from the old bioset, so you must walk
1976 : * through the queue to unprep.
1977 : */
1978 0 : goto out;
1979 : }
1980 :
1981 0 : BUG_ON(!p ||
1982 : bioset_initialized(&md->bs) ||
1983 : bioset_initialized(&md->io_bs));
1984 :
1985 0 : ret = bioset_init_from_src(&md->bs, &p->bs);
1986 0 : if (ret)
1987 0 : goto out;
1988 0 : ret = bioset_init_from_src(&md->io_bs, &p->io_bs);
1989 0 : if (ret)
1990 0 : bioset_exit(&md->bs);
1991 0 : out:
1992 : /* mempool bind completed, no longer need any mempools in the table */
1993 0 : dm_table_free_md_mempools(t);
1994 0 : return ret;
1995 : }
1996 :
1997 : /*
1998 : * Bind a table to the device.
1999 : */
2000 0 : static void event_callback(void *context)
2001 : {
2002 0 : unsigned long flags;
2003 0 : LIST_HEAD(uevents);
2004 0 : struct mapped_device *md = (struct mapped_device *) context;
2005 :
2006 0 : spin_lock_irqsave(&md->uevent_lock, flags);
2007 0 : list_splice_init(&md->uevent_list, &uevents);
2008 0 : spin_unlock_irqrestore(&md->uevent_lock, flags);
2009 :
2010 0 : dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
2011 :
2012 0 : atomic_inc(&md->event_nr);
2013 0 : wake_up(&md->eventq);
2014 0 : dm_issue_global_event();
2015 0 : }
2016 :
2017 : /*
2018 : * Returns old map, which caller must destroy.
2019 : */
2020 0 : static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
2021 : struct queue_limits *limits)
2022 : {
2023 0 : struct dm_table *old_map;
2024 0 : struct request_queue *q = md->queue;
2025 0 : bool request_based = dm_table_request_based(t);
2026 0 : sector_t size;
2027 0 : int ret;
2028 :
2029 0 : lockdep_assert_held(&md->suspend_lock);
2030 :
2031 0 : size = dm_table_get_size(t);
2032 :
2033 : /*
2034 : * Wipe any geometry if the size of the table changed.
2035 : */
2036 0 : if (size != dm_get_size(md))
2037 0 : memset(&md->geometry, 0, sizeof(md->geometry));
2038 :
2039 0 : set_capacity_and_notify(md->disk, size);
2040 :
2041 0 : dm_table_event_callback(t, event_callback, md);
2042 :
2043 : /*
2044 : * The queue hasn't been stopped yet, if the old table type wasn't
2045 : * for request-based during suspension. So stop it to prevent
2046 : * I/O mapping before resume.
2047 : * This must be done before setting the queue restrictions,
2048 : * because request-based dm may be run just after the setting.
2049 : */
2050 0 : if (request_based)
2051 0 : dm_stop_queue(q);
2052 :
2053 0 : if (request_based) {
2054 : /*
2055 : * Leverage the fact that request-based DM targets are
2056 : * immutable singletons - used to optimize dm_mq_queue_rq.
2057 : */
2058 0 : md->immutable_target = dm_table_get_immutable_target(t);
2059 : }
2060 :
2061 0 : ret = __bind_mempools(md, t);
2062 0 : if (ret) {
2063 0 : old_map = ERR_PTR(ret);
2064 0 : goto out;
2065 : }
2066 :
2067 0 : old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2068 0 : rcu_assign_pointer(md->map, (void *)t);
2069 0 : md->immutable_target_type = dm_table_get_immutable_target_type(t);
2070 :
2071 0 : dm_table_set_restrictions(t, q, limits);
2072 0 : if (old_map)
2073 0 : dm_sync_table(md);
2074 :
2075 0 : out:
2076 0 : return old_map;
2077 : }
2078 :
2079 : /*
2080 : * Returns unbound table for the caller to free.
2081 : */
2082 0 : static struct dm_table *__unbind(struct mapped_device *md)
2083 : {
2084 0 : struct dm_table *map = rcu_dereference_protected(md->map, 1);
2085 :
2086 0 : if (!map)
2087 : return NULL;
2088 :
2089 0 : dm_table_event_callback(map, NULL, NULL);
2090 0 : RCU_INIT_POINTER(md->map, NULL);
2091 0 : dm_sync_table(md);
2092 :
2093 0 : return map;
2094 : }
2095 :
2096 : /*
2097 : * Constructor for a new device.
2098 : */
2099 0 : int dm_create(int minor, struct mapped_device **result)
2100 : {
2101 0 : int r;
2102 0 : struct mapped_device *md;
2103 :
2104 0 : md = alloc_dev(minor);
2105 0 : if (!md)
2106 : return -ENXIO;
2107 :
2108 0 : r = dm_sysfs_init(md);
2109 0 : if (r) {
2110 0 : free_dev(md);
2111 0 : return r;
2112 : }
2113 :
2114 0 : *result = md;
2115 0 : return 0;
2116 : }
2117 :
2118 : /*
2119 : * Functions to manage md->type.
2120 : * All are required to hold md->type_lock.
2121 : */
2122 0 : void dm_lock_md_type(struct mapped_device *md)
2123 : {
2124 0 : mutex_lock(&md->type_lock);
2125 0 : }
2126 :
2127 0 : void dm_unlock_md_type(struct mapped_device *md)
2128 : {
2129 0 : mutex_unlock(&md->type_lock);
2130 0 : }
2131 :
2132 0 : void dm_set_md_type(struct mapped_device *md, enum dm_queue_mode type)
2133 : {
2134 0 : BUG_ON(!mutex_is_locked(&md->type_lock));
2135 0 : md->type = type;
2136 0 : }
2137 :
2138 0 : enum dm_queue_mode dm_get_md_type(struct mapped_device *md)
2139 : {
2140 0 : return md->type;
2141 : }
2142 :
2143 0 : struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
2144 : {
2145 0 : return md->immutable_target_type;
2146 : }
2147 :
2148 : /*
2149 : * The queue_limits are only valid as long as you have a reference
2150 : * count on 'md'.
2151 : */
2152 0 : struct queue_limits *dm_get_queue_limits(struct mapped_device *md)
2153 : {
2154 0 : BUG_ON(!atomic_read(&md->holders));
2155 0 : return &md->queue->limits;
2156 : }
2157 : EXPORT_SYMBOL_GPL(dm_get_queue_limits);
2158 :
2159 : /*
2160 : * Setup the DM device's queue based on md's type
2161 : */
2162 0 : int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
2163 : {
2164 0 : int r;
2165 0 : struct queue_limits limits;
2166 0 : enum dm_queue_mode type = dm_get_md_type(md);
2167 :
2168 0 : switch (type) {
2169 0 : case DM_TYPE_REQUEST_BASED:
2170 0 : md->disk->fops = &dm_rq_blk_dops;
2171 0 : r = dm_mq_init_request_queue(md, t);
2172 0 : if (r) {
2173 0 : DMERR("Cannot initialize queue for request-based dm mapped device");
2174 0 : return r;
2175 : }
2176 : break;
2177 : case DM_TYPE_BIO_BASED:
2178 : case DM_TYPE_DAX_BIO_BASED:
2179 : break;
2180 : case DM_TYPE_NONE:
2181 0 : WARN_ON_ONCE(true);
2182 0 : break;
2183 : }
2184 :
2185 0 : r = dm_calculate_queue_limits(t, &limits);
2186 0 : if (r) {
2187 0 : DMERR("Cannot calculate initial queue limits");
2188 0 : return r;
2189 : }
2190 0 : dm_table_set_restrictions(t, md->queue, &limits);
2191 0 : blk_register_queue(md->disk);
2192 :
2193 0 : return 0;
2194 : }
2195 :
2196 0 : struct mapped_device *dm_get_md(dev_t dev)
2197 : {
2198 0 : struct mapped_device *md;
2199 0 : unsigned minor = MINOR(dev);
2200 :
2201 0 : if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
2202 : return NULL;
2203 :
2204 0 : spin_lock(&_minor_lock);
2205 :
2206 0 : md = idr_find(&_minor_idr, minor);
2207 0 : if (!md || md == MINOR_ALLOCED || (MINOR(disk_devt(dm_disk(md))) != minor) ||
2208 0 : test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
2209 0 : md = NULL;
2210 0 : goto out;
2211 : }
2212 0 : dm_get(md);
2213 0 : out:
2214 0 : spin_unlock(&_minor_lock);
2215 :
2216 0 : return md;
2217 : }
2218 : EXPORT_SYMBOL_GPL(dm_get_md);
2219 :
2220 0 : void *dm_get_mdptr(struct mapped_device *md)
2221 : {
2222 0 : return md->interface_ptr;
2223 : }
2224 :
2225 0 : void dm_set_mdptr(struct mapped_device *md, void *ptr)
2226 : {
2227 0 : md->interface_ptr = ptr;
2228 0 : }
2229 :
2230 0 : void dm_get(struct mapped_device *md)
2231 : {
2232 0 : atomic_inc(&md->holders);
2233 0 : BUG_ON(test_bit(DMF_FREEING, &md->flags));
2234 0 : }
2235 :
2236 0 : int dm_hold(struct mapped_device *md)
2237 : {
2238 0 : spin_lock(&_minor_lock);
2239 0 : if (test_bit(DMF_FREEING, &md->flags)) {
2240 0 : spin_unlock(&_minor_lock);
2241 0 : return -EBUSY;
2242 : }
2243 0 : dm_get(md);
2244 0 : spin_unlock(&_minor_lock);
2245 0 : return 0;
2246 : }
2247 : EXPORT_SYMBOL_GPL(dm_hold);
2248 :
2249 0 : const char *dm_device_name(struct mapped_device *md)
2250 : {
2251 0 : return md->name;
2252 : }
2253 : EXPORT_SYMBOL_GPL(dm_device_name);
2254 :
2255 0 : static void __dm_destroy(struct mapped_device *md, bool wait)
2256 : {
2257 0 : struct dm_table *map;
2258 0 : int srcu_idx;
2259 :
2260 0 : might_sleep();
2261 :
2262 0 : spin_lock(&_minor_lock);
2263 0 : idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
2264 0 : set_bit(DMF_FREEING, &md->flags);
2265 0 : spin_unlock(&_minor_lock);
2266 :
2267 0 : blk_set_queue_dying(md->queue);
2268 :
2269 : /*
2270 : * Take suspend_lock so that presuspend and postsuspend methods
2271 : * do not race with internal suspend.
2272 : */
2273 0 : mutex_lock(&md->suspend_lock);
2274 0 : map = dm_get_live_table(md, &srcu_idx);
2275 0 : if (!dm_suspended_md(md)) {
2276 0 : dm_table_presuspend_targets(map);
2277 0 : set_bit(DMF_SUSPENDED, &md->flags);
2278 0 : set_bit(DMF_POST_SUSPENDING, &md->flags);
2279 0 : dm_table_postsuspend_targets(map);
2280 : }
2281 : /* dm_put_live_table must be before msleep, otherwise deadlock is possible */
2282 0 : dm_put_live_table(md, srcu_idx);
2283 0 : mutex_unlock(&md->suspend_lock);
2284 :
2285 : /*
2286 : * Rare, but there may be I/O requests still going to complete,
2287 : * for example. Wait for all references to disappear.
2288 : * No one should increment the reference count of the mapped_device,
2289 : * after the mapped_device state becomes DMF_FREEING.
2290 : */
2291 0 : if (wait)
2292 0 : while (atomic_read(&md->holders))
2293 0 : msleep(1);
2294 0 : else if (atomic_read(&md->holders))
2295 0 : DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)",
2296 : dm_device_name(md), atomic_read(&md->holders));
2297 :
2298 0 : dm_sysfs_exit(md);
2299 0 : dm_table_destroy(__unbind(md));
2300 0 : free_dev(md);
2301 0 : }
2302 :
2303 0 : void dm_destroy(struct mapped_device *md)
2304 : {
2305 0 : __dm_destroy(md, true);
2306 0 : }
2307 :
2308 0 : void dm_destroy_immediate(struct mapped_device *md)
2309 : {
2310 0 : __dm_destroy(md, false);
2311 0 : }
2312 :
2313 0 : void dm_put(struct mapped_device *md)
2314 : {
2315 0 : atomic_dec(&md->holders);
2316 0 : }
2317 : EXPORT_SYMBOL_GPL(dm_put);
2318 :
2319 0 : static bool md_in_flight_bios(struct mapped_device *md)
2320 : {
2321 0 : int cpu;
2322 0 : struct block_device *part = dm_disk(md)->part0;
2323 0 : long sum = 0;
2324 :
2325 0 : for_each_possible_cpu(cpu) {
2326 0 : sum += part_stat_local_read_cpu(part, in_flight[0], cpu);
2327 0 : sum += part_stat_local_read_cpu(part, in_flight[1], cpu);
2328 : }
2329 :
2330 0 : return sum != 0;
2331 : }
2332 :
2333 0 : static int dm_wait_for_bios_completion(struct mapped_device *md, long task_state)
2334 : {
2335 0 : int r = 0;
2336 0 : DEFINE_WAIT(wait);
2337 :
2338 0 : while (true) {
2339 0 : prepare_to_wait(&md->wait, &wait, task_state);
2340 :
2341 0 : if (!md_in_flight_bios(md))
2342 : break;
2343 :
2344 0 : if (signal_pending_state(task_state, current)) {
2345 : r = -EINTR;
2346 : break;
2347 : }
2348 :
2349 0 : io_schedule();
2350 : }
2351 0 : finish_wait(&md->wait, &wait);
2352 :
2353 0 : return r;
2354 : }
2355 :
2356 0 : static int dm_wait_for_completion(struct mapped_device *md, long task_state)
2357 : {
2358 0 : int r = 0;
2359 :
2360 0 : if (!queue_is_mq(md->queue))
2361 0 : return dm_wait_for_bios_completion(md, task_state);
2362 :
2363 0 : while (true) {
2364 0 : if (!blk_mq_queue_inflight(md->queue))
2365 : break;
2366 :
2367 0 : if (signal_pending_state(task_state, current)) {
2368 : r = -EINTR;
2369 : break;
2370 : }
2371 :
2372 0 : msleep(5);
2373 : }
2374 :
2375 : return r;
2376 : }
2377 :
2378 : /*
2379 : * Process the deferred bios
2380 : */
2381 0 : static void dm_wq_work(struct work_struct *work)
2382 : {
2383 0 : struct mapped_device *md = container_of(work, struct mapped_device, work);
2384 0 : struct bio *bio;
2385 :
2386 0 : while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
2387 0 : spin_lock_irq(&md->deferred_lock);
2388 0 : bio = bio_list_pop(&md->deferred);
2389 0 : spin_unlock_irq(&md->deferred_lock);
2390 :
2391 0 : if (!bio)
2392 : break;
2393 :
2394 0 : submit_bio_noacct(bio);
2395 : }
2396 0 : }
2397 :
2398 0 : static void dm_queue_flush(struct mapped_device *md)
2399 : {
2400 0 : clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2401 0 : smp_mb__after_atomic();
2402 0 : queue_work(md->wq, &md->work);
2403 0 : }
2404 :
2405 : /*
2406 : * Swap in a new table, returning the old one for the caller to destroy.
2407 : */
2408 0 : struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table)
2409 : {
2410 0 : struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL);
2411 0 : struct queue_limits limits;
2412 0 : int r;
2413 :
2414 0 : mutex_lock(&md->suspend_lock);
2415 :
2416 : /* device must be suspended */
2417 0 : if (!dm_suspended_md(md))
2418 0 : goto out;
2419 :
2420 : /*
2421 : * If the new table has no data devices, retain the existing limits.
2422 : * This helps multipath with queue_if_no_path if all paths disappear,
2423 : * then new I/O is queued based on these limits, and then some paths
2424 : * reappear.
2425 : */
2426 0 : if (dm_table_has_no_data_devices(table)) {
2427 0 : live_map = dm_get_live_table_fast(md);
2428 0 : if (live_map)
2429 0 : limits = md->queue->limits;
2430 0 : dm_put_live_table_fast(md);
2431 : }
2432 :
2433 0 : if (!live_map) {
2434 0 : r = dm_calculate_queue_limits(table, &limits);
2435 0 : if (r) {
2436 0 : map = ERR_PTR(r);
2437 0 : goto out;
2438 : }
2439 : }
2440 :
2441 0 : map = __bind(md, table, &limits);
2442 0 : dm_issue_global_event();
2443 :
2444 0 : out:
2445 0 : mutex_unlock(&md->suspend_lock);
2446 0 : return map;
2447 : }
2448 :
2449 : /*
2450 : * Functions to lock and unlock any filesystem running on the
2451 : * device.
2452 : */
2453 0 : static int lock_fs(struct mapped_device *md)
2454 : {
2455 0 : int r;
2456 :
2457 0 : WARN_ON(test_bit(DMF_FROZEN, &md->flags));
2458 :
2459 0 : r = freeze_bdev(md->disk->part0);
2460 0 : if (!r)
2461 0 : set_bit(DMF_FROZEN, &md->flags);
2462 0 : return r;
2463 : }
2464 :
2465 0 : static void unlock_fs(struct mapped_device *md)
2466 : {
2467 0 : if (!test_bit(DMF_FROZEN, &md->flags))
2468 : return;
2469 0 : thaw_bdev(md->disk->part0);
2470 0 : clear_bit(DMF_FROZEN, &md->flags);
2471 : }
2472 :
2473 : /*
2474 : * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG
2475 : * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE
2476 : * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY
2477 : *
2478 : * If __dm_suspend returns 0, the device is completely quiescent
2479 : * now. There is no request-processing activity. All new requests
2480 : * are being added to md->deferred list.
2481 : */
2482 0 : static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
2483 : unsigned suspend_flags, long task_state,
2484 : int dmf_suspended_flag)
2485 : {
2486 0 : bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG;
2487 0 : bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG;
2488 0 : int r;
2489 :
2490 0 : lockdep_assert_held(&md->suspend_lock);
2491 :
2492 : /*
2493 : * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
2494 : * This flag is cleared before dm_suspend returns.
2495 : */
2496 0 : if (noflush)
2497 0 : set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2498 : else
2499 : DMDEBUG("%s: suspending with flush", dm_device_name(md));
2500 :
2501 : /*
2502 : * This gets reverted if there's an error later and the targets
2503 : * provide the .presuspend_undo hook.
2504 : */
2505 0 : dm_table_presuspend_targets(map);
2506 :
2507 : /*
2508 : * Flush I/O to the device.
2509 : * Any I/O submitted after lock_fs() may not be flushed.
2510 : * noflush takes precedence over do_lockfs.
2511 : * (lock_fs() flushes I/Os and waits for them to complete.)
2512 : */
2513 0 : if (!noflush && do_lockfs) {
2514 0 : r = lock_fs(md);
2515 0 : if (r) {
2516 0 : dm_table_presuspend_undo_targets(map);
2517 0 : return r;
2518 : }
2519 : }
2520 :
2521 : /*
2522 : * Here we must make sure that no processes are submitting requests
2523 : * to target drivers i.e. no one may be executing
2524 : * __split_and_process_bio from dm_submit_bio.
2525 : *
2526 : * To get all processes out of __split_and_process_bio in dm_submit_bio,
2527 : * we take the write lock. To prevent any process from reentering
2528 : * __split_and_process_bio from dm_submit_bio and quiesce the thread
2529 : * (dm_wq_work), we set DMF_BLOCK_IO_FOR_SUSPEND and call
2530 : * flush_workqueue(md->wq).
2531 : */
2532 0 : set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2533 0 : if (map)
2534 0 : synchronize_srcu(&md->io_barrier);
2535 :
2536 : /*
2537 : * Stop md->queue before flushing md->wq in case request-based
2538 : * dm defers requests to md->wq from md->queue.
2539 : */
2540 0 : if (dm_request_based(md))
2541 0 : dm_stop_queue(md->queue);
2542 :
2543 0 : flush_workqueue(md->wq);
2544 :
2545 : /*
2546 : * At this point no more requests are entering target request routines.
2547 : * We call dm_wait_for_completion to wait for all existing requests
2548 : * to finish.
2549 : */
2550 0 : r = dm_wait_for_completion(md, task_state);
2551 0 : if (!r)
2552 0 : set_bit(dmf_suspended_flag, &md->flags);
2553 :
2554 0 : if (noflush)
2555 0 : clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2556 0 : if (map)
2557 0 : synchronize_srcu(&md->io_barrier);
2558 :
2559 : /* were we interrupted ? */
2560 0 : if (r < 0) {
2561 0 : dm_queue_flush(md);
2562 :
2563 0 : if (dm_request_based(md))
2564 0 : dm_start_queue(md->queue);
2565 :
2566 0 : unlock_fs(md);
2567 0 : dm_table_presuspend_undo_targets(map);
2568 : /* pushback list is already flushed, so skip flush */
2569 : }
2570 :
2571 : return r;
2572 : }
2573 :
2574 : /*
2575 : * We need to be able to change a mapping table under a mounted
2576 : * filesystem. For example we might want to move some data in
2577 : * the background. Before the table can be swapped with
2578 : * dm_bind_table, dm_suspend must be called to flush any in
2579 : * flight bios and ensure that any further io gets deferred.
2580 : */
2581 : /*
2582 : * Suspend mechanism in request-based dm.
2583 : *
2584 : * 1. Flush all I/Os by lock_fs() if needed.
2585 : * 2. Stop dispatching any I/O by stopping the request_queue.
2586 : * 3. Wait for all in-flight I/Os to be completed or requeued.
2587 : *
2588 : * To abort suspend, start the request_queue.
2589 : */
2590 0 : int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
2591 : {
2592 0 : struct dm_table *map = NULL;
2593 0 : int r = 0;
2594 :
2595 0 : retry:
2596 0 : mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2597 :
2598 0 : if (dm_suspended_md(md)) {
2599 0 : r = -EINVAL;
2600 0 : goto out_unlock;
2601 : }
2602 :
2603 0 : if (dm_suspended_internally_md(md)) {
2604 : /* already internally suspended, wait for internal resume */
2605 0 : mutex_unlock(&md->suspend_lock);
2606 0 : r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2607 0 : if (r)
2608 0 : return r;
2609 0 : goto retry;
2610 : }
2611 :
2612 0 : map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2613 :
2614 0 : r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED);
2615 0 : if (r)
2616 0 : goto out_unlock;
2617 :
2618 0 : set_bit(DMF_POST_SUSPENDING, &md->flags);
2619 0 : dm_table_postsuspend_targets(map);
2620 0 : clear_bit(DMF_POST_SUSPENDING, &md->flags);
2621 :
2622 0 : out_unlock:
2623 0 : mutex_unlock(&md->suspend_lock);
2624 0 : return r;
2625 : }
2626 :
2627 0 : static int __dm_resume(struct mapped_device *md, struct dm_table *map)
2628 : {
2629 0 : if (map) {
2630 0 : int r = dm_table_resume_targets(map);
2631 0 : if (r)
2632 : return r;
2633 : }
2634 :
2635 0 : dm_queue_flush(md);
2636 :
2637 : /*
2638 : * Flushing deferred I/Os must be done after targets are resumed
2639 : * so that mapping of targets can work correctly.
2640 : * Request-based dm is queueing the deferred I/Os in its request_queue.
2641 : */
2642 0 : if (dm_request_based(md))
2643 0 : dm_start_queue(md->queue);
2644 :
2645 0 : unlock_fs(md);
2646 :
2647 0 : return 0;
2648 : }
2649 :
2650 0 : int dm_resume(struct mapped_device *md)
2651 : {
2652 0 : int r;
2653 0 : struct dm_table *map = NULL;
2654 :
2655 0 : retry:
2656 0 : r = -EINVAL;
2657 0 : mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2658 :
2659 0 : if (!dm_suspended_md(md))
2660 0 : goto out;
2661 :
2662 0 : if (dm_suspended_internally_md(md)) {
2663 : /* already internally suspended, wait for internal resume */
2664 0 : mutex_unlock(&md->suspend_lock);
2665 0 : r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2666 0 : if (r)
2667 0 : return r;
2668 0 : goto retry;
2669 : }
2670 :
2671 0 : map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2672 0 : if (!map || !dm_table_get_size(map))
2673 0 : goto out;
2674 :
2675 0 : r = __dm_resume(md, map);
2676 0 : if (r)
2677 0 : goto out;
2678 :
2679 0 : clear_bit(DMF_SUSPENDED, &md->flags);
2680 0 : out:
2681 0 : mutex_unlock(&md->suspend_lock);
2682 :
2683 0 : return r;
2684 : }
2685 :
2686 : /*
2687 : * Internal suspend/resume works like userspace-driven suspend. It waits
2688 : * until all bios finish and prevents issuing new bios to the target drivers.
2689 : * It may be used only from the kernel.
2690 : */
2691 :
2692 0 : static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags)
2693 : {
2694 0 : struct dm_table *map = NULL;
2695 :
2696 0 : lockdep_assert_held(&md->suspend_lock);
2697 :
2698 0 : if (md->internal_suspend_count++)
2699 : return; /* nested internal suspend */
2700 :
2701 0 : if (dm_suspended_md(md)) {
2702 0 : set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2703 0 : return; /* nest suspend */
2704 : }
2705 :
2706 0 : map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2707 :
2708 : /*
2709 : * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is
2710 : * supported. Properly supporting a TASK_INTERRUPTIBLE internal suspend
2711 : * would require changing .presuspend to return an error -- avoid this
2712 : * until there is a need for more elaborate variants of internal suspend.
2713 : */
2714 0 : (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE,
2715 : DMF_SUSPENDED_INTERNALLY);
2716 :
2717 0 : set_bit(DMF_POST_SUSPENDING, &md->flags);
2718 0 : dm_table_postsuspend_targets(map);
2719 0 : clear_bit(DMF_POST_SUSPENDING, &md->flags);
2720 : }
2721 :
2722 0 : static void __dm_internal_resume(struct mapped_device *md)
2723 : {
2724 0 : BUG_ON(!md->internal_suspend_count);
2725 :
2726 0 : if (--md->internal_suspend_count)
2727 : return; /* resume from nested internal suspend */
2728 :
2729 0 : if (dm_suspended_md(md))
2730 0 : goto done; /* resume from nested suspend */
2731 :
2732 : /*
2733 : * NOTE: existing callers don't need to call dm_table_resume_targets
2734 : * (which may fail -- so best to avoid it for now by passing NULL map)
2735 : */
2736 0 : (void) __dm_resume(md, NULL);
2737 :
2738 0 : done:
2739 0 : clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2740 0 : smp_mb__after_atomic();
2741 0 : wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY);
2742 : }
2743 :
2744 0 : void dm_internal_suspend_noflush(struct mapped_device *md)
2745 : {
2746 0 : mutex_lock(&md->suspend_lock);
2747 0 : __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG);
2748 0 : mutex_unlock(&md->suspend_lock);
2749 0 : }
2750 : EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush);
2751 :
2752 0 : void dm_internal_resume(struct mapped_device *md)
2753 : {
2754 0 : mutex_lock(&md->suspend_lock);
2755 0 : __dm_internal_resume(md);
2756 0 : mutex_unlock(&md->suspend_lock);
2757 0 : }
2758 : EXPORT_SYMBOL_GPL(dm_internal_resume);
2759 :
2760 : /*
2761 : * Fast variants of internal suspend/resume hold md->suspend_lock,
2762 : * which prevents interaction with userspace-driven suspend.
2763 : */
2764 :
2765 0 : void dm_internal_suspend_fast(struct mapped_device *md)
2766 : {
2767 0 : mutex_lock(&md->suspend_lock);
2768 0 : if (dm_suspended_md(md) || dm_suspended_internally_md(md))
2769 0 : return;
2770 :
2771 0 : set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2772 0 : synchronize_srcu(&md->io_barrier);
2773 0 : flush_workqueue(md->wq);
2774 0 : dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
2775 : }
2776 : EXPORT_SYMBOL_GPL(dm_internal_suspend_fast);
2777 :
2778 0 : void dm_internal_resume_fast(struct mapped_device *md)
2779 : {
2780 0 : if (dm_suspended_md(md) || dm_suspended_internally_md(md))
2781 0 : goto done;
2782 :
2783 0 : dm_queue_flush(md);
2784 :
2785 0 : done:
2786 0 : mutex_unlock(&md->suspend_lock);
2787 0 : }
2788 : EXPORT_SYMBOL_GPL(dm_internal_resume_fast);
2789 :
2790 : /*-----------------------------------------------------------------
2791 : * Event notification.
2792 : *---------------------------------------------------------------*/
2793 0 : int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,
2794 : unsigned cookie)
2795 : {
2796 0 : int r;
2797 0 : unsigned noio_flag;
2798 0 : char udev_cookie[DM_COOKIE_LENGTH];
2799 0 : char *envp[] = { udev_cookie, NULL };
2800 :
2801 0 : noio_flag = memalloc_noio_save();
2802 :
2803 0 : if (!cookie)
2804 0 : r = kobject_uevent(&disk_to_dev(md->disk)->kobj, action);
2805 : else {
2806 0 : snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u",
2807 : DM_COOKIE_ENV_VAR_NAME, cookie);
2808 0 : r = kobject_uevent_env(&disk_to_dev(md->disk)->kobj,
2809 : action, envp);
2810 : }
2811 :
2812 0 : memalloc_noio_restore(noio_flag);
2813 :
2814 0 : return r;
2815 : }
2816 :
2817 0 : uint32_t dm_next_uevent_seq(struct mapped_device *md)
2818 : {
2819 0 : return atomic_add_return(1, &md->uevent_seq);
2820 : }
2821 :
2822 0 : uint32_t dm_get_event_nr(struct mapped_device *md)
2823 : {
2824 0 : return atomic_read(&md->event_nr);
2825 : }
2826 :
2827 0 : int dm_wait_event(struct mapped_device *md, int event_nr)
2828 : {
2829 0 : return wait_event_interruptible(md->eventq,
2830 : (event_nr != atomic_read(&md->event_nr)));
2831 : }
2832 :
2833 0 : void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
2834 : {
2835 0 : unsigned long flags;
2836 :
2837 0 : spin_lock_irqsave(&md->uevent_lock, flags);
2838 0 : list_add(elist, &md->uevent_list);
2839 0 : spin_unlock_irqrestore(&md->uevent_lock, flags);
2840 0 : }
2841 :
2842 : /*
2843 : * The gendisk is only valid as long as you have a reference
2844 : * count on 'md'.
2845 : */
2846 0 : struct gendisk *dm_disk(struct mapped_device *md)
2847 : {
2848 0 : return md->disk;
2849 : }
2850 : EXPORT_SYMBOL_GPL(dm_disk);
2851 :
2852 0 : struct kobject *dm_kobject(struct mapped_device *md)
2853 : {
2854 0 : return &md->kobj_holder.kobj;
2855 : }
2856 :
2857 0 : struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
2858 : {
2859 0 : struct mapped_device *md;
2860 :
2861 0 : md = container_of(kobj, struct mapped_device, kobj_holder.kobj);
2862 :
2863 0 : spin_lock(&_minor_lock);
2864 0 : if (test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
2865 0 : md = NULL;
2866 0 : goto out;
2867 : }
2868 0 : dm_get(md);
2869 0 : out:
2870 0 : spin_unlock(&_minor_lock);
2871 :
2872 0 : return md;
2873 : }
2874 :
2875 0 : int dm_suspended_md(struct mapped_device *md)
2876 : {
2877 0 : return test_bit(DMF_SUSPENDED, &md->flags);
2878 : }
2879 :
2880 0 : static int dm_post_suspending_md(struct mapped_device *md)
2881 : {
2882 0 : return test_bit(DMF_POST_SUSPENDING, &md->flags);
2883 : }
2884 :
2885 0 : int dm_suspended_internally_md(struct mapped_device *md)
2886 : {
2887 0 : return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2888 : }
2889 :
2890 0 : int dm_test_deferred_remove_flag(struct mapped_device *md)
2891 : {
2892 0 : return test_bit(DMF_DEFERRED_REMOVE, &md->flags);
2893 : }
2894 :
2895 0 : int dm_suspended(struct dm_target *ti)
2896 : {
2897 0 : return dm_suspended_md(ti->table->md);
2898 : }
2899 : EXPORT_SYMBOL_GPL(dm_suspended);
2900 :
2901 0 : int dm_post_suspending(struct dm_target *ti)
2902 : {
2903 0 : return dm_post_suspending_md(ti->table->md);
2904 : }
2905 : EXPORT_SYMBOL_GPL(dm_post_suspending);
2906 :
2907 0 : int dm_noflush_suspending(struct dm_target *ti)
2908 : {
2909 0 : return __noflush_suspending(ti->table->md);
2910 : }
2911 : EXPORT_SYMBOL_GPL(dm_noflush_suspending);
2912 :
2913 0 : struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_queue_mode type,
2914 : unsigned integrity, unsigned per_io_data_size,
2915 : unsigned min_pool_size)
2916 : {
2917 0 : struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id);
2918 0 : unsigned int pool_size = 0;
2919 0 : unsigned int front_pad, io_front_pad;
2920 0 : int ret;
2921 :
2922 0 : if (!pools)
2923 : return NULL;
2924 :
2925 0 : switch (type) {
2926 : case DM_TYPE_BIO_BASED:
2927 : case DM_TYPE_DAX_BIO_BASED:
2928 0 : pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size);
2929 0 : front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + DM_TARGET_IO_BIO_OFFSET;
2930 0 : io_front_pad = roundup(per_io_data_size, __alignof__(struct dm_io)) + DM_IO_BIO_OFFSET;
2931 0 : ret = bioset_init(&pools->io_bs, pool_size, io_front_pad, 0);
2932 0 : if (ret)
2933 0 : goto out;
2934 0 : if (integrity && bioset_integrity_create(&pools->io_bs, pool_size))
2935 : goto out;
2936 : break;
2937 0 : case DM_TYPE_REQUEST_BASED:
2938 0 : pool_size = max(dm_get_reserved_rq_based_ios(), min_pool_size);
2939 0 : front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
2940 : /* per_io_data_size is used for blk-mq pdu at queue allocation */
2941 0 : break;
2942 0 : default:
2943 0 : BUG();
2944 : }
2945 :
2946 0 : ret = bioset_init(&pools->bs, pool_size, front_pad, 0);
2947 0 : if (ret)
2948 0 : goto out;
2949 :
2950 0 : if (integrity && bioset_integrity_create(&pools->bs, pool_size))
2951 : goto out;
2952 :
2953 : return pools;
2954 :
2955 0 : out:
2956 0 : dm_free_md_mempools(pools);
2957 :
2958 0 : return NULL;
2959 : }
2960 :
2961 0 : void dm_free_md_mempools(struct dm_md_mempools *pools)
2962 : {
2963 0 : if (!pools)
2964 : return;
2965 :
2966 0 : bioset_exit(&pools->bs);
2967 0 : bioset_exit(&pools->io_bs);
2968 :
2969 0 : kfree(pools);
2970 : }
2971 :
2972 : struct dm_pr {
2973 : u64 old_key;
2974 : u64 new_key;
2975 : u32 flags;
2976 : bool fail_early;
2977 : };
2978 :
2979 0 : static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn,
2980 : void *data)
2981 : {
2982 0 : struct mapped_device *md = bdev->bd_disk->private_data;
2983 0 : struct dm_table *table;
2984 0 : struct dm_target *ti;
2985 0 : int ret = -ENOTTY, srcu_idx;
2986 :
2987 0 : table = dm_get_live_table(md, &srcu_idx);
2988 0 : if (!table || !dm_table_get_size(table))
2989 0 : goto out;
2990 :
2991 : /* We only support devices that have a single target */
2992 0 : if (dm_table_get_num_targets(table) != 1)
2993 0 : goto out;
2994 0 : ti = dm_table_get_target(table, 0);
2995 :
2996 0 : ret = -EINVAL;
2997 0 : if (!ti->type->iterate_devices)
2998 0 : goto out;
2999 :
3000 0 : ret = ti->type->iterate_devices(ti, fn, data);
3001 0 : out:
3002 0 : dm_put_live_table(md, srcu_idx);
3003 0 : return ret;
3004 : }
3005 :
3006 : /*
3007 : * For register / unregister we need to manually call out to every path.
3008 : */
3009 0 : static int __dm_pr_register(struct dm_target *ti, struct dm_dev *dev,
3010 : sector_t start, sector_t len, void *data)
3011 : {
3012 0 : struct dm_pr *pr = data;
3013 0 : const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
3014 :
3015 0 : if (!ops || !ops->pr_register)
3016 : return -EOPNOTSUPP;
3017 0 : return ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags);
3018 : }
3019 :
3020 0 : static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key,
3021 : u32 flags)
3022 : {
3023 0 : struct dm_pr pr = {
3024 : .old_key = old_key,
3025 : .new_key = new_key,
3026 : .flags = flags,
3027 : .fail_early = true,
3028 : };
3029 0 : int ret;
3030 :
3031 0 : ret = dm_call_pr(bdev, __dm_pr_register, &pr);
3032 0 : if (ret && new_key) {
3033 : /* unregister all paths if we failed to register any path */
3034 0 : pr.old_key = new_key;
3035 0 : pr.new_key = 0;
3036 0 : pr.flags = 0;
3037 0 : pr.fail_early = false;
3038 0 : dm_call_pr(bdev, __dm_pr_register, &pr);
3039 : }
3040 :
3041 0 : return ret;
3042 : }
3043 :
3044 0 : static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type,
3045 : u32 flags)
3046 : {
3047 0 : struct mapped_device *md = bdev->bd_disk->private_data;
3048 0 : const struct pr_ops *ops;
3049 0 : int r, srcu_idx;
3050 :
3051 0 : r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
3052 0 : if (r < 0)
3053 0 : goto out;
3054 :
3055 0 : ops = bdev->bd_disk->fops->pr_ops;
3056 0 : if (ops && ops->pr_reserve)
3057 0 : r = ops->pr_reserve(bdev, key, type, flags);
3058 : else
3059 : r = -EOPNOTSUPP;
3060 0 : out:
3061 0 : dm_unprepare_ioctl(md, srcu_idx);
3062 0 : return r;
3063 : }
3064 :
3065 0 : static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
3066 : {
3067 0 : struct mapped_device *md = bdev->bd_disk->private_data;
3068 0 : const struct pr_ops *ops;
3069 0 : int r, srcu_idx;
3070 :
3071 0 : r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
3072 0 : if (r < 0)
3073 0 : goto out;
3074 :
3075 0 : ops = bdev->bd_disk->fops->pr_ops;
3076 0 : if (ops && ops->pr_release)
3077 0 : r = ops->pr_release(bdev, key, type);
3078 : else
3079 : r = -EOPNOTSUPP;
3080 0 : out:
3081 0 : dm_unprepare_ioctl(md, srcu_idx);
3082 0 : return r;
3083 : }
3084 :
3085 0 : static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key,
3086 : enum pr_type type, bool abort)
3087 : {
3088 0 : struct mapped_device *md = bdev->bd_disk->private_data;
3089 0 : const struct pr_ops *ops;
3090 0 : int r, srcu_idx;
3091 :
3092 0 : r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
3093 0 : if (r < 0)
3094 0 : goto out;
3095 :
3096 0 : ops = bdev->bd_disk->fops->pr_ops;
3097 0 : if (ops && ops->pr_preempt)
3098 0 : r = ops->pr_preempt(bdev, old_key, new_key, type, abort);
3099 : else
3100 : r = -EOPNOTSUPP;
3101 0 : out:
3102 0 : dm_unprepare_ioctl(md, srcu_idx);
3103 0 : return r;
3104 : }
3105 :
3106 0 : static int dm_pr_clear(struct block_device *bdev, u64 key)
3107 : {
3108 0 : struct mapped_device *md = bdev->bd_disk->private_data;
3109 0 : const struct pr_ops *ops;
3110 0 : int r, srcu_idx;
3111 :
3112 0 : r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
3113 0 : if (r < 0)
3114 0 : goto out;
3115 :
3116 0 : ops = bdev->bd_disk->fops->pr_ops;
3117 0 : if (ops && ops->pr_clear)
3118 0 : r = ops->pr_clear(bdev, key);
3119 : else
3120 : r = -EOPNOTSUPP;
3121 0 : out:
3122 0 : dm_unprepare_ioctl(md, srcu_idx);
3123 0 : return r;
3124 : }
3125 :
3126 : static const struct pr_ops dm_pr_ops = {
3127 : .pr_register = dm_pr_register,
3128 : .pr_reserve = dm_pr_reserve,
3129 : .pr_release = dm_pr_release,
3130 : .pr_preempt = dm_pr_preempt,
3131 : .pr_clear = dm_pr_clear,
3132 : };
3133 :
3134 : static const struct block_device_operations dm_blk_dops = {
3135 : .submit_bio = dm_submit_bio,
3136 : .open = dm_blk_open,
3137 : .release = dm_blk_close,
3138 : .ioctl = dm_blk_ioctl,
3139 : .getgeo = dm_blk_getgeo,
3140 : .report_zones = dm_blk_report_zones,
3141 : .pr_ops = &dm_pr_ops,
3142 : .owner = THIS_MODULE
3143 : };
3144 :
3145 : static const struct block_device_operations dm_rq_blk_dops = {
3146 : .open = dm_blk_open,
3147 : .release = dm_blk_close,
3148 : .ioctl = dm_blk_ioctl,
3149 : .getgeo = dm_blk_getgeo,
3150 : .pr_ops = &dm_pr_ops,
3151 : .owner = THIS_MODULE
3152 : };
3153 :
3154 : static const struct dax_operations dm_dax_ops = {
3155 : .direct_access = dm_dax_direct_access,
3156 : .dax_supported = dm_dax_supported,
3157 : .copy_from_iter = dm_dax_copy_from_iter,
3158 : .copy_to_iter = dm_dax_copy_to_iter,
3159 : .zero_page_range = dm_dax_zero_page_range,
3160 : };
3161 :
3162 : /*
3163 : * module hooks
3164 : */
3165 : module_init(dm_init);
3166 : module_exit(dm_exit);
3167 :
3168 : module_param(major, uint, 0);
3169 : MODULE_PARM_DESC(major, "The major number of the device mapper");
3170 :
3171 : module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR);
3172 : MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools");
3173 :
3174 : module_param(dm_numa_node, int, S_IRUGO | S_IWUSR);
3175 : MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations");
3176 :
3177 : module_param(swap_bios, int, S_IRUGO | S_IWUSR);
3178 : MODULE_PARM_DESC(swap_bios, "Maximum allowed inflight swap IOs");
3179 :
3180 : MODULE_DESCRIPTION(DM_NAME " driver");
3181 : MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
3182 : MODULE_LICENSE("GPL");
|
