Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * Block device elevator/IO-scheduler.
4 : *
5 : * Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
6 : *
7 : * 30042000 Jens Axboe <axboe@kernel.dk> :
8 : *
9 : * Split the elevator a bit so that it is possible to choose a different
10 : * one or even write a new "plug in". There are three pieces:
11 : * - elevator_fn, inserts a new request in the queue list
12 : * - elevator_merge_fn, decides whether a new buffer can be merged with
13 : * an existing request
14 : * - elevator_dequeue_fn, called when a request is taken off the active list
15 : *
16 : * 20082000 Dave Jones <davej@suse.de> :
17 : * Removed tests for max-bomb-segments, which was breaking elvtune
18 : * when run without -bN
19 : *
20 : * Jens:
21 : * - Rework again to work with bio instead of buffer_heads
22 : * - loose bi_dev comparisons, partition handling is right now
23 : * - completely modularize elevator setup and teardown
24 : *
25 : */
26 : #include <linux/kernel.h>
27 : #include <linux/fs.h>
28 : #include <linux/blkdev.h>
29 : #include <linux/elevator.h>
30 : #include <linux/bio.h>
31 : #include <linux/module.h>
32 : #include <linux/slab.h>
33 : #include <linux/init.h>
34 : #include <linux/compiler.h>
35 : #include <linux/blktrace_api.h>
36 : #include <linux/hash.h>
37 : #include <linux/uaccess.h>
38 : #include <linux/pm_runtime.h>
39 : #include <linux/blk-cgroup.h>
40 :
41 : #include <trace/events/block.h>
42 :
43 : #include "blk.h"
44 : #include "blk-mq-sched.h"
45 : #include "blk-pm.h"
46 : #include "blk-wbt.h"
47 :
48 : static DEFINE_SPINLOCK(elv_list_lock);
49 : static LIST_HEAD(elv_list);
50 :
51 : /*
52 : * Merge hash stuff.
53 : */
54 : #define rq_hash_key(rq) (blk_rq_pos(rq) + blk_rq_sectors(rq))
55 :
56 : /*
57 : * Query io scheduler to see if the current process issuing bio may be
58 : * merged with rq.
59 : */
60 0 : static int elv_iosched_allow_bio_merge(struct request *rq, struct bio *bio)
61 : {
62 0 : struct request_queue *q = rq->q;
63 0 : struct elevator_queue *e = q->elevator;
64 :
65 0 : if (e->type->ops.allow_merge)
66 0 : return e->type->ops.allow_merge(q, rq, bio);
67 :
68 : return 1;
69 : }
70 :
71 : /*
72 : * can we safely merge with this request?
73 : */
74 0 : bool elv_bio_merge_ok(struct request *rq, struct bio *bio)
75 : {
76 0 : if (!blk_rq_merge_ok(rq, bio))
77 : return false;
78 :
79 0 : if (!elv_iosched_allow_bio_merge(rq, bio))
80 0 : return false;
81 :
82 : return true;
83 : }
84 : EXPORT_SYMBOL(elv_bio_merge_ok);
85 :
86 0 : static inline bool elv_support_features(unsigned int elv_features,
87 : unsigned int required_features)
88 : {
89 0 : return (required_features & elv_features) == required_features;
90 : }
91 :
92 : /**
93 : * elevator_match - Test an elevator name and features
94 : * @e: Scheduler to test
95 : * @name: Elevator name to test
96 : * @required_features: Features that the elevator must provide
97 : *
98 : * Return true if the elevator @e name matches @name and if @e provides all
99 : * the features specified by @required_features.
100 : */
101 0 : static bool elevator_match(const struct elevator_type *e, const char *name,
102 : unsigned int required_features)
103 : {
104 0 : if (!elv_support_features(e->elevator_features, required_features))
105 : return false;
106 0 : if (!strcmp(e->elevator_name, name))
107 : return true;
108 0 : if (e->elevator_alias && !strcmp(e->elevator_alias, name))
109 0 : return true;
110 :
111 : return false;
112 : }
113 :
114 : /**
115 : * elevator_find - Find an elevator
116 : * @name: Name of the elevator to find
117 : * @required_features: Features that the elevator must provide
118 : *
119 : * Return the first registered scheduler with name @name and supporting the
120 : * features @required_features and NULL otherwise.
121 : */
122 9 : static struct elevator_type *elevator_find(const char *name,
123 : unsigned int required_features)
124 : {
125 9 : struct elevator_type *e;
126 :
127 9 : list_for_each_entry(e, &elv_list, list) {
128 0 : if (elevator_match(e, name, required_features))
129 0 : return e;
130 : }
131 :
132 : return NULL;
133 : }
134 :
135 0 : static void elevator_put(struct elevator_type *e)
136 : {
137 0 : module_put(e->elevator_owner);
138 0 : }
139 :
140 9 : static struct elevator_type *elevator_get(struct request_queue *q,
141 : const char *name, bool try_loading)
142 : {
143 9 : struct elevator_type *e;
144 :
145 9 : spin_lock(&elv_list_lock);
146 :
147 9 : e = elevator_find(name, q->required_elevator_features);
148 9 : if (!e && try_loading) {
149 0 : spin_unlock(&elv_list_lock);
150 0 : request_module("%s-iosched", name);
151 0 : spin_lock(&elv_list_lock);
152 0 : e = elevator_find(name, q->required_elevator_features);
153 : }
154 :
155 9 : if (e && !try_module_get(e->elevator_owner))
156 : e = NULL;
157 :
158 9 : spin_unlock(&elv_list_lock);
159 9 : return e;
160 : }
161 :
162 : static struct kobj_type elv_ktype;
163 :
164 0 : struct elevator_queue *elevator_alloc(struct request_queue *q,
165 : struct elevator_type *e)
166 : {
167 0 : struct elevator_queue *eq;
168 :
169 0 : eq = kzalloc_node(sizeof(*eq), GFP_KERNEL, q->node);
170 0 : if (unlikely(!eq))
171 : return NULL;
172 :
173 0 : eq->type = e;
174 0 : kobject_init(&eq->kobj, &elv_ktype);
175 0 : mutex_init(&eq->sysfs_lock);
176 0 : hash_init(eq->hash);
177 :
178 : return eq;
179 : }
180 : EXPORT_SYMBOL(elevator_alloc);
181 :
182 0 : static void elevator_release(struct kobject *kobj)
183 : {
184 0 : struct elevator_queue *e;
185 :
186 0 : e = container_of(kobj, struct elevator_queue, kobj);
187 0 : elevator_put(e->type);
188 0 : kfree(e);
189 0 : }
190 :
191 0 : void __elevator_exit(struct request_queue *q, struct elevator_queue *e)
192 : {
193 0 : mutex_lock(&e->sysfs_lock);
194 0 : blk_mq_exit_sched(q, e);
195 0 : mutex_unlock(&e->sysfs_lock);
196 :
197 0 : kobject_put(&e->kobj);
198 0 : }
199 :
200 0 : static inline void __elv_rqhash_del(struct request *rq)
201 : {
202 0 : hash_del(&rq->hash);
203 0 : rq->rq_flags &= ~RQF_HASHED;
204 0 : }
205 :
206 0 : void elv_rqhash_del(struct request_queue *q, struct request *rq)
207 : {
208 0 : if (ELV_ON_HASH(rq))
209 0 : __elv_rqhash_del(rq);
210 0 : }
211 : EXPORT_SYMBOL_GPL(elv_rqhash_del);
212 :
213 0 : void elv_rqhash_add(struct request_queue *q, struct request *rq)
214 : {
215 0 : struct elevator_queue *e = q->elevator;
216 :
217 0 : BUG_ON(ELV_ON_HASH(rq));
218 0 : hash_add(e->hash, &rq->hash, rq_hash_key(rq));
219 0 : rq->rq_flags |= RQF_HASHED;
220 0 : }
221 : EXPORT_SYMBOL_GPL(elv_rqhash_add);
222 :
223 0 : void elv_rqhash_reposition(struct request_queue *q, struct request *rq)
224 : {
225 0 : __elv_rqhash_del(rq);
226 0 : elv_rqhash_add(q, rq);
227 0 : }
228 :
229 0 : struct request *elv_rqhash_find(struct request_queue *q, sector_t offset)
230 : {
231 0 : struct elevator_queue *e = q->elevator;
232 0 : struct hlist_node *next;
233 0 : struct request *rq;
234 :
235 0 : hash_for_each_possible_safe(e->hash, rq, next, hash, offset) {
236 0 : BUG_ON(!ELV_ON_HASH(rq));
237 :
238 0 : if (unlikely(!rq_mergeable(rq))) {
239 0 : __elv_rqhash_del(rq);
240 0 : continue;
241 : }
242 :
243 0 : if (rq_hash_key(rq) == offset)
244 0 : return rq;
245 : }
246 :
247 : return NULL;
248 : }
249 :
250 : /*
251 : * RB-tree support functions for inserting/lookup/removal of requests
252 : * in a sorted RB tree.
253 : */
254 0 : void elv_rb_add(struct rb_root *root, struct request *rq)
255 : {
256 0 : struct rb_node **p = &root->rb_node;
257 0 : struct rb_node *parent = NULL;
258 0 : struct request *__rq;
259 :
260 0 : while (*p) {
261 0 : parent = *p;
262 0 : __rq = rb_entry(parent, struct request, rb_node);
263 :
264 0 : if (blk_rq_pos(rq) < blk_rq_pos(__rq))
265 0 : p = &(*p)->rb_left;
266 0 : else if (blk_rq_pos(rq) >= blk_rq_pos(__rq))
267 0 : p = &(*p)->rb_right;
268 : }
269 :
270 0 : rb_link_node(&rq->rb_node, parent, p);
271 0 : rb_insert_color(&rq->rb_node, root);
272 0 : }
273 : EXPORT_SYMBOL(elv_rb_add);
274 :
275 0 : void elv_rb_del(struct rb_root *root, struct request *rq)
276 : {
277 0 : BUG_ON(RB_EMPTY_NODE(&rq->rb_node));
278 0 : rb_erase(&rq->rb_node, root);
279 0 : RB_CLEAR_NODE(&rq->rb_node);
280 0 : }
281 : EXPORT_SYMBOL(elv_rb_del);
282 :
283 0 : struct request *elv_rb_find(struct rb_root *root, sector_t sector)
284 : {
285 0 : struct rb_node *n = root->rb_node;
286 0 : struct request *rq;
287 :
288 0 : while (n) {
289 0 : rq = rb_entry(n, struct request, rb_node);
290 :
291 0 : if (sector < blk_rq_pos(rq))
292 0 : n = n->rb_left;
293 0 : else if (sector > blk_rq_pos(rq))
294 0 : n = n->rb_right;
295 : else
296 0 : return rq;
297 : }
298 :
299 : return NULL;
300 : }
301 : EXPORT_SYMBOL(elv_rb_find);
302 :
303 0 : enum elv_merge elv_merge(struct request_queue *q, struct request **req,
304 : struct bio *bio)
305 : {
306 0 : struct elevator_queue *e = q->elevator;
307 0 : struct request *__rq;
308 :
309 : /*
310 : * Levels of merges:
311 : * nomerges: No merges at all attempted
312 : * noxmerges: Only simple one-hit cache try
313 : * merges: All merge tries attempted
314 : */
315 0 : if (blk_queue_nomerges(q) || !bio_mergeable(bio))
316 : return ELEVATOR_NO_MERGE;
317 :
318 : /*
319 : * First try one-hit cache.
320 : */
321 0 : if (q->last_merge && elv_bio_merge_ok(q->last_merge, bio)) {
322 0 : enum elv_merge ret = blk_try_merge(q->last_merge, bio);
323 :
324 0 : if (ret != ELEVATOR_NO_MERGE) {
325 0 : *req = q->last_merge;
326 0 : return ret;
327 : }
328 : }
329 :
330 0 : if (blk_queue_noxmerges(q))
331 : return ELEVATOR_NO_MERGE;
332 :
333 : /*
334 : * See if our hash lookup can find a potential backmerge.
335 : */
336 0 : __rq = elv_rqhash_find(q, bio->bi_iter.bi_sector);
337 0 : if (__rq && elv_bio_merge_ok(__rq, bio)) {
338 0 : *req = __rq;
339 0 : return ELEVATOR_BACK_MERGE;
340 : }
341 :
342 0 : if (e->type->ops.request_merge)
343 0 : return e->type->ops.request_merge(q, req, bio);
344 :
345 : return ELEVATOR_NO_MERGE;
346 : }
347 :
348 : /*
349 : * Attempt to do an insertion back merge. Only check for the case where
350 : * we can append 'rq' to an existing request, so we can throw 'rq' away
351 : * afterwards.
352 : *
353 : * Returns true if we merged, false otherwise
354 : */
355 0 : bool elv_attempt_insert_merge(struct request_queue *q, struct request *rq)
356 : {
357 0 : struct request *__rq;
358 0 : bool ret;
359 :
360 0 : if (blk_queue_nomerges(q))
361 : return false;
362 :
363 : /*
364 : * First try one-hit cache.
365 : */
366 0 : if (q->last_merge && blk_attempt_req_merge(q, q->last_merge, rq))
367 : return true;
368 :
369 0 : if (blk_queue_noxmerges(q))
370 : return false;
371 :
372 : ret = false;
373 : /*
374 : * See if our hash lookup can find a potential backmerge.
375 : */
376 0 : while (1) {
377 0 : __rq = elv_rqhash_find(q, blk_rq_pos(rq));
378 0 : if (!__rq || !blk_attempt_req_merge(q, __rq, rq))
379 : break;
380 :
381 : /* The merged request could be merged with others, try again */
382 : ret = true;
383 : rq = __rq;
384 : }
385 :
386 : return ret;
387 : }
388 :
389 0 : void elv_merged_request(struct request_queue *q, struct request *rq,
390 : enum elv_merge type)
391 : {
392 0 : struct elevator_queue *e = q->elevator;
393 :
394 0 : if (e->type->ops.request_merged)
395 0 : e->type->ops.request_merged(q, rq, type);
396 :
397 0 : if (type == ELEVATOR_BACK_MERGE)
398 0 : elv_rqhash_reposition(q, rq);
399 :
400 0 : q->last_merge = rq;
401 0 : }
402 :
403 0 : void elv_merge_requests(struct request_queue *q, struct request *rq,
404 : struct request *next)
405 : {
406 0 : struct elevator_queue *e = q->elevator;
407 :
408 0 : if (e->type->ops.requests_merged)
409 0 : e->type->ops.requests_merged(q, rq, next);
410 :
411 0 : elv_rqhash_reposition(q, rq);
412 0 : q->last_merge = rq;
413 0 : }
414 :
415 0 : struct request *elv_latter_request(struct request_queue *q, struct request *rq)
416 : {
417 0 : struct elevator_queue *e = q->elevator;
418 :
419 0 : if (e->type->ops.next_request)
420 0 : return e->type->ops.next_request(q, rq);
421 :
422 : return NULL;
423 : }
424 :
425 0 : struct request *elv_former_request(struct request_queue *q, struct request *rq)
426 : {
427 0 : struct elevator_queue *e = q->elevator;
428 :
429 0 : if (e->type->ops.former_request)
430 0 : return e->type->ops.former_request(q, rq);
431 :
432 : return NULL;
433 : }
434 :
435 : #define to_elv(atr) container_of((atr), struct elv_fs_entry, attr)
436 :
437 : static ssize_t
438 0 : elv_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
439 : {
440 0 : struct elv_fs_entry *entry = to_elv(attr);
441 0 : struct elevator_queue *e;
442 0 : ssize_t error;
443 :
444 0 : if (!entry->show)
445 : return -EIO;
446 :
447 0 : e = container_of(kobj, struct elevator_queue, kobj);
448 0 : mutex_lock(&e->sysfs_lock);
449 0 : error = e->type ? entry->show(e, page) : -ENOENT;
450 0 : mutex_unlock(&e->sysfs_lock);
451 0 : return error;
452 : }
453 :
454 : static ssize_t
455 0 : elv_attr_store(struct kobject *kobj, struct attribute *attr,
456 : const char *page, size_t length)
457 : {
458 0 : struct elv_fs_entry *entry = to_elv(attr);
459 0 : struct elevator_queue *e;
460 0 : ssize_t error;
461 :
462 0 : if (!entry->store)
463 : return -EIO;
464 :
465 0 : e = container_of(kobj, struct elevator_queue, kobj);
466 0 : mutex_lock(&e->sysfs_lock);
467 0 : error = e->type ? entry->store(e, page, length) : -ENOENT;
468 0 : mutex_unlock(&e->sysfs_lock);
469 0 : return error;
470 : }
471 :
472 : static const struct sysfs_ops elv_sysfs_ops = {
473 : .show = elv_attr_show,
474 : .store = elv_attr_store,
475 : };
476 :
477 : static struct kobj_type elv_ktype = {
478 : .sysfs_ops = &elv_sysfs_ops,
479 : .release = elevator_release,
480 : };
481 :
482 0 : int elv_register_queue(struct request_queue *q, bool uevent)
483 : {
484 0 : struct elevator_queue *e = q->elevator;
485 0 : int error;
486 :
487 0 : lockdep_assert_held(&q->sysfs_lock);
488 :
489 0 : error = kobject_add(&e->kobj, &q->kobj, "%s", "iosched");
490 0 : if (!error) {
491 0 : struct elv_fs_entry *attr = e->type->elevator_attrs;
492 0 : if (attr) {
493 0 : while (attr->attr.name) {
494 0 : if (sysfs_create_file(&e->kobj, &attr->attr))
495 : break;
496 0 : attr++;
497 : }
498 : }
499 0 : if (uevent)
500 0 : kobject_uevent(&e->kobj, KOBJ_ADD);
501 :
502 0 : e->registered = 1;
503 : }
504 0 : return error;
505 : }
506 :
507 0 : void elv_unregister_queue(struct request_queue *q)
508 : {
509 0 : lockdep_assert_held(&q->sysfs_lock);
510 :
511 0 : if (q) {
512 0 : struct elevator_queue *e = q->elevator;
513 :
514 0 : kobject_uevent(&e->kobj, KOBJ_REMOVE);
515 0 : kobject_del(&e->kobj);
516 :
517 0 : e->registered = 0;
518 : /* Re-enable throttling in case elevator disabled it */
519 0 : wbt_enable_default(q);
520 : }
521 0 : }
522 :
523 0 : int elv_register(struct elevator_type *e)
524 : {
525 : /* create icq_cache if requested */
526 0 : if (e->icq_size) {
527 0 : if (WARN_ON(e->icq_size < sizeof(struct io_cq)) ||
528 0 : WARN_ON(e->icq_align < __alignof__(struct io_cq)))
529 : return -EINVAL;
530 :
531 0 : snprintf(e->icq_cache_name, sizeof(e->icq_cache_name),
532 : "%s_io_cq", e->elevator_name);
533 0 : e->icq_cache = kmem_cache_create(e->icq_cache_name, e->icq_size,
534 0 : e->icq_align, 0, NULL);
535 0 : if (!e->icq_cache)
536 : return -ENOMEM;
537 : }
538 :
539 : /* register, don't allow duplicate names */
540 0 : spin_lock(&elv_list_lock);
541 0 : if (elevator_find(e->elevator_name, 0)) {
542 0 : spin_unlock(&elv_list_lock);
543 0 : kmem_cache_destroy(e->icq_cache);
544 0 : return -EBUSY;
545 : }
546 0 : list_add_tail(&e->list, &elv_list);
547 0 : spin_unlock(&elv_list_lock);
548 :
549 0 : printk(KERN_INFO "io scheduler %s registered\n", e->elevator_name);
550 :
551 0 : return 0;
552 : }
553 : EXPORT_SYMBOL_GPL(elv_register);
554 :
555 0 : void elv_unregister(struct elevator_type *e)
556 : {
557 : /* unregister */
558 0 : spin_lock(&elv_list_lock);
559 0 : list_del_init(&e->list);
560 0 : spin_unlock(&elv_list_lock);
561 :
562 : /*
563 : * Destroy icq_cache if it exists. icq's are RCU managed. Make
564 : * sure all RCU operations are complete before proceeding.
565 : */
566 0 : if (e->icq_cache) {
567 0 : rcu_barrier();
568 0 : kmem_cache_destroy(e->icq_cache);
569 0 : e->icq_cache = NULL;
570 : }
571 0 : }
572 : EXPORT_SYMBOL_GPL(elv_unregister);
573 :
574 0 : int elevator_switch_mq(struct request_queue *q,
575 : struct elevator_type *new_e)
576 : {
577 0 : int ret;
578 :
579 0 : lockdep_assert_held(&q->sysfs_lock);
580 :
581 0 : if (q->elevator) {
582 0 : if (q->elevator->registered)
583 0 : elv_unregister_queue(q);
584 :
585 0 : ioc_clear_queue(q);
586 0 : elevator_exit(q, q->elevator);
587 : }
588 :
589 0 : ret = blk_mq_init_sched(q, new_e);
590 0 : if (ret)
591 0 : goto out;
592 :
593 0 : if (new_e) {
594 0 : ret = elv_register_queue(q, true);
595 0 : if (ret) {
596 0 : elevator_exit(q, q->elevator);
597 0 : goto out;
598 : }
599 : }
600 :
601 : if (new_e)
602 : blk_add_trace_msg(q, "elv switch: %s", new_e->elevator_name);
603 : else
604 0 : blk_add_trace_msg(q, "elv switch: none");
605 :
606 0 : out:
607 0 : return ret;
608 : }
609 :
610 9 : static inline bool elv_support_iosched(struct request_queue *q)
611 : {
612 18 : if (!queue_is_mq(q) ||
613 9 : (q->tag_set && (q->tag_set->flags & BLK_MQ_F_NO_SCHED)))
614 0 : return false;
615 : return true;
616 : }
617 :
618 : /*
619 : * For single queue devices, default to using mq-deadline. If we have multiple
620 : * queues or mq-deadline is not available, default to "none".
621 : */
622 9 : static struct elevator_type *elevator_get_default(struct request_queue *q)
623 : {
624 9 : if (q->nr_hw_queues != 1)
625 : return NULL;
626 :
627 9 : return elevator_get(q, "mq-deadline", false);
628 : }
629 :
630 : /*
631 : * Get the first elevator providing the features required by the request queue.
632 : * Default to "none" if no matching elevator is found.
633 : */
634 0 : static struct elevator_type *elevator_get_by_features(struct request_queue *q)
635 : {
636 0 : struct elevator_type *e, *found = NULL;
637 :
638 0 : spin_lock(&elv_list_lock);
639 :
640 0 : list_for_each_entry(e, &elv_list, list) {
641 0 : if (elv_support_features(e->elevator_features,
642 : q->required_elevator_features)) {
643 : found = e;
644 : break;
645 : }
646 : }
647 :
648 0 : if (found && !try_module_get(found->elevator_owner))
649 : found = NULL;
650 :
651 0 : spin_unlock(&elv_list_lock);
652 0 : return found;
653 : }
654 :
655 : /*
656 : * For a device queue that has no required features, use the default elevator
657 : * settings. Otherwise, use the first elevator available matching the required
658 : * features. If no suitable elevator is find or if the chosen elevator
659 : * initialization fails, fall back to the "none" elevator (no elevator).
660 : */
661 9 : void elevator_init_mq(struct request_queue *q)
662 : {
663 9 : struct elevator_type *e;
664 9 : int err;
665 :
666 9 : if (!elv_support_iosched(q))
667 : return;
668 :
669 9 : WARN_ON_ONCE(blk_queue_registered(q));
670 :
671 9 : if (unlikely(q->elevator))
672 : return;
673 :
674 9 : if (!q->required_elevator_features)
675 9 : e = elevator_get_default(q);
676 : else
677 0 : e = elevator_get_by_features(q);
678 9 : if (!e)
679 : return;
680 :
681 0 : blk_mq_freeze_queue(q);
682 0 : blk_mq_quiesce_queue(q);
683 :
684 0 : err = blk_mq_init_sched(q, e);
685 :
686 0 : blk_mq_unquiesce_queue(q);
687 0 : blk_mq_unfreeze_queue(q);
688 :
689 0 : if (err) {
690 0 : pr_warn("\"%s\" elevator initialization failed, "
691 : "falling back to \"none\"\n", e->elevator_name);
692 0 : elevator_put(e);
693 : }
694 : }
695 :
696 :
697 : /*
698 : * switch to new_e io scheduler. be careful not to introduce deadlocks -
699 : * we don't free the old io scheduler, before we have allocated what we
700 : * need for the new one. this way we have a chance of going back to the old
701 : * one, if the new one fails init for some reason.
702 : */
703 0 : static int elevator_switch(struct request_queue *q, struct elevator_type *new_e)
704 : {
705 0 : int err;
706 :
707 0 : lockdep_assert_held(&q->sysfs_lock);
708 :
709 0 : blk_mq_freeze_queue(q);
710 0 : blk_mq_quiesce_queue(q);
711 :
712 0 : err = elevator_switch_mq(q, new_e);
713 :
714 0 : blk_mq_unquiesce_queue(q);
715 0 : blk_mq_unfreeze_queue(q);
716 :
717 0 : return err;
718 : }
719 :
720 : /*
721 : * Switch this queue to the given IO scheduler.
722 : */
723 0 : static int __elevator_change(struct request_queue *q, const char *name)
724 : {
725 0 : char elevator_name[ELV_NAME_MAX];
726 0 : struct elevator_type *e;
727 :
728 : /* Make sure queue is not in the middle of being removed */
729 0 : if (!blk_queue_registered(q))
730 : return -ENOENT;
731 :
732 : /*
733 : * Special case for mq, turn off scheduling
734 : */
735 0 : if (!strncmp(name, "none", 4)) {
736 0 : if (!q->elevator)
737 : return 0;
738 0 : return elevator_switch(q, NULL);
739 : }
740 :
741 0 : strlcpy(elevator_name, name, sizeof(elevator_name));
742 0 : e = elevator_get(q, strstrip(elevator_name), true);
743 0 : if (!e)
744 : return -EINVAL;
745 :
746 0 : if (q->elevator &&
747 0 : elevator_match(q->elevator->type, elevator_name, 0)) {
748 0 : elevator_put(e);
749 : return 0;
750 : }
751 :
752 0 : return elevator_switch(q, e);
753 : }
754 :
755 0 : ssize_t elv_iosched_store(struct request_queue *q, const char *name,
756 : size_t count)
757 : {
758 0 : int ret;
759 :
760 0 : if (!elv_support_iosched(q))
761 0 : return count;
762 :
763 0 : ret = __elevator_change(q, name);
764 0 : if (!ret)
765 0 : return count;
766 :
767 0 : return ret;
768 : }
769 :
770 0 : ssize_t elv_iosched_show(struct request_queue *q, char *name)
771 : {
772 0 : struct elevator_queue *e = q->elevator;
773 0 : struct elevator_type *elv = NULL;
774 0 : struct elevator_type *__e;
775 0 : int len = 0;
776 :
777 0 : if (!queue_is_mq(q))
778 0 : return sprintf(name, "none\n");
779 :
780 0 : if (!q->elevator)
781 0 : len += sprintf(name+len, "[none] ");
782 : else
783 0 : elv = e->type;
784 :
785 0 : spin_lock(&elv_list_lock);
786 0 : list_for_each_entry(__e, &elv_list, list) {
787 0 : if (elv && elevator_match(elv, __e->elevator_name, 0)) {
788 0 : len += sprintf(name+len, "[%s] ", elv->elevator_name);
789 0 : continue;
790 : }
791 0 : if (elv_support_iosched(q) &&
792 0 : elevator_match(__e, __e->elevator_name,
793 : q->required_elevator_features))
794 0 : len += sprintf(name+len, "%s ", __e->elevator_name);
795 : }
796 0 : spin_unlock(&elv_list_lock);
797 :
798 0 : if (q->elevator)
799 0 : len += sprintf(name+len, "none");
800 :
801 0 : len += sprintf(len+name, "\n");
802 0 : return len;
803 : }
804 :
805 0 : struct request *elv_rb_former_request(struct request_queue *q,
806 : struct request *rq)
807 : {
808 0 : struct rb_node *rbprev = rb_prev(&rq->rb_node);
809 :
810 0 : if (rbprev)
811 0 : return rb_entry_rq(rbprev);
812 :
813 : return NULL;
814 : }
815 : EXPORT_SYMBOL(elv_rb_former_request);
816 :
817 0 : struct request *elv_rb_latter_request(struct request_queue *q,
818 : struct request *rq)
819 : {
820 0 : struct rb_node *rbnext = rb_next(&rq->rb_node);
821 :
822 0 : if (rbnext)
823 0 : return rb_entry_rq(rbnext);
824 :
825 : return NULL;
826 : }
827 : EXPORT_SYMBOL(elv_rb_latter_request);
828 :
829 0 : static int __init elevator_setup(char *str)
830 : {
831 0 : pr_warn("Kernel parameter elevator= does not have any effect anymore.\n"
832 : "Please use sysfs to set IO scheduler for individual devices.\n");
833 0 : return 1;
834 : }
835 :
836 : __setup("elevator=", elevator_setup);
|
