Line data    Source code

       1             : // SPDX-License-Identifier: GPL-2.0
       2             : /*
       3             :  * Functions related to sysfs handling
       4             :  */
       5             : #include <linux/kernel.h>
       6             : #include <linux/slab.h>
       7             : #include <linux/module.h>
       8             : #include <linux/bio.h>
       9             : #include <linux/blkdev.h>
      10             : #include <linux/backing-dev.h>
      11             : #include <linux/blktrace_api.h>
      12             : #include <linux/blk-mq.h>
      13             : #include <linux/blk-cgroup.h>
      14             : #include <linux/debugfs.h>
      15             : 
      16             : #include "blk.h"
      17             : #include "blk-mq.h"
      18             : #include "blk-mq-debugfs.h"
      19             : #include "blk-wbt.h"
      20             : 
      21             : struct queue_sysfs_entry {
      22             :         struct attribute attr;
      23             :         ssize_t (*show)(struct request_queue *, char *);
      24             :         ssize_t (*store)(struct request_queue *, const char *, size_t);
      25             : };
      26             : 
      27             : static ssize_t
      28           1 : queue_var_show(unsigned long var, char *page)
      29             : {
      30           1 :         return sprintf(page, "%lu\n", var);
      31             : }
      32             : 
      33             : static ssize_t
      34           0 : queue_var_store(unsigned long *var, const char *page, size_t count)
      35             : {
      36           0 :         int err;
      37           0 :         unsigned long v;
      38             : 
      39           0 :         err = kstrtoul(page, 10, &v);
      40           0 :         if (err || v > UINT_MAX)
      41             :                 return -EINVAL;
      42             : 
      43           0 :         *var = v;
      44             : 
      45           0 :         return count;
      46             : }
      47             : 
      48           0 : static ssize_t queue_var_store64(s64 *var, const char *page)
      49             : {
      50           0 :         int err;
      51           0 :         s64 v;
      52             : 
      53           0 :         err = kstrtos64(page, 10, &v);
      54           0 :         if (err < 0)
      55           0 :                 return err;
      56             : 
      57           0 :         *var = v;
      58           0 :         return 0;
      59             : }
      60             : 
      61           0 : static ssize_t queue_requests_show(struct request_queue *q, char *page)
      62             : {
      63           0 :         return queue_var_show(q->nr_requests, (page));
      64             : }
      65             : 
      66             : static ssize_t
      67           0 : queue_requests_store(struct request_queue *q, const char *page, size_t count)
      68             : {
      69           0 :         unsigned long nr;
      70           0 :         int ret, err;
      71             : 
      72           0 :         if (!queue_is_mq(q))
      73             :                 return -EINVAL;
      74             : 
      75           0 :         ret = queue_var_store(&nr, page, count);
      76           0 :         if (ret < 0)
      77           0 :                 return ret;
      78             : 
      79           0 :         if (nr < BLKDEV_MIN_RQ)
      80           0 :                 nr = BLKDEV_MIN_RQ;
      81             : 
      82           0 :         err = blk_mq_update_nr_requests(q, nr);
      83           0 :         if (err)
      84           0 :                 return err;
      85             : 
      86           0 :         return ret;
      87             : }
      88             : 
      89           0 : static ssize_t queue_ra_show(struct request_queue *q, char *page)
      90             : {
      91           0 :         unsigned long ra_kb = q->backing_dev_info->ra_pages <<
      92             :                                         (PAGE_SHIFT - 10);
      93             : 
      94           0 :         return queue_var_show(ra_kb, (page));
      95             : }
      96             : 
      97             : static ssize_t
      98           0 : queue_ra_store(struct request_queue *q, const char *page, size_t count)
      99             : {
     100           0 :         unsigned long ra_kb;
     101           0 :         ssize_t ret = queue_var_store(&ra_kb, page, count);
     102             : 
     103           0 :         if (ret < 0)
     104             :                 return ret;
     105             : 
     106           0 :         q->backing_dev_info->ra_pages = ra_kb >> (PAGE_SHIFT - 10);
     107             : 
     108           0 :         return ret;
     109             : }
     110             : 
     111           0 : static ssize_t queue_max_sectors_show(struct request_queue *q, char *page)
     112             : {
     113           0 :         int max_sectors_kb = queue_max_sectors(q) >> 1;
     114             : 
     115           0 :         return queue_var_show(max_sectors_kb, (page));
     116             : }
     117             : 
     118           0 : static ssize_t queue_max_segments_show(struct request_queue *q, char *page)
     119             : {
     120           0 :         return queue_var_show(queue_max_segments(q), (page));
     121             : }
     122             : 
     123           0 : static ssize_t queue_max_discard_segments_show(struct request_queue *q,
     124             :                 char *page)
     125             : {
     126           0 :         return queue_var_show(queue_max_discard_segments(q), (page));
     127             : }
     128             : 
     129           0 : static ssize_t queue_max_integrity_segments_show(struct request_queue *q, char *page)
     130             : {
     131           0 :         return queue_var_show(q->limits.max_integrity_segments, (page));
     132             : }
     133             : 
     134           0 : static ssize_t queue_max_segment_size_show(struct request_queue *q, char *page)
     135             : {
     136           0 :         return queue_var_show(queue_max_segment_size(q), (page));
     137             : }
     138             : 
     139           0 : static ssize_t queue_logical_block_size_show(struct request_queue *q, char *page)
     140             : {
     141           0 :         return queue_var_show(queue_logical_block_size(q), page);
     142             : }
     143             : 
     144           0 : static ssize_t queue_physical_block_size_show(struct request_queue *q, char *page)
     145             : {
     146           0 :         return queue_var_show(queue_physical_block_size(q), page);
     147             : }
     148             : 
     149           0 : static ssize_t queue_chunk_sectors_show(struct request_queue *q, char *page)
     150             : {
     151           0 :         return queue_var_show(q->limits.chunk_sectors, page);
     152             : }
     153             : 
     154           0 : static ssize_t queue_io_min_show(struct request_queue *q, char *page)
     155             : {
     156           0 :         return queue_var_show(queue_io_min(q), page);
     157             : }
     158             : 
     159           0 : static ssize_t queue_io_opt_show(struct request_queue *q, char *page)
     160             : {
     161           0 :         return queue_var_show(queue_io_opt(q), page);
     162             : }
     163             : 
     164           0 : static ssize_t queue_discard_granularity_show(struct request_queue *q, char *page)
     165             : {
     166           0 :         return queue_var_show(q->limits.discard_granularity, page);
     167             : }
     168             : 
     169           0 : static ssize_t queue_discard_max_hw_show(struct request_queue *q, char *page)
     170             : {
     171             : 
     172           0 :         return sprintf(page, "%llu\n",
     173           0 :                 (unsigned long long)q->limits.max_hw_discard_sectors << 9);
     174             : }
     175             : 
     176           0 : static ssize_t queue_discard_max_show(struct request_queue *q, char *page)
     177             : {
     178           0 :         return sprintf(page, "%llu\n",
     179           0 :                        (unsigned long long)q->limits.max_discard_sectors << 9);
     180             : }
     181             : 
     182           0 : static ssize_t queue_discard_max_store(struct request_queue *q,
     183             :                                        const char *page, size_t count)
     184             : {
     185           0 :         unsigned long max_discard;
     186           0 :         ssize_t ret = queue_var_store(&max_discard, page, count);
     187             : 
     188           0 :         if (ret < 0)
     189             :                 return ret;
     190             : 
     191           0 :         if (max_discard & (q->limits.discard_granularity - 1))
     192             :                 return -EINVAL;
     193             : 
     194           0 :         max_discard >>= 9;
     195           0 :         if (max_discard > UINT_MAX)
     196             :                 return -EINVAL;
     197             : 
     198           0 :         if (max_discard > q->limits.max_hw_discard_sectors)
     199           0 :                 max_discard = q->limits.max_hw_discard_sectors;
     200             : 
     201           0 :         q->limits.max_discard_sectors = max_discard;
     202           0 :         return ret;
     203             : }
     204             : 
     205           0 : static ssize_t queue_discard_zeroes_data_show(struct request_queue *q, char *page)
     206             : {
     207           0 :         return queue_var_show(0, page);
     208             : }
     209             : 
     210           0 : static ssize_t queue_write_same_max_show(struct request_queue *q, char *page)
     211             : {
     212           0 :         return sprintf(page, "%llu\n",
     213           0 :                 (unsigned long long)q->limits.max_write_same_sectors << 9);
     214             : }
     215             : 
     216           0 : static ssize_t queue_write_zeroes_max_show(struct request_queue *q, char *page)
     217             : {
     218           0 :         return sprintf(page, "%llu\n",
     219           0 :                 (unsigned long long)q->limits.max_write_zeroes_sectors << 9);
     220             : }
     221             : 
     222           0 : static ssize_t queue_zone_write_granularity_show(struct request_queue *q,
     223             :                                                  char *page)
     224             : {
     225           0 :         return queue_var_show(queue_zone_write_granularity(q), page);
     226             : }
     227             : 
     228           0 : static ssize_t queue_zone_append_max_show(struct request_queue *q, char *page)
     229             : {
     230           0 :         unsigned long long max_sectors = q->limits.max_zone_append_sectors;
     231             : 
     232           0 :         return sprintf(page, "%llu\n", max_sectors << SECTOR_SHIFT);
     233             : }
     234             : 
     235             : static ssize_t
     236           0 : queue_max_sectors_store(struct request_queue *q, const char *page, size_t count)
     237             : {
     238           0 :         unsigned long max_sectors_kb,
     239           0 :                 max_hw_sectors_kb = queue_max_hw_sectors(q) >> 1,
     240           0 :                         page_kb = 1 << (PAGE_SHIFT - 10);
     241           0 :         ssize_t ret = queue_var_store(&max_sectors_kb, page, count);
     242             : 
     243           0 :         if (ret < 0)
     244             :                 return ret;
     245             : 
     246           0 :         max_hw_sectors_kb = min_not_zero(max_hw_sectors_kb, (unsigned long)
     247             :                                          q->limits.max_dev_sectors >> 1);
     248             : 
     249           0 :         if (max_sectors_kb > max_hw_sectors_kb || max_sectors_kb < page_kb)
     250             :                 return -EINVAL;
     251             : 
     252           0 :         spin_lock_irq(&q->queue_lock);
     253           0 :         q->limits.max_sectors = max_sectors_kb << 1;
     254           0 :         q->backing_dev_info->io_pages = max_sectors_kb >> (PAGE_SHIFT - 10);
     255           0 :         spin_unlock_irq(&q->queue_lock);
     256             : 
     257           0 :         return ret;
     258             : }
     259             : 
     260           0 : static ssize_t queue_max_hw_sectors_show(struct request_queue *q, char *page)
     261             : {
     262           0 :         int max_hw_sectors_kb = queue_max_hw_sectors(q) >> 1;
     263             : 
     264           0 :         return queue_var_show(max_hw_sectors_kb, (page));
     265             : }
     266             : 
     267             : #define QUEUE_SYSFS_BIT_FNS(name, flag, neg)                            \
     268             : static ssize_t                                                          \
     269             : queue_##name##_show(struct request_queue *q, char *page)                \
     270             : {                                                                       \
     271             :         int bit;                                                        \
     272             :         bit = test_bit(QUEUE_FLAG_##flag, &q->queue_flags);              \
     273             :         return queue_var_show(neg ? !bit : bit, page);                  \
     274             : }                                                                       \
     275             : static ssize_t                                                          \
     276             : queue_##name##_store(struct request_queue *q, const char *page, size_t count) \
     277             : {                                                                       \
     278             :         unsigned long val;                                              \
     279             :         ssize_t ret;                                                    \
     280             :         ret = queue_var_store(&val, page, count);                   \
     281             :         if (ret < 0)                                                 \
     282             :                  return ret;                                            \
     283             :         if (neg)                                                        \
     284             :                 val = !val;                                             \
     285             :                                                                         \
     286             :         if (val)                                                        \
     287             :                 blk_queue_flag_set(QUEUE_FLAG_##flag, q);               \
     288             :         else                                                            \
     289             :                 blk_queue_flag_clear(QUEUE_FLAG_##flag, q);             \
     290             :         return ret;                                                     \
     291             : }
     292             : 
     293           1 : QUEUE_SYSFS_BIT_FNS(nonrot, NONROT, 1);
     294           0 : QUEUE_SYSFS_BIT_FNS(random, ADD_RANDOM, 0);
     295           0 : QUEUE_SYSFS_BIT_FNS(iostats, IO_STAT, 0);
     296           0 : QUEUE_SYSFS_BIT_FNS(stable_writes, STABLE_WRITES, 0);
     297             : #undef QUEUE_SYSFS_BIT_FNS
     298             : 
     299           0 : static ssize_t queue_zoned_show(struct request_queue *q, char *page)
     300             : {
     301           0 :         switch (blk_queue_zoned_model(q)) {
     302             :         case BLK_ZONED_HA:
     303             :                 return sprintf(page, "host-aware\n");
     304             :         case BLK_ZONED_HM:
     305             :                 return sprintf(page, "host-managed\n");
     306             :         default:
     307           0 :                 return sprintf(page, "none\n");
     308             :         }
     309             : }
     310             : 
     311           0 : static ssize_t queue_nr_zones_show(struct request_queue *q, char *page)
     312             : {
     313           0 :         return queue_var_show(blk_queue_nr_zones(q), page);
     314             : }
     315             : 
     316           0 : static ssize_t queue_max_open_zones_show(struct request_queue *q, char *page)
     317             : {
     318           0 :         return queue_var_show(queue_max_open_zones(q), page);
     319             : }
     320             : 
     321           0 : static ssize_t queue_max_active_zones_show(struct request_queue *q, char *page)
     322             : {
     323           0 :         return queue_var_show(queue_max_active_zones(q), page);
     324             : }
     325             : 
     326           0 : static ssize_t queue_nomerges_show(struct request_queue *q, char *page)
     327             : {
     328           0 :         return queue_var_show((blk_queue_nomerges(q) << 1) |
     329           0 :                                blk_queue_noxmerges(q), page);
     330             : }
     331             : 
     332           0 : static ssize_t queue_nomerges_store(struct request_queue *q, const char *page,
     333             :                                     size_t count)
     334             : {
     335           0 :         unsigned long nm;
     336           0 :         ssize_t ret = queue_var_store(&nm, page, count);
     337             : 
     338           0 :         if (ret < 0)
     339             :                 return ret;
     340             : 
     341           0 :         blk_queue_flag_clear(QUEUE_FLAG_NOMERGES, q);
     342           0 :         blk_queue_flag_clear(QUEUE_FLAG_NOXMERGES, q);
     343           0 :         if (nm == 2)
     344           0 :                 blk_queue_flag_set(QUEUE_FLAG_NOMERGES, q);
     345           0 :         else if (nm)
     346           0 :                 blk_queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
     347             : 
     348             :         return ret;
     349             : }
     350             : 
     351           0 : static ssize_t queue_rq_affinity_show(struct request_queue *q, char *page)
     352             : {
     353           0 :         bool set = test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags);
     354           0 :         bool force = test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags);
     355             : 
     356           0 :         return queue_var_show(set << force, page);
     357             : }
     358             : 
     359             : static ssize_t
     360           0 : queue_rq_affinity_store(struct request_queue *q, const char *page, size_t count)
     361             : {
     362           0 :         ssize_t ret = -EINVAL;
     363             : #ifdef CONFIG_SMP
     364           0 :         unsigned long val;
     365             : 
     366           0 :         ret = queue_var_store(&val, page, count);
     367           0 :         if (ret < 0)
     368             :                 return ret;
     369             : 
     370           0 :         if (val == 2) {
     371           0 :                 blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
     372           0 :                 blk_queue_flag_set(QUEUE_FLAG_SAME_FORCE, q);
     373           0 :         } else if (val == 1) {
     374           0 :                 blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
     375           0 :                 blk_queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
     376           0 :         } else if (val == 0) {
     377           0 :                 blk_queue_flag_clear(QUEUE_FLAG_SAME_COMP, q);
     378           0 :                 blk_queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
     379             :         }
     380             : #endif
     381             :         return ret;
     382             : }
     383             : 
     384           0 : static ssize_t queue_poll_delay_show(struct request_queue *q, char *page)
     385             : {
     386           0 :         int val;
     387             : 
     388           0 :         if (q->poll_nsec == BLK_MQ_POLL_CLASSIC)
     389             :                 val = BLK_MQ_POLL_CLASSIC;
     390             :         else
     391           0 :                 val = q->poll_nsec / 1000;
     392             : 
     393           0 :         return sprintf(page, "%d\n", val);
     394             : }
     395             : 
     396           0 : static ssize_t queue_poll_delay_store(struct request_queue *q, const char *page,
     397             :                                 size_t count)
     398             : {
     399           0 :         int err, val;
     400             : 
     401           0 :         if (!q->mq_ops || !q->mq_ops->poll)
     402             :                 return -EINVAL;
     403             : 
     404           0 :         err = kstrtoint(page, 10, &val);
     405           0 :         if (err < 0)
     406           0 :                 return err;
     407             : 
     408           0 :         if (val == BLK_MQ_POLL_CLASSIC)
     409           0 :                 q->poll_nsec = BLK_MQ_POLL_CLASSIC;
     410           0 :         else if (val >= 0)
     411           0 :                 q->poll_nsec = val * 1000;
     412             :         else
     413             :                 return -EINVAL;
     414             : 
     415           0 :         return count;
     416             : }
     417             : 
     418           0 : static ssize_t queue_poll_show(struct request_queue *q, char *page)
     419             : {
     420           0 :         return queue_var_show(test_bit(QUEUE_FLAG_POLL, &q->queue_flags), page);
     421             : }
     422             : 
     423           0 : static ssize_t queue_poll_store(struct request_queue *q, const char *page,
     424             :                                 size_t count)
     425             : {
     426           0 :         unsigned long poll_on;
     427           0 :         ssize_t ret;
     428             : 
     429           0 :         if (!q->tag_set || q->tag_set->nr_maps <= HCTX_TYPE_POLL ||
     430           0 :             !q->tag_set->map[HCTX_TYPE_POLL].nr_queues)
     431             :                 return -EINVAL;
     432             : 
     433           0 :         ret = queue_var_store(&poll_on, page, count);
     434           0 :         if (ret < 0)
     435             :                 return ret;
     436             : 
     437           0 :         if (poll_on) {
     438           0 :                 blk_queue_flag_set(QUEUE_FLAG_POLL, q);
     439             :         } else {
     440           0 :                 blk_mq_freeze_queue(q);
     441           0 :                 blk_queue_flag_clear(QUEUE_FLAG_POLL, q);
     442           0 :                 blk_mq_unfreeze_queue(q);
     443             :         }
     444             : 
     445             :         return ret;
     446             : }
     447             : 
     448           0 : static ssize_t queue_io_timeout_show(struct request_queue *q, char *page)
     449             : {
     450           0 :         return sprintf(page, "%u\n", jiffies_to_msecs(q->rq_timeout));
     451             : }
     452             : 
     453           0 : static ssize_t queue_io_timeout_store(struct request_queue *q, const char *page,
     454             :                                   size_t count)
     455             : {
     456           0 :         unsigned int val;
     457           0 :         int err;
     458             : 
     459           0 :         err = kstrtou32(page, 10, &val);
     460           0 :         if (err || val == 0)
     461             :                 return -EINVAL;
     462             : 
     463           0 :         blk_queue_rq_timeout(q, msecs_to_jiffies(val));
     464             : 
     465           0 :         return count;
     466             : }
     467             : 
     468           0 : static ssize_t queue_wb_lat_show(struct request_queue *q, char *page)
     469             : {
     470           0 :         if (!wbt_rq_qos(q))
     471             :                 return -EINVAL;
     472             : 
     473           0 :         return sprintf(page, "%llu\n", div_u64(wbt_get_min_lat(q), 1000));
     474             : }
     475             : 
     476           0 : static ssize_t queue_wb_lat_store(struct request_queue *q, const char *page,
     477             :                                   size_t count)
     478             : {
     479           0 :         struct rq_qos *rqos;
     480           0 :         ssize_t ret;
     481           0 :         s64 val;
     482             : 
     483           0 :         ret = queue_var_store64(&val, page);
     484           0 :         if (ret < 0)
     485             :                 return ret;
     486           0 :         if (val < -1)
     487             :                 return -EINVAL;
     488             : 
     489           0 :         rqos = wbt_rq_qos(q);
     490           0 :         if (!rqos) {
     491           0 :                 ret = wbt_init(q);
     492             :                 if (ret)
     493             :                         return ret;
     494             :         }
     495             : 
     496           0 :         if (val == -1)
     497           0 :                 val = wbt_default_latency_nsec(q);
     498           0 :         else if (val >= 0)
     499           0 :                 val *= 1000ULL;
     500             : 
     501           0 :         if (wbt_get_min_lat(q) == val)
     502           0 :                 return count;
     503             : 
     504             :         /*
     505             :          * Ensure that the queue is idled, in case the latency update
     506             :          * ends up either enabling or disabling wbt completely. We can't
     507             :          * have IO inflight if that happens.
     508             :          */
     509           0 :         blk_mq_freeze_queue(q);
     510           0 :         blk_mq_quiesce_queue(q);
     511             : 
     512           0 :         wbt_set_min_lat(q, val);
     513             : 
     514           0 :         blk_mq_unquiesce_queue(q);
     515           0 :         blk_mq_unfreeze_queue(q);
     516             : 
     517           0 :         return count;
     518             : }
     519             : 
     520           0 : static ssize_t queue_wc_show(struct request_queue *q, char *page)
     521             : {
     522           0 :         if (test_bit(QUEUE_FLAG_WC, &q->queue_flags))
     523           0 :                 return sprintf(page, "write back\n");
     524             : 
     525           0 :         return sprintf(page, "write through\n");
     526             : }
     527             : 
     528           0 : static ssize_t queue_wc_store(struct request_queue *q, const char *page,
     529             :                               size_t count)
     530             : {
     531           0 :         int set = -1;
     532             : 
     533           0 :         if (!strncmp(page, "write back", 10))
     534             :                 set = 1;
     535           0 :         else if (!strncmp(page, "write through", 13) ||
     536           0 :                  !strncmp(page, "none", 4))
     537             :                 set = 0;
     538             : 
     539             :         if (set == -1)
     540             :                 return -EINVAL;
     541             : 
     542           0 :         if (set)
     543           0 :                 blk_queue_flag_set(QUEUE_FLAG_WC, q);
     544             :         else
     545           0 :                 blk_queue_flag_clear(QUEUE_FLAG_WC, q);
     546             : 
     547           0 :         return count;
     548             : }
     549             : 
     550           0 : static ssize_t queue_fua_show(struct request_queue *q, char *page)
     551             : {
     552           0 :         return sprintf(page, "%u\n", test_bit(QUEUE_FLAG_FUA, &q->queue_flags));
     553             : }
     554             : 
     555           0 : static ssize_t queue_dax_show(struct request_queue *q, char *page)
     556             : {
     557           0 :         return queue_var_show(blk_queue_dax(q), page);
     558             : }
     559             : 
     560             : #define QUEUE_RO_ENTRY(_prefix, _name)                  \
     561             : static struct queue_sysfs_entry _prefix##_entry = {     \
     562             :         .attr   = { .name = _name, .mode = 0444 },      \
     563             :         .show   = _prefix##_show,                       \
     564             : };
     565             : 
     566             : #define QUEUE_RW_ENTRY(_prefix, _name)                  \
     567             : static struct queue_sysfs_entry _prefix##_entry = {     \
     568             :         .attr   = { .name = _name, .mode = 0644 },      \
     569             :         .show   = _prefix##_show,                       \
     570             :         .store  = _prefix##_store,                      \
     571             : };
     572             : 
     573             : QUEUE_RW_ENTRY(queue_requests, "nr_requests");
     574             : QUEUE_RW_ENTRY(queue_ra, "read_ahead_kb");
     575             : QUEUE_RW_ENTRY(queue_max_sectors, "max_sectors_kb");
     576             : QUEUE_RO_ENTRY(queue_max_hw_sectors, "max_hw_sectors_kb");
     577             : QUEUE_RO_ENTRY(queue_max_segments, "max_segments");
     578             : QUEUE_RO_ENTRY(queue_max_integrity_segments, "max_integrity_segments");
     579             : QUEUE_RO_ENTRY(queue_max_segment_size, "max_segment_size");
     580             : QUEUE_RW_ENTRY(elv_iosched, "scheduler");
     581             : 
     582             : QUEUE_RO_ENTRY(queue_logical_block_size, "logical_block_size");
     583             : QUEUE_RO_ENTRY(queue_physical_block_size, "physical_block_size");
     584             : QUEUE_RO_ENTRY(queue_chunk_sectors, "chunk_sectors");
     585             : QUEUE_RO_ENTRY(queue_io_min, "minimum_io_size");
     586             : QUEUE_RO_ENTRY(queue_io_opt, "optimal_io_size");
     587             : 
     588             : QUEUE_RO_ENTRY(queue_max_discard_segments, "max_discard_segments");
     589             : QUEUE_RO_ENTRY(queue_discard_granularity, "discard_granularity");
     590             : QUEUE_RO_ENTRY(queue_discard_max_hw, "discard_max_hw_bytes");
     591             : QUEUE_RW_ENTRY(queue_discard_max, "discard_max_bytes");
     592             : QUEUE_RO_ENTRY(queue_discard_zeroes_data, "discard_zeroes_data");
     593             : 
     594             : QUEUE_RO_ENTRY(queue_write_same_max, "write_same_max_bytes");
     595             : QUEUE_RO_ENTRY(queue_write_zeroes_max, "write_zeroes_max_bytes");
     596             : QUEUE_RO_ENTRY(queue_zone_append_max, "zone_append_max_bytes");
     597             : QUEUE_RO_ENTRY(queue_zone_write_granularity, "zone_write_granularity");
     598             : 
     599             : QUEUE_RO_ENTRY(queue_zoned, "zoned");
     600             : QUEUE_RO_ENTRY(queue_nr_zones, "nr_zones");
     601             : QUEUE_RO_ENTRY(queue_max_open_zones, "max_open_zones");
     602             : QUEUE_RO_ENTRY(queue_max_active_zones, "max_active_zones");
     603             : 
     604             : QUEUE_RW_ENTRY(queue_nomerges, "nomerges");
     605             : QUEUE_RW_ENTRY(queue_rq_affinity, "rq_affinity");
     606             : QUEUE_RW_ENTRY(queue_poll, "io_poll");
     607             : QUEUE_RW_ENTRY(queue_poll_delay, "io_poll_delay");
     608             : QUEUE_RW_ENTRY(queue_wc, "write_cache");
     609             : QUEUE_RO_ENTRY(queue_fua, "fua");
     610             : QUEUE_RO_ENTRY(queue_dax, "dax");
     611             : QUEUE_RW_ENTRY(queue_io_timeout, "io_timeout");
     612             : QUEUE_RW_ENTRY(queue_wb_lat, "wbt_lat_usec");
     613             : 
     614             : #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
     615             : QUEUE_RW_ENTRY(blk_throtl_sample_time, "throttle_sample_time");
     616             : #endif
     617             : 
     618             : /* legacy alias for logical_block_size: */
     619             : static struct queue_sysfs_entry queue_hw_sector_size_entry = {
     620             :         .attr = {.name = "hw_sector_size", .mode = 0444 },
     621             :         .show = queue_logical_block_size_show,
     622             : };
     623             : 
     624             : QUEUE_RW_ENTRY(queue_nonrot, "rotational");
     625             : QUEUE_RW_ENTRY(queue_iostats, "iostats");
     626             : QUEUE_RW_ENTRY(queue_random, "add_random");
     627             : QUEUE_RW_ENTRY(queue_stable_writes, "stable_writes");
     628             : 
     629             : static struct attribute *queue_attrs[] = {
     630             :         &queue_requests_entry.attr,
     631             :         &queue_ra_entry.attr,
     632             :         &queue_max_hw_sectors_entry.attr,
     633             :         &queue_max_sectors_entry.attr,
     634             :         &queue_max_segments_entry.attr,
     635             :         &queue_max_discard_segments_entry.attr,
     636             :         &queue_max_integrity_segments_entry.attr,
     637             :         &queue_max_segment_size_entry.attr,
     638             :         &elv_iosched_entry.attr,
     639             :         &queue_hw_sector_size_entry.attr,
     640             :         &queue_logical_block_size_entry.attr,
     641             :         &queue_physical_block_size_entry.attr,
     642             :         &queue_chunk_sectors_entry.attr,
     643             :         &queue_io_min_entry.attr,
     644             :         &queue_io_opt_entry.attr,
     645             :         &queue_discard_granularity_entry.attr,
     646             :         &queue_discard_max_entry.attr,
     647             :         &queue_discard_max_hw_entry.attr,
     648             :         &queue_discard_zeroes_data_entry.attr,
     649             :         &queue_write_same_max_entry.attr,
     650             :         &queue_write_zeroes_max_entry.attr,
     651             :         &queue_zone_append_max_entry.attr,
     652             :         &queue_zone_write_granularity_entry.attr,
     653             :         &queue_nonrot_entry.attr,
     654             :         &queue_zoned_entry.attr,
     655             :         &queue_nr_zones_entry.attr,
     656             :         &queue_max_open_zones_entry.attr,
     657             :         &queue_max_active_zones_entry.attr,
     658             :         &queue_nomerges_entry.attr,
     659             :         &queue_rq_affinity_entry.attr,
     660             :         &queue_iostats_entry.attr,
     661             :         &queue_stable_writes_entry.attr,
     662             :         &queue_random_entry.attr,
     663             :         &queue_poll_entry.attr,
     664             :         &queue_wc_entry.attr,
     665             :         &queue_fua_entry.attr,
     666             :         &queue_dax_entry.attr,
     667             :         &queue_wb_lat_entry.attr,
     668             :         &queue_poll_delay_entry.attr,
     669             :         &queue_io_timeout_entry.attr,
     670             : #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
     671             :         &blk_throtl_sample_time_entry.attr,
     672             : #endif
     673             :         NULL,
     674             : };
     675             : 
     676         360 : static umode_t queue_attr_visible(struct kobject *kobj, struct attribute *attr,
     677             :                                 int n)
     678             : {
     679         360 :         struct request_queue *q =
     680         360 :                 container_of(kobj, struct request_queue, kobj);
     681             : 
     682         360 :         if (attr == &queue_io_timeout_entry.attr &&
     683           9 :                 (!q->mq_ops || !q->mq_ops->timeout))
     684             :                         return 0;
     685             : 
     686         351 :         if ((attr == &queue_max_open_zones_entry.attr ||
     687             :              attr == &queue_max_active_zones_entry.attr) &&
     688         360 :             !blk_queue_is_zoned(q))
     689             :                 return 0;
     690             : 
     691         333 :         return attr->mode;
     692             : }
     693             : 
     694             : static struct attribute_group queue_attr_group = {
     695             :         .attrs = queue_attrs,
     696             :         .is_visible = queue_attr_visible,
     697             : };
     698             : 
     699             : 
     700             : #define to_queue(atr) container_of((atr), struct queue_sysfs_entry, attr)
     701             : 
     702             : static ssize_t
     703           1 : queue_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
     704             : {
     705           1 :         struct queue_sysfs_entry *entry = to_queue(attr);
     706           1 :         struct request_queue *q =
     707           1 :                 container_of(kobj, struct request_queue, kobj);
     708           1 :         ssize_t res;
     709             : 
     710           1 :         if (!entry->show)
     711             :                 return -EIO;
     712           1 :         mutex_lock(&q->sysfs_lock);
     713           1 :         res = entry->show(q, page);
     714           1 :         mutex_unlock(&q->sysfs_lock);
     715           1 :         return res;
     716             : }
     717             : 
     718             : static ssize_t
     719           0 : queue_attr_store(struct kobject *kobj, struct attribute *attr,
     720             :                     const char *page, size_t length)
     721             : {
     722           0 :         struct queue_sysfs_entry *entry = to_queue(attr);
     723           0 :         struct request_queue *q;
     724           0 :         ssize_t res;
     725             : 
     726           0 :         if (!entry->store)
     727             :                 return -EIO;
     728             : 
     729           0 :         q = container_of(kobj, struct request_queue, kobj);
     730           0 :         mutex_lock(&q->sysfs_lock);
     731           0 :         res = entry->store(q, page, length);
     732           0 :         mutex_unlock(&q->sysfs_lock);
     733           0 :         return res;
     734             : }
     735             : 
     736           0 : static void blk_free_queue_rcu(struct rcu_head *rcu_head)
     737             : {
     738           0 :         struct request_queue *q = container_of(rcu_head, struct request_queue,
     739             :                                                rcu_head);
     740           0 :         kmem_cache_free(blk_requestq_cachep, q);
     741           0 : }
     742             : 
     743             : /* Unconfigure the I/O scheduler and dissociate from the cgroup controller. */
     744           0 : static void blk_exit_queue(struct request_queue *q)
     745             : {
     746             :         /*
     747             :          * Since the I/O scheduler exit code may access cgroup information,
     748             :          * perform I/O scheduler exit before disassociating from the block
     749             :          * cgroup controller.
     750             :          */
     751           0 :         if (q->elevator) {
     752           0 :                 ioc_clear_queue(q);
     753           0 :                 __elevator_exit(q, q->elevator);
     754             :         }
     755             : 
     756             :         /*
     757             :          * Remove all references to @q from the block cgroup controller before
     758             :          * restoring @q->queue_lock to avoid that restoring this pointer causes
     759             :          * e.g. blkcg_print_blkgs() to crash.
     760             :          */
     761           0 :         blkcg_exit_queue(q);
     762             : 
     763             :         /*
     764             :          * Since the cgroup code may dereference the @q->backing_dev_info
     765             :          * pointer, only decrease its reference count after having removed the
     766             :          * association with the block cgroup controller.
     767             :          */
     768           0 :         bdi_put(q->backing_dev_info);
     769           0 : }
     770             : 
     771             : /**
     772             :  * blk_release_queue - releases all allocated resources of the request_queue
     773             :  * @kobj: pointer to a kobject, whose container is a request_queue
     774             :  *
     775             :  * This function releases all allocated resources of the request queue.
     776             :  *
     777             :  * The struct request_queue refcount is incremented with blk_get_queue() and
     778             :  * decremented with blk_put_queue(). Once the refcount reaches 0 this function
     779             :  * is called.
     780             :  *
     781             :  * For drivers that have a request_queue on a gendisk and added with
     782             :  * __device_add_disk() the refcount to request_queue will reach 0 with
     783             :  * the last put_disk() called by the driver. For drivers which don't use
     784             :  * __device_add_disk() this happens with blk_cleanup_queue().
     785             :  *
     786             :  * Drivers exist which depend on the release of the request_queue to be
     787             :  * synchronous, it should not be deferred.
     788             :  *
     789             :  * Context: can sleep
     790             :  */
     791           0 : static void blk_release_queue(struct kobject *kobj)
     792             : {
     793           0 :         struct request_queue *q =
     794           0 :                 container_of(kobj, struct request_queue, kobj);
     795             : 
     796           0 :         might_sleep();
     797             : 
     798           0 :         if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags))
     799           0 :                 blk_stat_remove_callback(q, q->poll_cb);
     800           0 :         blk_stat_free_callback(q->poll_cb);
     801             : 
     802           0 :         blk_free_queue_stats(q->stats);
     803             : 
     804           0 :         if (queue_is_mq(q)) {
     805           0 :                 struct blk_mq_hw_ctx *hctx;
     806           0 :                 int i;
     807             : 
     808           0 :                 cancel_delayed_work_sync(&q->requeue_work);
     809             : 
     810           0 :                 queue_for_each_hw_ctx(q, hctx, i)
     811           0 :                         cancel_delayed_work_sync(&hctx->run_work);
     812             :         }
     813             : 
     814           0 :         blk_exit_queue(q);
     815             : 
     816           0 :         blk_queue_free_zone_bitmaps(q);
     817             : 
     818           0 :         if (queue_is_mq(q))
     819           0 :                 blk_mq_release(q);
     820             : 
     821           0 :         blk_trace_shutdown(q);
     822           0 :         mutex_lock(&q->debugfs_mutex);
     823           0 :         debugfs_remove_recursive(q->debugfs_dir);
     824           0 :         mutex_unlock(&q->debugfs_mutex);
     825             : 
     826           0 :         if (queue_is_mq(q))
     827           0 :                 blk_mq_debugfs_unregister(q);
     828             : 
     829           0 :         bioset_exit(&q->bio_split);
     830             : 
     831           0 :         ida_simple_remove(&blk_queue_ida, q->id);
     832           0 :         call_rcu(&q->rcu_head, blk_free_queue_rcu);
     833           0 : }
     834             : 
     835             : static const struct sysfs_ops queue_sysfs_ops = {
     836             :         .show   = queue_attr_show,
     837             :         .store  = queue_attr_store,
     838             : };
     839             : 
     840             : struct kobj_type blk_queue_ktype = {
     841             :         .sysfs_ops      = &queue_sysfs_ops,
     842             :         .release        = blk_release_queue,
     843             : };
     844             : 
     845             : /**
     846             :  * blk_register_queue - register a block layer queue with sysfs
     847             :  * @disk: Disk of which the request queue should be registered with sysfs.
     848             :  */
     849           9 : int blk_register_queue(struct gendisk *disk)
     850             : {
     851           9 :         int ret;
     852           9 :         struct device *dev = disk_to_dev(disk);
     853           9 :         struct request_queue *q = disk->queue;
     854             : 
     855           9 :         if (WARN_ON(!q))
     856             :                 return -ENXIO;
     857             : 
     858           9 :         WARN_ONCE(blk_queue_registered(q),
     859             :                   "%s is registering an already registered queue\n",
     860             :                   kobject_name(&dev->kobj));
     861             : 
     862             :         /*
     863             :          * SCSI probing may synchronously create and destroy a lot of
     864             :          * request_queues for non-existent devices.  Shutting down a fully
     865             :          * functional queue takes measureable wallclock time as RCU grace
     866             :          * periods are involved.  To avoid excessive latency in these
     867             :          * cases, a request_queue starts out in a degraded mode which is
     868             :          * faster to shut down and is made fully functional here as
     869             :          * request_queues for non-existent devices never get registered.
     870             :          */
     871           9 :         if (!blk_queue_init_done(q)) {
     872           9 :                 blk_queue_flag_set(QUEUE_FLAG_INIT_DONE, q);
     873           9 :                 percpu_ref_switch_to_percpu(&q->q_usage_counter);
     874             :         }
     875             : 
     876           9 :         blk_queue_update_readahead(q);
     877             : 
     878           9 :         ret = blk_trace_init_sysfs(dev);
     879           9 :         if (ret)
     880             :                 return ret;
     881             : 
     882           9 :         mutex_lock(&q->sysfs_dir_lock);
     883             : 
     884           9 :         ret = kobject_add(&q->kobj, kobject_get(&dev->kobj), "%s", "queue");
     885           9 :         if (ret < 0) {
     886           0 :                 blk_trace_remove_sysfs(dev);
     887           0 :                 goto unlock;
     888             :         }
     889             : 
     890           9 :         ret = sysfs_create_group(&q->kobj, &queue_attr_group);
     891           9 :         if (ret) {
     892           0 :                 blk_trace_remove_sysfs(dev);
     893           0 :                 kobject_del(&q->kobj);
     894           0 :                 kobject_put(&dev->kobj);
     895           0 :                 goto unlock;
     896             :         }
     897             : 
     898           9 :         mutex_lock(&q->debugfs_mutex);
     899           9 :         q->debugfs_dir = debugfs_create_dir(kobject_name(q->kobj.parent),
     900             :                                             blk_debugfs_root);
     901           9 :         mutex_unlock(&q->debugfs_mutex);
     902             : 
     903           9 :         if (queue_is_mq(q)) {
     904           9 :                 __blk_mq_register_dev(dev, q);
     905           9 :                 blk_mq_debugfs_register(q);
     906             :         }
     907             : 
     908           9 :         mutex_lock(&q->sysfs_lock);
     909           9 :         if (q->elevator) {
     910           0 :                 ret = elv_register_queue(q, false);
     911           0 :                 if (ret) {
     912           0 :                         mutex_unlock(&q->sysfs_lock);
     913           0 :                         mutex_unlock(&q->sysfs_dir_lock);
     914           0 :                         kobject_del(&q->kobj);
     915           0 :                         blk_trace_remove_sysfs(dev);
     916           0 :                         kobject_put(&dev->kobj);
     917           0 :                         return ret;
     918             :                 }
     919             :         }
     920             : 
     921           9 :         blk_queue_flag_set(QUEUE_FLAG_REGISTERED, q);
     922           9 :         wbt_enable_default(q);
     923           9 :         blk_throtl_register_queue(q);
     924             : 
     925             :         /* Now everything is ready and send out KOBJ_ADD uevent */
     926           9 :         kobject_uevent(&q->kobj, KOBJ_ADD);
     927           9 :         if (q->elevator)
     928           0 :                 kobject_uevent(&q->elevator->kobj, KOBJ_ADD);
     929           9 :         mutex_unlock(&q->sysfs_lock);
     930             : 
     931           9 :         ret = 0;
     932           9 : unlock:
     933           9 :         mutex_unlock(&q->sysfs_dir_lock);
     934           9 :         return ret;
     935             : }
     936             : EXPORT_SYMBOL_GPL(blk_register_queue);
     937             : 
     938             : /**
     939             :  * blk_unregister_queue - counterpart of blk_register_queue()
     940             :  * @disk: Disk of which the request queue should be unregistered from sysfs.
     941             :  *
     942             :  * Note: the caller is responsible for guaranteeing that this function is called
     943             :  * after blk_register_queue() has finished.
     944             :  */
     945           0 : void blk_unregister_queue(struct gendisk *disk)
     946             : {
     947           0 :         struct request_queue *q = disk->queue;
     948             : 
     949           0 :         if (WARN_ON(!q))
     950             :                 return;
     951             : 
     952             :         /* Return early if disk->queue was never registered. */
     953           0 :         if (!blk_queue_registered(q))
     954             :                 return;
     955             : 
     956             :         /*
     957             :          * Since sysfs_remove_dir() prevents adding new directory entries
     958             :          * before removal of existing entries starts, protect against
     959             :          * concurrent elv_iosched_store() calls.
     960             :          */
     961           0 :         mutex_lock(&q->sysfs_lock);
     962           0 :         blk_queue_flag_clear(QUEUE_FLAG_REGISTERED, q);
     963           0 :         mutex_unlock(&q->sysfs_lock);
     964             : 
     965           0 :         mutex_lock(&q->sysfs_dir_lock);
     966             :         /*
     967             :          * Remove the sysfs attributes before unregistering the queue data
     968             :          * structures that can be modified through sysfs.
     969             :          */
     970           0 :         if (queue_is_mq(q))
     971           0 :                 blk_mq_unregister_dev(disk_to_dev(disk), q);
     972             : 
     973           0 :         kobject_uevent(&q->kobj, KOBJ_REMOVE);
     974           0 :         kobject_del(&q->kobj);
     975           0 :         blk_trace_remove_sysfs(disk_to_dev(disk));
     976             : 
     977           0 :         mutex_lock(&q->sysfs_lock);
     978           0 :         if (q->elevator)
     979           0 :                 elv_unregister_queue(q);
     980           0 :         mutex_unlock(&q->sysfs_lock);
     981           0 :         mutex_unlock(&q->sysfs_dir_lock);
     982             : 
     983           0 :         kobject_put(&disk_to_dev(disk)->kobj);
     984             : }