Line data    Source code

       1             : /* SPDX-License-Identifier: GPL-2.0 */
       2             : #ifndef INT_BLK_MQ_TAG_H
       3             : #define INT_BLK_MQ_TAG_H
       4             : 
       5             : /*
       6             :  * Tag address space map.
       7             :  */
       8             : struct blk_mq_tags {
       9             :         unsigned int nr_tags;
      10             :         unsigned int nr_reserved_tags;
      11             : 
      12             :         atomic_t active_queues;
      13             : 
      14             :         struct sbitmap_queue *bitmap_tags;
      15             :         struct sbitmap_queue *breserved_tags;
      16             : 
      17             :         struct sbitmap_queue __bitmap_tags;
      18             :         struct sbitmap_queue __breserved_tags;
      19             : 
      20             :         struct request **rqs;
      21             :         struct request **static_rqs;
      22             :         struct list_head page_list;
      23             : };
      24             : 
      25             : extern struct blk_mq_tags *blk_mq_init_tags(unsigned int nr_tags,
      26             :                                         unsigned int reserved_tags,
      27             :                                         int node, unsigned int flags);
      28             : extern void blk_mq_free_tags(struct blk_mq_tags *tags, unsigned int flags);
      29             : 
      30             : extern int blk_mq_init_shared_sbitmap(struct blk_mq_tag_set *set,
      31             :                                       unsigned int flags);
      32             : extern void blk_mq_exit_shared_sbitmap(struct blk_mq_tag_set *set);
      33             : 
      34             : extern unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data);
      35             : extern void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx,
      36             :                            unsigned int tag);
      37             : extern int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx,
      38             :                                         struct blk_mq_tags **tags,
      39             :                                         unsigned int depth, bool can_grow);
      40             : extern void blk_mq_tag_resize_shared_sbitmap(struct blk_mq_tag_set *set,
      41             :                                              unsigned int size);
      42             : 
      43             : extern void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool);
      44             : void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn,
      45             :                 void *priv);
      46             : void blk_mq_all_tag_iter(struct blk_mq_tags *tags, busy_tag_iter_fn *fn,
      47             :                 void *priv);
      48             : 
      49           0 : static inline struct sbq_wait_state *bt_wait_ptr(struct sbitmap_queue *bt,
      50             :                                                  struct blk_mq_hw_ctx *hctx)
      51             : {
      52           0 :         if (!hctx)
      53           0 :                 return &bt->ws[0];
      54           0 :         return sbq_wait_ptr(bt, &hctx->wait_index);
      55             : }
      56             : 
      57             : enum {
      58             :         BLK_MQ_NO_TAG           = -1U,
      59             :         BLK_MQ_TAG_MIN          = 1,
      60             :         BLK_MQ_TAG_MAX          = BLK_MQ_NO_TAG - 1,
      61             : };
      62             : 
      63             : extern bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *);
      64             : extern void __blk_mq_tag_idle(struct blk_mq_hw_ctx *);
      65             : 
      66        3254 : static inline bool blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx)
      67             : {
      68        3254 :         if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED))
      69             :                 return false;
      70             : 
      71           0 :         return __blk_mq_tag_busy(hctx);
      72             : }
      73             : 
      74           8 : static inline void blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx)
      75             : {
      76           8 :         if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED))
      77             :                 return;
      78             : 
      79           0 :         __blk_mq_tag_idle(hctx);
      80             : }
      81             : 
      82        3254 : static inline bool blk_mq_tag_is_reserved(struct blk_mq_tags *tags,
      83             :                                           unsigned int tag)
      84             : {
      85        3254 :         return tag < tags->nr_reserved_tags;
      86             : }
      87             : 
      88             : #endif