Line data    Source code

       1             : // SPDX-License-Identifier: GPL-2.0
       2             : #include <linux/spinlock.h>
       3             : #include <linux/task_work.h>
       4             : #include <linux/tracehook.h>
       5             : 
       6             : static struct callback_head work_exited; /* all we need is ->next == NULL */
       7             : 
       8             : /**
       9             :  * task_work_add - ask the @task to execute @work->func()
      10             :  * @task: the task which should run the callback
      11             :  * @work: the callback to run
      12             :  * @notify: how to notify the targeted task
      13             :  *
      14             :  * Queue @work for task_work_run() below and notify the @task if @notify
      15             :  * is @TWA_RESUME or @TWA_SIGNAL. @TWA_SIGNAL works like signals, in that the
      16             :  * it will interrupt the targeted task and run the task_work. @TWA_RESUME
      17             :  * work is run only when the task exits the kernel and returns to user mode,
      18             :  * or before entering guest mode. Fails if the @task is exiting/exited and thus
      19             :  * it can't process this @work. Otherwise @work->func() will be called when the
      20             :  * @task goes through one of the aforementioned transitions, or exits.
      21             :  *
      22             :  * If the targeted task is exiting, then an error is returned and the work item
      23             :  * is not queued. It's up to the caller to arrange for an alternative mechanism
      24             :  * in that case.
      25             :  *
      26             :  * Note: there is no ordering guarantee on works queued here. The task_work
      27             :  * list is LIFO.
      28             :  *
      29             :  * RETURNS:
      30             :  * 0 if succeeds or -ESRCH.
      31             :  */
      32       56846 : int task_work_add(struct task_struct *task, struct callback_head *work,
      33             :                   enum task_work_notify_mode notify)
      34             : {
      35       56853 :         struct callback_head *head;
      36             : 
      37       56853 :         do {
      38       56853 :                 head = READ_ONCE(task->task_works);
      39       56853 :                 if (unlikely(head == &work_exited))
      40             :                         return -ESRCH;
      41       56853 :                 work->next = head;
      42       56853 :         } while (cmpxchg(&task->task_works, head, work) != head);
      43             : 
      44       56853 :         switch (notify) {
      45             :         case TWA_NONE:
      46             :                 break;
      47       56853 :         case TWA_RESUME:
      48       56853 :                 set_notify_resume(task);
      49       56853 :                 break;
      50           0 :         case TWA_SIGNAL:
      51           0 :                 set_notify_signal(task);
      52           0 :                 break;
      53             :         default:
      54           0 :                 WARN_ON_ONCE(1);
      55           0 :                 break;
      56             :         }
      57             : 
      58             :         return 0;
      59             : }
      60             : 
      61             : /**
      62             :  * task_work_cancel - cancel a pending work added by task_work_add()
      63             :  * @task: the task which should execute the work
      64             :  * @func: identifies the work to remove
      65             :  *
      66             :  * Find the last queued pending work with ->func == @func and remove
      67             :  * it from queue.
      68             :  *
      69             :  * RETURNS:
      70             :  * The found work or NULL if not found.
      71             :  */
      72             : struct callback_head *
      73           0 : task_work_cancel(struct task_struct *task, task_work_func_t func)
      74             : {
      75           0 :         struct callback_head **pprev = &task->task_works;
      76           0 :         struct callback_head *work;
      77           0 :         unsigned long flags;
      78             : 
      79           0 :         if (likely(!task->task_works))
      80             :                 return NULL;
      81             :         /*
      82             :          * If cmpxchg() fails we continue without updating pprev.
      83             :          * Either we raced with task_work_add() which added the
      84             :          * new entry before this work, we will find it again. Or
      85             :          * we raced with task_work_run(), *pprev == NULL/exited.
      86             :          */
      87           0 :         raw_spin_lock_irqsave(&task->pi_lock, flags);
      88           0 :         while ((work = READ_ONCE(*pprev))) {
      89           0 :                 if (work->func != func)
      90           0 :                         pprev = &work->next;
      91           0 :                 else if (cmpxchg(pprev, work, work->next) == work)
      92             :                         break;
      93             :         }
      94           0 :         raw_spin_unlock_irqrestore(&task->pi_lock, flags);
      95             : 
      96           0 :         return work;
      97             : }
      98             : 
      99             : /**
     100             :  * task_work_run - execute the works added by task_work_add()
     101             :  *
     102             :  * Flush the pending works. Should be used by the core kernel code.
     103             :  * Called before the task returns to the user-mode or stops, or when
     104             :  * it exits. In the latter case task_work_add() can no longer add the
     105             :  * new work after task_work_run() returns.
     106             :  */
     107       51366 : void task_work_run(void)
     108             : {
     109       51366 :         struct task_struct *task = current;
     110      102699 :         struct callback_head *work, *head, *next;
     111             : 
     112      102699 :         for (;;) {
     113             :                 /*
     114             :                  * work->func() can do task_work_add(), do not set
     115             :                  * work_exited unless the list is empty.
     116             :                  */
     117      102699 :                 do {
     118      102699 :                         head = NULL;
     119      102699 :                         work = READ_ONCE(task->task_works);
     120      102699 :                         if (!work) {
     121       51344 :                                 if (task->flags & PF_EXITING)
     122             :                                         head = &work_exited;
     123             :                                 else
     124             :                                         break;
     125             :                         }
     126       52218 :                 } while (cmpxchg(&task->task_works, work, head) != work);
     127             : 
     128      102702 :                 if (!work)
     129             :                         break;
     130             :                 /*
     131             :                  * Synchronize with task_work_cancel(). It can not remove
     132             :                  * the first entry == work, cmpxchg(task_works) must fail.
     133             :                  * But it can remove another entry from the ->next list.
     134             :                  */
     135       51354 :                 raw_spin_lock_irq(&task->pi_lock);
     136       51355 :                 raw_spin_unlock_irq(&task->pi_lock);
     137             : 
     138       56840 :                 do {
     139       56840 :                         next = work->next;
     140       56840 :                         work->func(work);
     141       56835 :                         work = next;
     142       56835 :                         cond_resched();
     143       56833 :                 } while (work);
     144             :         }
     145       51348 : }