Line data    Source code

       1             : /* SPDX-License-Identifier: GPL-2.0 */
       2             : #ifndef __VDSO_HELPERS_H
       3             : #define __VDSO_HELPERS_H
       4             : 
       5             : #ifndef __ASSEMBLY__
       6             : 
       7             : #include <vdso/datapage.h>
       8             : 
       9             : static __always_inline u32 vdso_read_begin(const struct vdso_data *vd)
      10             : {
      11             :         u32 seq;
      12             : 
      13             :         while (unlikely((seq = READ_ONCE(vd->seq)) & 1))
      14             :                 cpu_relax();
      15             : 
      16             :         smp_rmb();
      17             :         return seq;
      18             : }
      19             : 
      20             : static __always_inline u32 vdso_read_retry(const struct vdso_data *vd,
      21             :                                            u32 start)
      22             : {
      23             :         u32 seq;
      24             : 
      25             :         smp_rmb();
      26             :         seq = READ_ONCE(vd->seq);
      27             :         return seq != start;
      28             : }
      29             : 
      30        8311 : static __always_inline void vdso_write_begin(struct vdso_data *vd)
      31             : {
      32             :         /*
      33             :          * WRITE_ONCE it is required otherwise the compiler can validly tear
      34             :          * updates to vd[x].seq and it is possible that the value seen by the
      35             :          * reader it is inconsistent.
      36             :          */
      37        8311 :         WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1);
      38        8311 :         WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1);
      39        8311 :         smp_wmb();
      40             : }
      41             : 
      42        8311 : static __always_inline void vdso_write_end(struct vdso_data *vd)
      43             : {
      44        8311 :         smp_wmb();
      45             :         /*
      46             :          * WRITE_ONCE it is required otherwise the compiler can validly tear
      47             :          * updates to vd[x].seq and it is possible that the value seen by the
      48             :          * reader it is inconsistent.
      49             :          */
      50        8311 :         WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1);
      51           0 :         WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1);
      52             : }
      53             : 
      54             : #endif /* !__ASSEMBLY__ */
      55             : 
      56             : #endif /* __VDSO_HELPERS_H */