Line data    Source code

       1             : // SPDX-License-Identifier: GPL-2.0-or-later
       2             : /*  paravirtual clock -- common code used by kvm/xen
       3             : 
       4             : */
       5             : 
       6             : #include <linux/clocksource.h>
       7             : #include <linux/kernel.h>
       8             : #include <linux/percpu.h>
       9             : #include <linux/notifier.h>
      10             : #include <linux/sched.h>
      11             : #include <linux/gfp.h>
      12             : #include <linux/memblock.h>
      13             : #include <linux/nmi.h>
      14             : 
      15             : #include <asm/fixmap.h>
      16             : #include <asm/pvclock.h>
      17             : #include <asm/vgtod.h>
      18             : 
      19             : static u8 valid_flags __read_mostly = 0;
      20             : static struct pvclock_vsyscall_time_info *pvti_cpu0_va __read_mostly;
      21             : 
      22           1 : void pvclock_set_flags(u8 flags)
      23             : {
      24           1 :         valid_flags = flags;
      25           1 : }
      26             : 
      27           3 : unsigned long pvclock_tsc_khz(struct pvclock_vcpu_time_info *src)
      28             : {
      29           3 :         u64 pv_tsc_khz = 1000000ULL << 32;
      30             : 
      31           3 :         do_div(pv_tsc_khz, src->tsc_to_system_mul);
      32           3 :         if (src->tsc_shift < 0)
      33           3 :                 pv_tsc_khz <<= -src->tsc_shift;
      34             :         else
      35           0 :                 pv_tsc_khz >>= src->tsc_shift;
      36           3 :         return pv_tsc_khz;
      37             : }
      38             : 
      39           0 : void pvclock_touch_watchdogs(void)
      40             : {
      41           0 :         touch_softlockup_watchdog_sync();
      42           0 :         clocksource_touch_watchdog();
      43           0 :         rcu_cpu_stall_reset();
      44           0 :         reset_hung_task_detector();
      45           0 : }
      46             : 
      47             : static atomic64_t last_value = ATOMIC64_INIT(0);
      48             : 
      49           0 : void pvclock_resume(void)
      50             : {
      51           0 :         atomic64_set(&last_value, 0);
      52           0 : }
      53             : 
      54           2 : u8 pvclock_read_flags(struct pvclock_vcpu_time_info *src)
      55             : {
      56           2 :         unsigned version;
      57           2 :         u8 flags;
      58             : 
      59           2 :         do {
      60           2 :                 version = pvclock_read_begin(src);
      61           2 :                 flags = src->flags;
      62           2 :         } while (pvclock_read_retry(src, version));
      63             : 
      64           2 :         return flags & valid_flags;
      65             : }
      66             : 
      67      654683 : u64 pvclock_clocksource_read(struct pvclock_vcpu_time_info *src)
      68             : {
      69      654683 :         unsigned version;
      70      654683 :         u64 ret;
      71      654683 :         u64 last;
      72      654683 :         u8 flags;
      73             : 
      74      654683 :         do {
      75      654683 :                 version = pvclock_read_begin(src);
      76      659352 :                 ret = __pvclock_read_cycles(src, rdtsc_ordered());
      77      661193 :                 flags = src->flags;
      78      661193 :         } while (pvclock_read_retry(src, version));
      79             : 
      80      662599 :         if (unlikely((flags & PVCLOCK_GUEST_STOPPED) != 0)) {
      81           0 :                 src->flags &= ~PVCLOCK_GUEST_STOPPED;
      82           0 :                 pvclock_touch_watchdogs();
      83             :         }
      84             : 
      85      662599 :         if ((valid_flags & PVCLOCK_TSC_STABLE_BIT) &&
      86             :                 (flags & PVCLOCK_TSC_STABLE_BIT))
      87             :                 return ret;
      88             : 
      89             :         /*
      90             :          * Assumption here is that last_value, a global accumulator, always goes
      91             :          * forward. If we are less than that, we should not be much smaller.
      92             :          * We assume there is an error marging we're inside, and then the correction
      93             :          * does not sacrifice accuracy.
      94             :          *
      95             :          * For reads: global may have changed between test and return,
      96             :          * but this means someone else updated poked the clock at a later time.
      97             :          * We just need to make sure we are not seeing a backwards event.
      98             :          *
      99             :          * For updates: last_value = ret is not enough, since two vcpus could be
     100             :          * updating at the same time, and one of them could be slightly behind,
     101             :          * making the assumption that last_value always go forward fail to hold.
     102             :          */
     103           0 :         last = atomic64_read(&last_value);
     104           0 :         do {
     105           0 :                 if (ret < last)
     106           0 :                         return last;
     107           0 :                 last = atomic64_cmpxchg(&last_value, last, ret);
     108           0 :         } while (unlikely(last != ret));
     109             : 
     110             :         return ret;
     111             : }
     112             : 
     113           1 : void pvclock_read_wallclock(struct pvclock_wall_clock *wall_clock,
     114             :                             struct pvclock_vcpu_time_info *vcpu_time,
     115             :                             struct timespec64 *ts)
     116             : {
     117           1 :         u32 version;
     118           1 :         u64 delta;
     119           1 :         struct timespec64 now;
     120             : 
     121             :         /* get wallclock at system boot */
     122           1 :         do {
     123           1 :                 version = wall_clock->version;
     124           1 :                 rmb();          /* fetch version before time */
     125             :                 /*
     126             :                  * Note: wall_clock->sec is a u32 value, so it can
     127             :                  * only store dates between 1970 and 2106. To allow
     128             :                  * times beyond that, we need to create a new hypercall
     129             :                  * interface with an extended pvclock_wall_clock structure
     130             :                  * like ARM has.
     131             :                  */
     132           1 :                 now.tv_sec  = wall_clock->sec;
     133           1 :                 now.tv_nsec = wall_clock->nsec;
     134           1 :                 rmb();          /* fetch time before checking version */
     135           1 :         } while ((wall_clock->version & 1) || (version != wall_clock->version));
     136             : 
     137           1 :         delta = pvclock_clocksource_read(vcpu_time);    /* time since system boot */
     138           1 :         delta += now.tv_sec * NSEC_PER_SEC + now.tv_nsec;
     139             : 
     140           1 :         now.tv_nsec = do_div(delta, NSEC_PER_SEC);
     141           1 :         now.tv_sec = delta;
     142             : 
     143           1 :         set_normalized_timespec64(ts, now.tv_sec, now.tv_nsec);
     144           1 : }
     145             : 
     146           1 : void pvclock_set_pvti_cpu0_va(struct pvclock_vsyscall_time_info *pvti)
     147             : {
     148           1 :         WARN_ON(vclock_was_used(VDSO_CLOCKMODE_PVCLOCK));
     149           1 :         pvti_cpu0_va = pvti;
     150           1 : }
     151             : 
     152          58 : struct pvclock_vsyscall_time_info *pvclock_get_pvti_cpu0_va(void)
     153             : {
     154          58 :         return pvti_cpu0_va;
     155             : }
     156             : EXPORT_SYMBOL_GPL(pvclock_get_pvti_cpu0_va);