1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __VDSO_HELPERS_H 3 #define __VDSO_HELPERS_H 4 5 #ifndef __ASSEMBLY__ 6 7 #include <asm/barrier.h> 8 #include <vdso/datapage.h> 9 vdso_read_begin(const struct vdso_clock * vc)10static __always_inline u32 vdso_read_begin(const struct vdso_clock *vc) 11 { 12 u32 seq; 13 14 while (unlikely((seq = READ_ONCE(vc->seq)) & 1)) 15 cpu_relax(); 16 17 smp_rmb(); 18 return seq; 19 } 20 vdso_read_retry(const struct vdso_clock * vc,u32 start)21static __always_inline u32 vdso_read_retry(const struct vdso_clock *vc, 22 u32 start) 23 { 24 u32 seq; 25 26 smp_rmb(); 27 seq = READ_ONCE(vc->seq); 28 return seq != start; 29 } 30 vdso_write_seq_begin(struct vdso_clock * vc)31static __always_inline void vdso_write_seq_begin(struct vdso_clock *vc) 32 { 33 /* 34 * WRITE_ONCE() is required otherwise the compiler can validly tear 35 * updates to vc->seq and it is possible that the value seen by the 36 * reader is inconsistent. 37 */ 38 WRITE_ONCE(vc->seq, vc->seq + 1); 39 } 40 vdso_write_seq_end(struct vdso_clock * vc)41static __always_inline void vdso_write_seq_end(struct vdso_clock *vc) 42 { 43 /* 44 * WRITE_ONCE() is required otherwise the compiler can validly tear 45 * updates to vc->seq and it is possible that the value seen by the 46 * reader is inconsistent. 47 */ 48 WRITE_ONCE(vc->seq, vc->seq + 1); 49 } 50 vdso_write_begin_clock(struct vdso_clock * vc)51static __always_inline void vdso_write_begin_clock(struct vdso_clock *vc) 52 { 53 vdso_write_seq_begin(vc); 54 /* Ensure the sequence invalidation is visible before data is modified */ 55 smp_wmb(); 56 } 57 vdso_write_end_clock(struct vdso_clock * vc)58static __always_inline void vdso_write_end_clock(struct vdso_clock *vc) 59 { 60 /* Ensure the data update is visible before the sequence is set valid again */ 61 smp_wmb(); 62 vdso_write_seq_end(vc); 63 } 64 vdso_write_begin(struct vdso_time_data * vd)65static __always_inline void vdso_write_begin(struct vdso_time_data *vd) 66 { 67 struct vdso_clock *vc = vd->clock_data; 68 69 vdso_write_seq_begin(&vc[CS_HRES_COARSE]); 70 vdso_write_seq_begin(&vc[CS_RAW]); 71 /* Ensure the sequence invalidation is visible before data is modified */ 72 smp_wmb(); 73 } 74 vdso_write_end(struct vdso_time_data * vd)75static __always_inline void vdso_write_end(struct vdso_time_data *vd) 76 { 77 struct vdso_clock *vc = vd->clock_data; 78 79 /* Ensure the data update is visible before the sequence is set valid again */ 80 smp_wmb(); 81 vdso_write_seq_end(&vc[CS_HRES_COARSE]); 82 vdso_write_seq_end(&vc[CS_RAW]); 83 } 84 85 #endif /* !__ASSEMBLY__ */ 86 87 #endif /* __VDSO_HELPERS_H */ 88