1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * hldio.h - NVMe Direct I/O (HLDIO) infrastructure for Habana Labs Driver
4 *
5 * This feature requires specific hardware setup and must not be built
6 * under COMPILE_TEST.
7 */
8
9 #ifndef __HL_HLDIO_H__
10 #define __HL_HLDIO_H__
11
12 #include <linux/types.h>
13 #include <linux/fs.h>
14 #include <linux/seq_file.h>
15 #include <linux/ktime.h> /* ktime functions */
16 #include <linux/delay.h> /* usleep_range */
17 #include <linux/kernel.h> /* might_sleep_if */
18 #include <linux/errno.h> /* error codes */
19
20 /* Forward declarations */
21 struct hl_device;
22 struct file;
23
24 /* Enable only if Kconfig selected */
25 #ifdef CONFIG_HL_HLDIO
26 /**
27 * struct hl_p2p_region - describes a single P2P memory region
28 * @p2ppages: array of page structs for the P2P memory
29 * @p2pmem: virtual address of the P2P memory region
30 * @device_pa: physical address on the device
31 * @bar_offset: offset within the BAR
32 * @size: size of the region in bytes
33 * @bar: BAR number containing this region
34 */
35 struct hl_p2p_region {
36 struct page **p2ppages;
37 void *p2pmem;
38 u64 device_pa;
39 u64 bar_offset;
40 u64 size;
41 int bar;
42 };
43
44 /**
45 * struct hl_dio_stats - Direct I/O statistics
46 * @total_ops: total number of operations attempted
47 * @successful_ops: number of successful operations
48 * @failed_ops: number of failed operations
49 * @bytes_transferred: total bytes successfully transferred
50 * @last_len_read: length of the last read operation
51 */
52 struct hl_dio_stats {
53 u64 total_ops;
54 u64 successful_ops;
55 u64 failed_ops;
56 u64 bytes_transferred;
57 size_t last_len_read;
58 };
59
60 /**
61 * struct hl_dio - describes habanalabs direct storage interaction interface
62 * @p2prs: array of p2p regions
63 * @inflight_ios: percpu counter for inflight ios
64 * @np2prs: number of elements in p2prs
65 * @io_enabled: 1 if io is enabled 0 otherwise
66 */
67 struct hl_dio {
68 struct hl_p2p_region *p2prs;
69 s64 __percpu *inflight_ios;
70 u8 np2prs;
71 u8 io_enabled;
72 };
73
74 int hl_dio_ssd2hl(struct hl_device *hdev, struct hl_ctx *ctx, int fd,
75 u64 device_va, off_t off_bytes, size_t len_bytes,
76 size_t *len_read);
77 void hl_p2p_region_fini_all(struct hl_device *hdev);
78 int hl_p2p_region_init(struct hl_device *hdev, struct hl_p2p_region *p2pr);
79 int hl_dio_start(struct hl_device *hdev);
80 void hl_dio_stop(struct hl_device *hdev);
81
82 /* Init/teardown */
83 int hl_hldio_init(struct hl_device *hdev);
84 void hl_hldio_fini(struct hl_device *hdev);
85
86 /* File operations */
87 long hl_hldio_ioctl(struct file *filep, unsigned int cmd, unsigned long arg);
88
89 /* DebugFS hooks */
90 #ifdef CONFIG_DEBUG_FS
91 void hl_hldio_debugfs_init(struct hl_device *hdev);
92 void hl_hldio_debugfs_fini(struct hl_device *hdev);
93 #else
hl_hldio_debugfs_init(struct hl_device * hdev)94 static inline void hl_hldio_debugfs_init(struct hl_device *hdev) { }
hl_hldio_debugfs_fini(struct hl_device * hdev)95 static inline void hl_hldio_debugfs_fini(struct hl_device *hdev) { }
96 #endif
97
98 #else /* !CONFIG_HL_HLDIO */
99
100 struct hl_p2p_region;
101 /* Stubs when HLDIO is disabled */
hl_dio_ssd2hl(struct hl_device * hdev,struct hl_ctx * ctx,int fd,u64 device_va,off_t off_bytes,size_t len_bytes,size_t * len_read)102 static inline int hl_dio_ssd2hl(struct hl_device *hdev, struct hl_ctx *ctx, int fd,
103 u64 device_va, off_t off_bytes, size_t len_bytes,
104 size_t *len_read)
105 { return -EOPNOTSUPP; }
hl_p2p_region_fini_all(struct hl_device * hdev)106 static inline void hl_p2p_region_fini_all(struct hl_device *hdev) {}
hl_p2p_region_init(struct hl_device * hdev,struct hl_p2p_region * p2pr)107 static inline int hl_p2p_region_init(struct hl_device *hdev, struct hl_p2p_region *p2pr)
108 { return -EOPNOTSUPP; }
hl_dio_start(struct hl_device * hdev)109 static inline int hl_dio_start(struct hl_device *hdev) { return -EOPNOTSUPP; }
hl_dio_stop(struct hl_device * hdev)110 static inline void hl_dio_stop(struct hl_device *hdev) {}
111
hl_hldio_init(struct hl_device * hdev)112 static inline int hl_hldio_init(struct hl_device *hdev) { return 0; }
hl_hldio_fini(struct hl_device * hdev)113 static inline void hl_hldio_fini(struct hl_device *hdev) { }
hl_hldio_ioctl(struct file * f,unsigned int c,unsigned long a)114 static inline long hl_hldio_ioctl(struct file *f, unsigned int c,
115 unsigned long a)
116 { return -ENOTTY; }
hl_hldio_debugfs_init(struct hl_device * hdev)117 static inline void hl_hldio_debugfs_init(struct hl_device *hdev) { }
hl_hldio_debugfs_fini(struct hl_device * hdev)118 static inline void hl_hldio_debugfs_fini(struct hl_device *hdev) { }
119
120 #endif /* CONFIG_HL_HLDIO */
121
122 /* Simplified polling macro for HLDIO (no simulator support) */
123 #define hl_poll_timeout_condition(hdev, cond, sleep_us, timeout_us) \
124 ({ \
125 ktime_t __timeout = ktime_add_us(ktime_get(), timeout_us); \
126 might_sleep_if(sleep_us); \
127 (void)(hdev); /* keep signature consistent, hdev unused */ \
128 for (;;) { \
129 mb(); /* ensure ordering of memory operations */ \
130 if (cond) \
131 break; \
132 if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) \
133 break; \
134 if (sleep_us) \
135 usleep_range((sleep_us >> 2) + 1, sleep_us); \
136 } \
137 (cond) ? 0 : -ETIMEDOUT; \
138 })
139
140 #ifdef CONFIG_HL_HLDIO
141 bool hl_device_supports_nvme(struct hl_device *hdev);
142 #else
hl_device_supports_nvme(struct hl_device * hdev)143 static inline bool hl_device_supports_nvme(struct hl_device *hdev) { return false; }
144 #endif
145
146 #endif /* __HL_HLDIO_H__ */
147