1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef KUBLK_INTERNAL_H 3 #define KUBLK_INTERNAL_H 4 5 #include <unistd.h> 6 #include <stdlib.h> 7 #include <assert.h> 8 #include <stdio.h> 9 #include <stdarg.h> 10 #include <string.h> 11 #include <pthread.h> 12 #include <getopt.h> 13 #include <limits.h> 14 #include <poll.h> 15 #include <fcntl.h> 16 #include <sys/syscall.h> 17 #include <sys/mman.h> 18 #include <sys/ioctl.h> 19 #include <sys/inotify.h> 20 #include <sys/wait.h> 21 #include <sys/eventfd.h> 22 #include <sys/uio.h> 23 #include <sys/ipc.h> 24 #include <sys/shm.h> 25 #include <linux/io_uring.h> 26 #include <liburing.h> 27 #include <semaphore.h> 28 29 /* allow ublk_dep.h to override ublk_cmd.h */ 30 #include "ublk_dep.h" 31 #include <linux/ublk_cmd.h> 32 33 #define __maybe_unused __attribute__((unused)) 34 #define MAX_BACK_FILES 4 35 #ifndef min 36 #define min(a, b) ((a) < (b) ? (a) : (b)) 37 #endif 38 39 #define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0])) 40 41 /****************** part 1: libublk ********************/ 42 43 #define CTRL_DEV "/dev/ublk-control" 44 #define UBLKC_DEV "/dev/ublkc" 45 #define UBLKB_DEV "/dev/ublkb" 46 #define UBLK_CTRL_RING_DEPTH 32 47 #define ERROR_EVTFD_DEVID -2 48 49 /* queue idle timeout */ 50 #define UBLKSRV_IO_IDLE_SECS 20 51 52 #define UBLK_IO_MAX_BYTES (1 << 20) 53 #define UBLK_MAX_QUEUES 32 54 #define UBLK_QUEUE_DEPTH 1024 55 56 #define UBLK_DBG_DEV (1U << 0) 57 #define UBLK_DBG_QUEUE (1U << 1) 58 #define UBLK_DBG_IO_CMD (1U << 2) 59 #define UBLK_DBG_IO (1U << 3) 60 #define UBLK_DBG_CTRL_CMD (1U << 4) 61 #define UBLK_LOG (1U << 5) 62 63 struct ublk_dev; 64 struct ublk_queue; 65 66 struct stripe_ctx { 67 /* stripe */ 68 unsigned int chunk_size; 69 }; 70 71 struct fault_inject_ctx { 72 /* fault_inject */ 73 unsigned long delay_us; 74 }; 75 76 struct dev_ctx { 77 char tgt_type[16]; 78 unsigned long flags; 79 unsigned nr_hw_queues; 80 unsigned queue_depth; 81 int dev_id; 82 int nr_files; 83 char *files[MAX_BACK_FILES]; 84 unsigned int logging:1; 85 unsigned int all:1; 86 unsigned int fg:1; 87 unsigned int recovery:1; 88 89 int _evtfd; 90 int _shmid; 91 92 /* built from shmem, only for ublk_dump_dev() */ 93 struct ublk_dev *shadow_dev; 94 95 union { 96 struct stripe_ctx stripe; 97 struct fault_inject_ctx fault_inject; 98 }; 99 }; 100 101 struct ublk_ctrl_cmd_data { 102 __u32 cmd_op; 103 #define CTRL_CMD_HAS_DATA 1 104 #define CTRL_CMD_HAS_BUF 2 105 __u32 flags; 106 107 __u64 data[2]; 108 __u64 addr; 109 __u32 len; 110 }; 111 112 struct ublk_io { 113 char *buf_addr; 114 115 #define UBLKSRV_NEED_FETCH_RQ (1UL << 0) 116 #define UBLKSRV_NEED_COMMIT_RQ_COMP (1UL << 1) 117 #define UBLKSRV_IO_FREE (1UL << 2) 118 unsigned short flags; 119 unsigned short refs; /* used by target code only */ 120 121 int result; 122 123 unsigned short tgt_ios; 124 void *private_data; 125 }; 126 127 struct ublk_tgt_ops { 128 const char *name; 129 int (*init_tgt)(const struct dev_ctx *ctx, struct ublk_dev *); 130 void (*deinit_tgt)(struct ublk_dev *); 131 132 int (*queue_io)(struct ublk_queue *, int tag); 133 void (*tgt_io_done)(struct ublk_queue *, 134 int tag, const struct io_uring_cqe *); 135 136 /* 137 * Target specific command line handling 138 * 139 * each option requires argument for target command line 140 */ 141 void (*parse_cmd_line)(struct dev_ctx *ctx, int argc, char *argv[]); 142 void (*usage)(const struct ublk_tgt_ops *ops); 143 }; 144 145 struct ublk_tgt { 146 unsigned long dev_size; 147 unsigned int sq_depth; 148 unsigned int cq_depth; 149 const struct ublk_tgt_ops *ops; 150 struct ublk_params params; 151 152 int nr_backing_files; 153 unsigned long backing_file_size[MAX_BACK_FILES]; 154 char backing_file[MAX_BACK_FILES][PATH_MAX]; 155 }; 156 157 struct ublk_queue { 158 int q_id; 159 int q_depth; 160 unsigned int cmd_inflight; 161 unsigned int io_inflight; 162 struct ublk_dev *dev; 163 const struct ublk_tgt_ops *tgt_ops; 164 struct ublksrv_io_desc *io_cmd_buf; 165 struct io_uring ring; 166 struct ublk_io ios[UBLK_QUEUE_DEPTH]; 167 #define UBLKSRV_QUEUE_STOPPING (1U << 0) 168 #define UBLKSRV_QUEUE_IDLE (1U << 1) 169 #define UBLKSRV_NO_BUF (1U << 2) 170 #define UBLKSRV_ZC (1U << 3) 171 unsigned state; 172 pid_t tid; 173 pthread_t thread; 174 }; 175 176 struct ublk_dev { 177 struct ublk_tgt tgt; 178 struct ublksrv_ctrl_dev_info dev_info; 179 struct ublk_queue q[UBLK_MAX_QUEUES]; 180 181 int fds[MAX_BACK_FILES + 1]; /* fds[0] points to /dev/ublkcN */ 182 int nr_fds; 183 int ctrl_fd; 184 struct io_uring ring; 185 186 void *private_data; 187 }; 188 189 #ifndef offsetof 190 #define offsetof(TYPE, MEMBER) ((size_t)&((TYPE *)0)->MEMBER) 191 #endif 192 193 #ifndef container_of 194 #define container_of(ptr, type, member) ({ \ 195 unsigned long __mptr = (unsigned long)(ptr); \ 196 ((type *)(__mptr - offsetof(type, member))); }) 197 #endif 198 199 #define round_up(val, rnd) \ 200 (((val) + ((rnd) - 1)) & ~((rnd) - 1)) 201 202 203 extern unsigned int ublk_dbg_mask; 204 extern int ublk_queue_io_cmd(struct ublk_queue *q, struct ublk_io *io, unsigned tag); 205 206 static inline int is_target_io(__u64 user_data) 207 { 208 return (user_data & (1ULL << 63)) != 0; 209 } 210 211 static inline __u64 build_user_data(unsigned tag, unsigned op, 212 unsigned tgt_data, unsigned is_target_io) 213 { 214 assert(!(tag >> 16) && !(op >> 8) && !(tgt_data >> 16)); 215 216 return tag | (op << 16) | (tgt_data << 24) | (__u64)is_target_io << 63; 217 } 218 219 static inline unsigned int user_data_to_tag(__u64 user_data) 220 { 221 return user_data & 0xffff; 222 } 223 224 static inline unsigned int user_data_to_op(__u64 user_data) 225 { 226 return (user_data >> 16) & 0xff; 227 } 228 229 static inline unsigned int user_data_to_tgt_data(__u64 user_data) 230 { 231 return (user_data >> 24) & 0xffff; 232 } 233 234 static inline unsigned short ublk_cmd_op_nr(unsigned int op) 235 { 236 return _IOC_NR(op); 237 } 238 239 static inline void ublk_err(const char *fmt, ...) 240 { 241 va_list ap; 242 243 va_start(ap, fmt); 244 vfprintf(stderr, fmt, ap); 245 } 246 247 static inline void ublk_log(const char *fmt, ...) 248 { 249 if (ublk_dbg_mask & UBLK_LOG) { 250 va_list ap; 251 252 va_start(ap, fmt); 253 vfprintf(stdout, fmt, ap); 254 } 255 } 256 257 static inline void ublk_dbg(int level, const char *fmt, ...) 258 { 259 if (level & ublk_dbg_mask) { 260 va_list ap; 261 262 va_start(ap, fmt); 263 vfprintf(stdout, fmt, ap); 264 } 265 } 266 267 static inline int ublk_queue_alloc_sqes(struct ublk_queue *q, 268 struct io_uring_sqe *sqes[], int nr_sqes) 269 { 270 unsigned left = io_uring_sq_space_left(&q->ring); 271 int i; 272 273 if (left < nr_sqes) 274 io_uring_submit(&q->ring); 275 276 for (i = 0; i < nr_sqes; i++) { 277 sqes[i] = io_uring_get_sqe(&q->ring); 278 if (!sqes[i]) 279 return i; 280 } 281 282 return nr_sqes; 283 } 284 285 static inline void io_uring_prep_buf_register(struct io_uring_sqe *sqe, 286 int dev_fd, int tag, int q_id, __u64 index) 287 { 288 struct ublksrv_io_cmd *cmd = (struct ublksrv_io_cmd *)sqe->cmd; 289 290 io_uring_prep_read(sqe, dev_fd, 0, 0, 0); 291 sqe->opcode = IORING_OP_URING_CMD; 292 sqe->flags |= IOSQE_FIXED_FILE; 293 sqe->cmd_op = UBLK_U_IO_REGISTER_IO_BUF; 294 295 cmd->tag = tag; 296 cmd->addr = index; 297 cmd->q_id = q_id; 298 } 299 300 static inline void io_uring_prep_buf_unregister(struct io_uring_sqe *sqe, 301 int dev_fd, int tag, int q_id, __u64 index) 302 { 303 struct ublksrv_io_cmd *cmd = (struct ublksrv_io_cmd *)sqe->cmd; 304 305 io_uring_prep_read(sqe, dev_fd, 0, 0, 0); 306 sqe->opcode = IORING_OP_URING_CMD; 307 sqe->flags |= IOSQE_FIXED_FILE; 308 sqe->cmd_op = UBLK_U_IO_UNREGISTER_IO_BUF; 309 310 cmd->tag = tag; 311 cmd->addr = index; 312 cmd->q_id = q_id; 313 } 314 315 static inline void *ublk_get_sqe_cmd(const struct io_uring_sqe *sqe) 316 { 317 return (void *)&sqe->cmd; 318 } 319 320 static inline void ublk_set_io_res(struct ublk_queue *q, int tag, int res) 321 { 322 q->ios[tag].result = res; 323 } 324 325 static inline int ublk_get_io_res(const struct ublk_queue *q, unsigned tag) 326 { 327 return q->ios[tag].result; 328 } 329 330 static inline void ublk_mark_io_done(struct ublk_io *io, int res) 331 { 332 io->flags |= (UBLKSRV_NEED_COMMIT_RQ_COMP | UBLKSRV_IO_FREE); 333 io->result = res; 334 } 335 336 static inline const struct ublksrv_io_desc *ublk_get_iod(const struct ublk_queue *q, int tag) 337 { 338 return &q->io_cmd_buf[tag]; 339 } 340 341 static inline void ublk_set_sqe_cmd_op(struct io_uring_sqe *sqe, __u32 cmd_op) 342 { 343 __u32 *addr = (__u32 *)&sqe->off; 344 345 addr[0] = cmd_op; 346 addr[1] = 0; 347 } 348 349 static inline struct ublk_io *ublk_get_io(struct ublk_queue *q, unsigned tag) 350 { 351 return &q->ios[tag]; 352 } 353 354 static inline int ublk_complete_io(struct ublk_queue *q, unsigned tag, int res) 355 { 356 struct ublk_io *io = &q->ios[tag]; 357 358 ublk_mark_io_done(io, res); 359 360 return ublk_queue_io_cmd(q, io, tag); 361 } 362 363 static inline void ublk_queued_tgt_io(struct ublk_queue *q, unsigned tag, int queued) 364 { 365 if (queued < 0) 366 ublk_complete_io(q, tag, queued); 367 else { 368 struct ublk_io *io = ublk_get_io(q, tag); 369 370 q->io_inflight += queued; 371 io->tgt_ios = queued; 372 io->result = 0; 373 } 374 } 375 376 static inline int ublk_completed_tgt_io(struct ublk_queue *q, unsigned tag) 377 { 378 struct ublk_io *io = ublk_get_io(q, tag); 379 380 q->io_inflight--; 381 382 return --io->tgt_ios == 0; 383 } 384 385 static inline int ublk_queue_use_zc(const struct ublk_queue *q) 386 { 387 return q->state & UBLKSRV_ZC; 388 } 389 390 extern const struct ublk_tgt_ops null_tgt_ops; 391 extern const struct ublk_tgt_ops loop_tgt_ops; 392 extern const struct ublk_tgt_ops stripe_tgt_ops; 393 extern const struct ublk_tgt_ops fault_inject_tgt_ops; 394 395 void backing_file_tgt_deinit(struct ublk_dev *dev); 396 int backing_file_tgt_init(struct ublk_dev *dev); 397 398 static inline unsigned int ilog2(unsigned int x) 399 { 400 if (x == 0) 401 return 0; 402 return (sizeof(x) * 8 - 1) - __builtin_clz(x); 403 } 404 #endif 405