1 /* 2 * Copyright (C) 2020, Red Hat Inc, Eric Auger <eric.auger@redhat.com> 3 * 4 * Most of the code is copy-pasted from: 5 * drivers/irqchip/irq-gic-v3-its.c 6 * This work is licensed under the terms of the GNU LGPL, version 2. 7 */ 8 #include <asm/io.h> 9 #include <asm/gic.h> 10 11 #define ITS_ITT_ALIGN SZ_256 12 13 static const char * const its_cmd_string[] = { 14 [GITS_CMD_MAPD] = "MAPD", 15 [GITS_CMD_MAPC] = "MAPC", 16 [GITS_CMD_MAPTI] = "MAPTI", 17 [GITS_CMD_MAPI] = "MAPI", 18 [GITS_CMD_MOVI] = "MOVI", 19 [GITS_CMD_DISCARD] = "DISCARD", 20 [GITS_CMD_INV] = "INV", 21 [GITS_CMD_MOVALL] = "MOVALL", 22 [GITS_CMD_INVALL] = "INVALL", 23 [GITS_CMD_INT] = "INT", 24 [GITS_CMD_CLEAR] = "CLEAR", 25 [GITS_CMD_SYNC] = "SYNC", 26 }; 27 28 struct its_cmd_desc { 29 union { 30 struct { 31 struct its_device *dev; 32 u32 event_id; 33 } its_inv_cmd; 34 35 struct { 36 struct its_device *dev; 37 u32 event_id; 38 } its_int_cmd; 39 40 struct { 41 struct its_device *dev; 42 bool valid; 43 } its_mapd_cmd; 44 45 struct { 46 struct its_collection *col; 47 bool valid; 48 } its_mapc_cmd; 49 50 struct { 51 struct its_device *dev; 52 u32 phys_id; 53 u32 event_id; 54 u32 col_id; 55 } its_mapti_cmd; 56 57 struct { 58 struct its_device *dev; 59 struct its_collection *col; 60 u32 event_id; 61 } its_movi_cmd; 62 63 struct { 64 struct its_device *dev; 65 u32 event_id; 66 } its_discard_cmd; 67 68 struct { 69 struct its_device *dev; 70 u32 event_id; 71 } its_clear_cmd; 72 73 struct { 74 struct its_collection *col; 75 } its_invall_cmd; 76 77 struct { 78 struct its_collection *col; 79 } its_sync_cmd; 80 }; 81 bool verbose; 82 }; 83 84 typedef void (*its_cmd_builder_t)(struct its_cmd_block *, 85 struct its_cmd_desc *); 86 87 /* ITS COMMANDS */ 88 89 static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l) 90 { 91 u64 mask = GENMASK_ULL(h, l); 92 *raw_cmd &= ~mask; 93 *raw_cmd |= (val << l) & mask; 94 } 95 96 static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr) 97 { 98 its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0); 99 } 100 101 static void its_encode_devid(struct its_cmd_block *cmd, u32 devid) 102 { 103 its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32); 104 } 105 106 static void its_encode_event_id(struct its_cmd_block *cmd, u32 id) 107 { 108 its_mask_encode(&cmd->raw_cmd[1], id, 31, 0); 109 } 110 111 static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id) 112 { 113 its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32); 114 } 115 116 static void its_encode_size(struct its_cmd_block *cmd, u8 size) 117 { 118 its_mask_encode(&cmd->raw_cmd[1], size, 4, 0); 119 } 120 121 static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr) 122 { 123 its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 50, 8); 124 } 125 126 static void its_encode_valid(struct its_cmd_block *cmd, int valid) 127 { 128 its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63); 129 } 130 131 static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr) 132 { 133 its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 50, 16); 134 } 135 136 static void its_encode_collection(struct its_cmd_block *cmd, u16 col) 137 { 138 its_mask_encode(&cmd->raw_cmd[2], col, 15, 0); 139 } 140 141 static inline void its_fixup_cmd(struct its_cmd_block *cmd) 142 { 143 /* Let's fixup BE commands */ 144 cmd->raw_cmd[0] = cpu_to_le64(cmd->raw_cmd[0]); 145 cmd->raw_cmd[1] = cpu_to_le64(cmd->raw_cmd[1]); 146 cmd->raw_cmd[2] = cpu_to_le64(cmd->raw_cmd[2]); 147 cmd->raw_cmd[3] = cpu_to_le64(cmd->raw_cmd[3]); 148 } 149 150 static u64 its_cmd_ptr_to_offset(struct its_cmd_block *ptr) 151 { 152 return (ptr - its_data.cmd_base) * sizeof(*ptr); 153 } 154 155 static struct its_cmd_block *its_post_commands(void) 156 { 157 u64 wr = its_cmd_ptr_to_offset(its_data.cmd_write); 158 159 writeq(wr, its_data.base + GITS_CWRITER); 160 return its_data.cmd_write; 161 } 162 163 static struct its_cmd_block *its_allocate_entry(void) 164 { 165 struct its_cmd_block *cmd; 166 167 cmd = its_data.cmd_write++; 168 if ((u64)its_data.cmd_write == (u64)its_data.cmd_base + SZ_64K) 169 its_data.cmd_write = its_data.cmd_base; 170 return cmd; 171 } 172 173 static void its_wait_for_range_completion(struct its_cmd_block *from, 174 struct its_cmd_block *to) 175 { 176 u64 rd_idx, from_idx, to_idx; 177 u32 count = 1000000; /* 1s! */ 178 179 from_idx = its_cmd_ptr_to_offset(from); 180 to_idx = its_cmd_ptr_to_offset(to); 181 while (1) { 182 rd_idx = readq(its_data.base + GITS_CREADR); 183 if (rd_idx >= to_idx || rd_idx < from_idx) 184 break; 185 186 count--; 187 if (!count) { 188 unsigned int cmd_id = from->raw_cmd[0] & 0xFF; 189 190 assert_msg(false, "%s timeout!", 191 cmd_id <= 0xF ? its_cmd_string[cmd_id] : 192 "Unexpected"); 193 } 194 udelay(1); 195 } 196 } 197 198 static void its_send_single_command(its_cmd_builder_t builder, 199 struct its_cmd_desc *desc) 200 { 201 struct its_cmd_block *cmd, *next_cmd; 202 203 cmd = its_allocate_entry(); 204 builder(cmd, desc); 205 next_cmd = its_post_commands(); 206 207 its_wait_for_range_completion(cmd, next_cmd); 208 } 209 210 static void its_build_mapd_cmd(struct its_cmd_block *cmd, 211 struct its_cmd_desc *desc) 212 { 213 unsigned long itt_addr; 214 u8 size = desc->its_mapd_cmd.dev->nr_ites; 215 216 itt_addr = (unsigned long)(virt_to_phys(desc->its_mapd_cmd.dev->itt)); 217 itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN); 218 219 its_encode_cmd(cmd, GITS_CMD_MAPD); 220 its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id); 221 its_encode_size(cmd, size - 1); 222 its_encode_itt(cmd, itt_addr); 223 its_encode_valid(cmd, desc->its_mapd_cmd.valid); 224 its_fixup_cmd(cmd); 225 if (desc->verbose) 226 printf("ITS: MAPD devid=%d size = 0x%x itt=0x%lx valid=%d\n", 227 desc->its_mapd_cmd.dev->device_id, 228 size, itt_addr, desc->its_mapd_cmd.valid); 229 } 230 231 static void its_build_mapc_cmd(struct its_cmd_block *cmd, 232 struct its_cmd_desc *desc) 233 { 234 its_encode_cmd(cmd, GITS_CMD_MAPC); 235 its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id); 236 its_encode_target(cmd, desc->its_mapc_cmd.col->target_address); 237 its_encode_valid(cmd, desc->its_mapc_cmd.valid); 238 its_fixup_cmd(cmd); 239 if (desc->verbose) 240 printf("MAPC col_id=%d target_addr = 0x%lx valid=%d\n", 241 desc->its_mapc_cmd.col->col_id, 242 desc->its_mapc_cmd.col->target_address, 243 desc->its_mapc_cmd.valid); 244 } 245 246 static void its_build_mapti_cmd(struct its_cmd_block *cmd, 247 struct its_cmd_desc *desc) 248 { 249 its_encode_cmd(cmd, GITS_CMD_MAPTI); 250 its_encode_devid(cmd, desc->its_mapti_cmd.dev->device_id); 251 its_encode_event_id(cmd, desc->its_mapti_cmd.event_id); 252 its_encode_phys_id(cmd, desc->its_mapti_cmd.phys_id); 253 its_encode_collection(cmd, desc->its_mapti_cmd.col_id); 254 its_fixup_cmd(cmd); 255 if (desc->verbose) 256 printf("MAPTI dev_id=%d event_id=%d -> phys_id=%d, col_id=%d\n", 257 desc->its_mapti_cmd.dev->device_id, 258 desc->its_mapti_cmd.event_id, 259 desc->its_mapti_cmd.phys_id, 260 desc->its_mapti_cmd.col_id); 261 } 262 263 static void its_build_invall_cmd(struct its_cmd_block *cmd, 264 struct its_cmd_desc *desc) 265 { 266 its_encode_cmd(cmd, GITS_CMD_INVALL); 267 its_encode_collection(cmd, desc->its_invall_cmd.col->col_id); 268 its_fixup_cmd(cmd); 269 if (desc->verbose) 270 printf("INVALL col_id=%d\n", desc->its_invall_cmd.col->col_id); 271 } 272 273 static void its_build_clear_cmd(struct its_cmd_block *cmd, 274 struct its_cmd_desc *desc) 275 { 276 its_encode_cmd(cmd, GITS_CMD_CLEAR); 277 its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id); 278 its_encode_event_id(cmd, desc->its_clear_cmd.event_id); 279 its_fixup_cmd(cmd); 280 if (desc->verbose) 281 printf("CLEAR dev_id=%d event_id=%d\n", desc->its_clear_cmd.dev->device_id, desc->its_clear_cmd.event_id); 282 } 283 284 static void its_build_discard_cmd(struct its_cmd_block *cmd, 285 struct its_cmd_desc *desc) 286 { 287 its_encode_cmd(cmd, GITS_CMD_DISCARD); 288 its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id); 289 its_encode_event_id(cmd, desc->its_discard_cmd.event_id); 290 its_fixup_cmd(cmd); 291 if (desc->verbose) 292 printf("DISCARD dev_id=%d event_id=%d\n", 293 desc->its_clear_cmd.dev->device_id, desc->its_clear_cmd.event_id); 294 } 295 296 static void its_build_inv_cmd(struct its_cmd_block *cmd, 297 struct its_cmd_desc *desc) 298 { 299 its_encode_cmd(cmd, GITS_CMD_INV); 300 its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id); 301 its_encode_event_id(cmd, desc->its_inv_cmd.event_id); 302 its_fixup_cmd(cmd); 303 if (desc->verbose) 304 printf("INV dev_id=%d event_id=%d\n", 305 desc->its_inv_cmd.dev->device_id, 306 desc->its_inv_cmd.event_id); 307 } 308 309 static void its_build_int_cmd(struct its_cmd_block *cmd, 310 struct its_cmd_desc *desc) 311 { 312 its_encode_cmd(cmd, GITS_CMD_INT); 313 its_encode_devid(cmd, desc->its_int_cmd.dev->device_id); 314 its_encode_event_id(cmd, desc->its_int_cmd.event_id); 315 its_fixup_cmd(cmd); 316 if (desc->verbose) 317 printf("INT dev_id=%d event_id=%d\n", 318 desc->its_int_cmd.dev->device_id, 319 desc->its_int_cmd.event_id); 320 } 321 322 static void its_build_sync_cmd(struct its_cmd_block *cmd, 323 struct its_cmd_desc *desc) 324 { 325 its_encode_cmd(cmd, GITS_CMD_SYNC); 326 its_encode_target(cmd, desc->its_sync_cmd.col->target_address); 327 its_fixup_cmd(cmd); 328 if (desc->verbose) 329 printf("SYNC target_addr = 0x%lx\n", 330 desc->its_sync_cmd.col->target_address); 331 } 332 333 static void its_build_movi_cmd(struct its_cmd_block *cmd, 334 struct its_cmd_desc *desc) 335 { 336 its_encode_cmd(cmd, GITS_CMD_MOVI); 337 its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id); 338 its_encode_event_id(cmd, desc->its_movi_cmd.event_id); 339 its_encode_collection(cmd, desc->its_movi_cmd.col->col_id); 340 its_fixup_cmd(cmd); 341 if (desc->verbose) 342 printf("MOVI dev_id=%d event_id = %d col_id=%d\n", 343 desc->its_movi_cmd.dev->device_id, 344 desc->its_movi_cmd.event_id, 345 desc->its_movi_cmd.col->col_id); 346 } 347 348 void __its_send_mapd(struct its_device *dev, int valid, bool verbose) 349 { 350 struct its_cmd_desc desc; 351 352 desc.its_mapd_cmd.dev = dev; 353 desc.its_mapd_cmd.valid = !!valid; 354 desc.verbose = verbose; 355 356 its_send_single_command(its_build_mapd_cmd, &desc); 357 } 358 359 void __its_send_mapc(struct its_collection *col, int valid, bool verbose) 360 { 361 struct its_cmd_desc desc; 362 363 desc.its_mapc_cmd.col = col; 364 desc.its_mapc_cmd.valid = !!valid; 365 desc.verbose = verbose; 366 367 its_send_single_command(its_build_mapc_cmd, &desc); 368 } 369 370 void __its_send_mapti(struct its_device *dev, u32 irq_id, 371 u32 event_id, struct its_collection *col, bool verbose) 372 { 373 struct its_cmd_desc desc; 374 375 desc.its_mapti_cmd.dev = dev; 376 desc.its_mapti_cmd.phys_id = irq_id; 377 desc.its_mapti_cmd.event_id = event_id; 378 desc.its_mapti_cmd.col_id = col->col_id; 379 desc.verbose = verbose; 380 381 its_send_single_command(its_build_mapti_cmd, &desc); 382 } 383 384 void __its_send_int(struct its_device *dev, u32 event_id, bool verbose) 385 { 386 struct its_cmd_desc desc; 387 388 desc.its_int_cmd.dev = dev; 389 desc.its_int_cmd.event_id = event_id; 390 desc.verbose = verbose; 391 392 its_send_single_command(its_build_int_cmd, &desc); 393 } 394 395 void __its_send_movi(struct its_device *dev, struct its_collection *col, 396 u32 id, bool verbose) 397 { 398 struct its_cmd_desc desc; 399 400 desc.its_movi_cmd.dev = dev; 401 desc.its_movi_cmd.col = col; 402 desc.its_movi_cmd.event_id = id; 403 desc.verbose = verbose; 404 405 its_send_single_command(its_build_movi_cmd, &desc); 406 } 407 408 void __its_send_invall(struct its_collection *col, bool verbose) 409 { 410 struct its_cmd_desc desc; 411 412 desc.its_invall_cmd.col = col; 413 desc.verbose = verbose; 414 415 its_send_single_command(its_build_invall_cmd, &desc); 416 } 417 418 void __its_send_inv(struct its_device *dev, u32 event_id, bool verbose) 419 { 420 struct its_cmd_desc desc; 421 422 desc.its_inv_cmd.dev = dev; 423 desc.its_inv_cmd.event_id = event_id; 424 desc.verbose = verbose; 425 426 its_send_single_command(its_build_inv_cmd, &desc); 427 } 428 429 void __its_send_discard(struct its_device *dev, u32 event_id, bool verbose) 430 { 431 struct its_cmd_desc desc; 432 433 desc.its_discard_cmd.dev = dev; 434 desc.its_discard_cmd.event_id = event_id; 435 desc.verbose = verbose; 436 437 its_send_single_command(its_build_discard_cmd, &desc); 438 } 439 440 void __its_send_clear(struct its_device *dev, u32 event_id, bool verbose) 441 { 442 struct its_cmd_desc desc; 443 444 desc.its_clear_cmd.dev = dev; 445 desc.its_clear_cmd.event_id = event_id; 446 desc.verbose = verbose; 447 448 its_send_single_command(its_build_clear_cmd, &desc); 449 } 450 451 void __its_send_sync(struct its_collection *col, bool verbose) 452 { 453 struct its_cmd_desc desc; 454 455 desc.its_sync_cmd.col = col; 456 desc.verbose = verbose; 457 458 its_send_single_command(its_build_sync_cmd, &desc); 459 } 460 461