1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* 3 * Copyright(c) 2015 - 2018 Intel Corporation. 4 */ 5 6 #include <linux/spinlock.h> 7 #include <linux/seqlock.h> 8 #include <linux/netdevice.h> 9 #include <linux/moduleparam.h> 10 #include <linux/bitops.h> 11 #include <linux/timer.h> 12 #include <linux/vmalloc.h> 13 #include <linux/highmem.h> 14 15 #include "hfi.h" 16 #include "common.h" 17 #include "qp.h" 18 #include "sdma.h" 19 #include "iowait.h" 20 #include "trace.h" 21 22 /* must be a power of 2 >= 64 <= 32768 */ 23 #define SDMA_DESCQ_CNT 2048 24 #define SDMA_DESC_INTR 64 25 #define INVALID_TAIL 0xffff 26 #define SDMA_PAD max_t(size_t, MAX_16B_PADDING, sizeof(u32)) 27 28 static uint sdma_descq_cnt = SDMA_DESCQ_CNT; 29 module_param(sdma_descq_cnt, uint, S_IRUGO); 30 MODULE_PARM_DESC(sdma_descq_cnt, "Number of SDMA descq entries"); 31 32 static uint sdma_idle_cnt = 250; 33 module_param(sdma_idle_cnt, uint, S_IRUGO); 34 MODULE_PARM_DESC(sdma_idle_cnt, "sdma interrupt idle delay (ns,default 250)"); 35 36 uint mod_num_sdma; 37 module_param_named(num_sdma, mod_num_sdma, uint, S_IRUGO); 38 MODULE_PARM_DESC(num_sdma, "Set max number SDMA engines to use"); 39 40 static uint sdma_desct_intr = SDMA_DESC_INTR; 41 module_param_named(desct_intr, sdma_desct_intr, uint, S_IRUGO | S_IWUSR); 42 MODULE_PARM_DESC(desct_intr, "Number of SDMA descriptor before interrupt"); 43 44 #define SDMA_WAIT_BATCH_SIZE 20 45 /* max wait time for a SDMA engine to indicate it has halted */ 46 #define SDMA_ERR_HALT_TIMEOUT 10 /* ms */ 47 /* all SDMA engine errors that cause a halt */ 48 49 #define SD(name) SEND_DMA_##name 50 #define ALL_SDMA_ENG_HALT_ERRS \ 51 (SD(ENG_ERR_STATUS_SDMA_WRONG_DW_ERR_SMASK) \ 52 | SD(ENG_ERR_STATUS_SDMA_GEN_MISMATCH_ERR_SMASK) \ 53 | SD(ENG_ERR_STATUS_SDMA_TOO_LONG_ERR_SMASK) \ 54 | SD(ENG_ERR_STATUS_SDMA_TAIL_OUT_OF_BOUNDS_ERR_SMASK) \ 55 | SD(ENG_ERR_STATUS_SDMA_FIRST_DESC_ERR_SMASK) \ 56 | SD(ENG_ERR_STATUS_SDMA_MEM_READ_ERR_SMASK) \ 57 | SD(ENG_ERR_STATUS_SDMA_HALT_ERR_SMASK) \ 58 | SD(ENG_ERR_STATUS_SDMA_LENGTH_MISMATCH_ERR_SMASK) \ 59 | SD(ENG_ERR_STATUS_SDMA_PACKET_DESC_OVERFLOW_ERR_SMASK) \ 60 | SD(ENG_ERR_STATUS_SDMA_HEADER_SELECT_ERR_SMASK) \ 61 | SD(ENG_ERR_STATUS_SDMA_HEADER_ADDRESS_ERR_SMASK) \ 62 | SD(ENG_ERR_STATUS_SDMA_HEADER_LENGTH_ERR_SMASK) \ 63 | SD(ENG_ERR_STATUS_SDMA_TIMEOUT_ERR_SMASK) \ 64 | SD(ENG_ERR_STATUS_SDMA_DESC_TABLE_UNC_ERR_SMASK) \ 65 | SD(ENG_ERR_STATUS_SDMA_ASSEMBLY_UNC_ERR_SMASK) \ 66 | SD(ENG_ERR_STATUS_SDMA_PACKET_TRACKING_UNC_ERR_SMASK) \ 67 | SD(ENG_ERR_STATUS_SDMA_HEADER_STORAGE_UNC_ERR_SMASK) \ 68 | SD(ENG_ERR_STATUS_SDMA_HEADER_REQUEST_FIFO_UNC_ERR_SMASK)) 69 70 /* sdma_sendctrl operations */ 71 #define SDMA_SENDCTRL_OP_ENABLE BIT(0) 72 #define SDMA_SENDCTRL_OP_INTENABLE BIT(1) 73 #define SDMA_SENDCTRL_OP_HALT BIT(2) 74 #define SDMA_SENDCTRL_OP_CLEANUP BIT(3) 75 76 /* handle long defines */ 77 #define SDMA_EGRESS_PACKET_OCCUPANCY_SMASK \ 78 SEND_EGRESS_SEND_DMA_STATUS_SDMA_EGRESS_PACKET_OCCUPANCY_SMASK 79 #define SDMA_EGRESS_PACKET_OCCUPANCY_SHIFT \ 80 SEND_EGRESS_SEND_DMA_STATUS_SDMA_EGRESS_PACKET_OCCUPANCY_SHIFT 81 82 static const char * const sdma_state_names[] = { 83 [sdma_state_s00_hw_down] = "s00_HwDown", 84 [sdma_state_s10_hw_start_up_halt_wait] = "s10_HwStartUpHaltWait", 85 [sdma_state_s15_hw_start_up_clean_wait] = "s15_HwStartUpCleanWait", 86 [sdma_state_s20_idle] = "s20_Idle", 87 [sdma_state_s30_sw_clean_up_wait] = "s30_SwCleanUpWait", 88 [sdma_state_s40_hw_clean_up_wait] = "s40_HwCleanUpWait", 89 [sdma_state_s50_hw_halt_wait] = "s50_HwHaltWait", 90 [sdma_state_s60_idle_halt_wait] = "s60_IdleHaltWait", 91 [sdma_state_s80_hw_freeze] = "s80_HwFreeze", 92 [sdma_state_s82_freeze_sw_clean] = "s82_FreezeSwClean", 93 [sdma_state_s99_running] = "s99_Running", 94 }; 95 96 #ifdef CONFIG_SDMA_VERBOSITY 97 static const char * const sdma_event_names[] = { 98 [sdma_event_e00_go_hw_down] = "e00_GoHwDown", 99 [sdma_event_e10_go_hw_start] = "e10_GoHwStart", 100 [sdma_event_e15_hw_halt_done] = "e15_HwHaltDone", 101 [sdma_event_e25_hw_clean_up_done] = "e25_HwCleanUpDone", 102 [sdma_event_e30_go_running] = "e30_GoRunning", 103 [sdma_event_e40_sw_cleaned] = "e40_SwCleaned", 104 [sdma_event_e50_hw_cleaned] = "e50_HwCleaned", 105 [sdma_event_e60_hw_halted] = "e60_HwHalted", 106 [sdma_event_e70_go_idle] = "e70_GoIdle", 107 [sdma_event_e80_hw_freeze] = "e80_HwFreeze", 108 [sdma_event_e81_hw_frozen] = "e81_HwFrozen", 109 [sdma_event_e82_hw_unfreeze] = "e82_HwUnfreeze", 110 [sdma_event_e85_link_down] = "e85_LinkDown", 111 [sdma_event_e90_sw_halted] = "e90_SwHalted", 112 }; 113 #endif 114 115 static const struct sdma_set_state_action sdma_action_table[] = { 116 [sdma_state_s00_hw_down] = { 117 .go_s99_running_tofalse = 1, 118 .op_enable = 0, 119 .op_intenable = 0, 120 .op_halt = 0, 121 .op_cleanup = 0, 122 }, 123 [sdma_state_s10_hw_start_up_halt_wait] = { 124 .op_enable = 0, 125 .op_intenable = 0, 126 .op_halt = 1, 127 .op_cleanup = 0, 128 }, 129 [sdma_state_s15_hw_start_up_clean_wait] = { 130 .op_enable = 0, 131 .op_intenable = 1, 132 .op_halt = 0, 133 .op_cleanup = 1, 134 }, 135 [sdma_state_s20_idle] = { 136 .op_enable = 0, 137 .op_intenable = 1, 138 .op_halt = 0, 139 .op_cleanup = 0, 140 }, 141 [sdma_state_s30_sw_clean_up_wait] = { 142 .op_enable = 0, 143 .op_intenable = 0, 144 .op_halt = 0, 145 .op_cleanup = 0, 146 }, 147 [sdma_state_s40_hw_clean_up_wait] = { 148 .op_enable = 0, 149 .op_intenable = 0, 150 .op_halt = 0, 151 .op_cleanup = 1, 152 }, 153 [sdma_state_s50_hw_halt_wait] = { 154 .op_enable = 0, 155 .op_intenable = 0, 156 .op_halt = 0, 157 .op_cleanup = 0, 158 }, 159 [sdma_state_s60_idle_halt_wait] = { 160 .go_s99_running_tofalse = 1, 161 .op_enable = 0, 162 .op_intenable = 0, 163 .op_halt = 1, 164 .op_cleanup = 0, 165 }, 166 [sdma_state_s80_hw_freeze] = { 167 .op_enable = 0, 168 .op_intenable = 0, 169 .op_halt = 0, 170 .op_cleanup = 0, 171 }, 172 [sdma_state_s82_freeze_sw_clean] = { 173 .op_enable = 0, 174 .op_intenable = 0, 175 .op_halt = 0, 176 .op_cleanup = 0, 177 }, 178 [sdma_state_s99_running] = { 179 .op_enable = 1, 180 .op_intenable = 1, 181 .op_halt = 0, 182 .op_cleanup = 0, 183 .go_s99_running_totrue = 1, 184 }, 185 }; 186 187 #define SDMA_TAIL_UPDATE_THRESH 0x1F 188 189 /* declare all statics here rather than keep sorting */ 190 static void sdma_complete(struct kref *); 191 static void sdma_finalput(struct sdma_state *); 192 static void sdma_get(struct sdma_state *); 193 static void sdma_hw_clean_up_task(struct tasklet_struct *); 194 static void sdma_put(struct sdma_state *); 195 static void sdma_set_state(struct sdma_engine *, enum sdma_states); 196 static void sdma_start_hw_clean_up(struct sdma_engine *); 197 static void sdma_sw_clean_up_task(struct tasklet_struct *); 198 static void sdma_sendctrl(struct sdma_engine *, unsigned); 199 static void init_sdma_regs(struct sdma_engine *, u32, uint); 200 static void sdma_process_event( 201 struct sdma_engine *sde, 202 enum sdma_events event); 203 static void __sdma_process_event( 204 struct sdma_engine *sde, 205 enum sdma_events event); 206 static void dump_sdma_state(struct sdma_engine *sde); 207 static void sdma_make_progress(struct sdma_engine *sde, u64 status); 208 static void sdma_desc_avail(struct sdma_engine *sde, uint avail); 209 static void sdma_flush_descq(struct sdma_engine *sde); 210 211 /** 212 * sdma_state_name() - return state string from enum 213 * @state: state 214 */ 215 static const char *sdma_state_name(enum sdma_states state) 216 { 217 return sdma_state_names[state]; 218 } 219 220 static void sdma_get(struct sdma_state *ss) 221 { 222 kref_get(&ss->kref); 223 } 224 225 static void sdma_complete(struct kref *kref) 226 { 227 struct sdma_state *ss = 228 container_of(kref, struct sdma_state, kref); 229 230 complete(&ss->comp); 231 } 232 233 static void sdma_put(struct sdma_state *ss) 234 { 235 kref_put(&ss->kref, sdma_complete); 236 } 237 238 static void sdma_finalput(struct sdma_state *ss) 239 { 240 sdma_put(ss); 241 wait_for_completion(&ss->comp); 242 } 243 244 static inline void write_sde_csr( 245 struct sdma_engine *sde, 246 u32 offset0, 247 u64 value) 248 { 249 write_kctxt_csr(sde->dd, sde->this_idx, offset0, value); 250 } 251 252 static inline u64 read_sde_csr( 253 struct sdma_engine *sde, 254 u32 offset0) 255 { 256 return read_kctxt_csr(sde->dd, sde->this_idx, offset0); 257 } 258 259 /* 260 * sdma_wait_for_packet_egress() - wait for the VL FIFO occupancy for 261 * sdma engine 'sde' to drop to 0. 262 */ 263 static void sdma_wait_for_packet_egress(struct sdma_engine *sde, 264 int pause) 265 { 266 u64 off = 8 * sde->this_idx; 267 struct hfi1_devdata *dd = sde->dd; 268 int lcnt = 0; 269 u64 reg_prev; 270 u64 reg = 0; 271 272 while (1) { 273 reg_prev = reg; 274 reg = read_csr(dd, off + SEND_EGRESS_SEND_DMA_STATUS); 275 276 reg &= SDMA_EGRESS_PACKET_OCCUPANCY_SMASK; 277 reg >>= SDMA_EGRESS_PACKET_OCCUPANCY_SHIFT; 278 if (reg == 0) 279 break; 280 /* counter is reest if accupancy count changes */ 281 if (reg != reg_prev) 282 lcnt = 0; 283 if (lcnt++ > 500) { 284 /* timed out - bounce the link */ 285 dd_dev_err(dd, "%s: engine %u timeout waiting for packets to egress, remaining count %u, bouncing link\n", 286 __func__, sde->this_idx, (u32)reg); 287 queue_work(dd->pport->link_wq, 288 &dd->pport->link_bounce_work); 289 break; 290 } 291 udelay(1); 292 } 293 } 294 295 /* 296 * sdma_wait() - wait for packet egress to complete for all SDMA engines, 297 * and pause for credit return. 298 */ 299 void sdma_wait(struct hfi1_devdata *dd) 300 { 301 int i; 302 303 for (i = 0; i < dd->num_sdma; i++) { 304 struct sdma_engine *sde = &dd->per_sdma[i]; 305 306 sdma_wait_for_packet_egress(sde, 0); 307 } 308 } 309 310 static inline void sdma_set_desc_cnt(struct sdma_engine *sde, unsigned cnt) 311 { 312 u64 reg; 313 314 if (!(sde->dd->flags & HFI1_HAS_SDMA_TIMEOUT)) 315 return; 316 reg = cnt; 317 reg &= SD(DESC_CNT_CNT_MASK); 318 reg <<= SD(DESC_CNT_CNT_SHIFT); 319 write_sde_csr(sde, SD(DESC_CNT), reg); 320 } 321 322 static inline void complete_tx(struct sdma_engine *sde, 323 struct sdma_txreq *tx, 324 int res) 325 { 326 /* protect against complete modifying */ 327 struct iowait *wait = tx->wait; 328 callback_t complete = tx->complete; 329 330 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER 331 trace_hfi1_sdma_out_sn(sde, tx->sn); 332 if (WARN_ON_ONCE(sde->head_sn != tx->sn)) 333 dd_dev_err(sde->dd, "expected %llu got %llu\n", 334 sde->head_sn, tx->sn); 335 sde->head_sn++; 336 #endif 337 __sdma_txclean(sde->dd, tx); 338 if (complete) 339 (*complete)(tx, res); 340 if (iowait_sdma_dec(wait)) 341 iowait_drain_wakeup(wait); 342 } 343 344 /* 345 * Complete all the sdma requests with a SDMA_TXREQ_S_ABORTED status 346 * 347 * Depending on timing there can be txreqs in two places: 348 * - in the descq ring 349 * - in the flush list 350 * 351 * To avoid ordering issues the descq ring needs to be flushed 352 * first followed by the flush list. 353 * 354 * This routine is called from two places 355 * - From a work queue item 356 * - Directly from the state machine just before setting the 357 * state to running 358 * 359 * Must be called with head_lock held 360 * 361 */ 362 static void sdma_flush(struct sdma_engine *sde) 363 { 364 struct sdma_txreq *txp, *txp_next; 365 LIST_HEAD(flushlist); 366 unsigned long flags; 367 uint seq; 368 369 /* flush from head to tail */ 370 sdma_flush_descq(sde); 371 spin_lock_irqsave(&sde->flushlist_lock, flags); 372 /* copy flush list */ 373 list_splice_init(&sde->flushlist, &flushlist); 374 spin_unlock_irqrestore(&sde->flushlist_lock, flags); 375 /* flush from flush list */ 376 list_for_each_entry_safe(txp, txp_next, &flushlist, list) 377 complete_tx(sde, txp, SDMA_TXREQ_S_ABORTED); 378 /* wakeup QPs orphaned on the dmawait list */ 379 do { 380 struct iowait *w, *nw; 381 382 seq = read_seqbegin(&sde->waitlock); 383 if (!list_empty(&sde->dmawait)) { 384 write_seqlock(&sde->waitlock); 385 list_for_each_entry_safe(w, nw, &sde->dmawait, list) { 386 if (w->wakeup) { 387 w->wakeup(w, SDMA_AVAIL_REASON); 388 list_del_init(&w->list); 389 } 390 } 391 write_sequnlock(&sde->waitlock); 392 } 393 } while (read_seqretry(&sde->waitlock, seq)); 394 } 395 396 /* 397 * Fields a work request for flushing the descq ring 398 * and the flush list 399 * 400 * If the engine has been brought to running during 401 * the scheduling delay, the flush is ignored, assuming 402 * that the process of bringing the engine to running 403 * would have done this flush prior to going to running. 404 * 405 */ 406 static void sdma_field_flush(struct work_struct *work) 407 { 408 unsigned long flags; 409 struct sdma_engine *sde = 410 container_of(work, struct sdma_engine, flush_worker); 411 412 write_seqlock_irqsave(&sde->head_lock, flags); 413 if (!__sdma_running(sde)) 414 sdma_flush(sde); 415 write_sequnlock_irqrestore(&sde->head_lock, flags); 416 } 417 418 static void sdma_err_halt_wait(struct work_struct *work) 419 { 420 struct sdma_engine *sde = container_of(work, struct sdma_engine, 421 err_halt_worker); 422 u64 statuscsr; 423 unsigned long timeout; 424 425 timeout = jiffies + msecs_to_jiffies(SDMA_ERR_HALT_TIMEOUT); 426 while (1) { 427 statuscsr = read_sde_csr(sde, SD(STATUS)); 428 statuscsr &= SD(STATUS_ENG_HALTED_SMASK); 429 if (statuscsr) 430 break; 431 if (time_after(jiffies, timeout)) { 432 dd_dev_err(sde->dd, 433 "SDMA engine %d - timeout waiting for engine to halt\n", 434 sde->this_idx); 435 /* 436 * Continue anyway. This could happen if there was 437 * an uncorrectable error in the wrong spot. 438 */ 439 break; 440 } 441 usleep_range(80, 120); 442 } 443 444 sdma_process_event(sde, sdma_event_e15_hw_halt_done); 445 } 446 447 static void sdma_err_progress_check_schedule(struct sdma_engine *sde) 448 { 449 if (!is_bx(sde->dd) && HFI1_CAP_IS_KSET(SDMA_AHG)) { 450 unsigned index; 451 struct hfi1_devdata *dd = sde->dd; 452 453 for (index = 0; index < dd->num_sdma; index++) { 454 struct sdma_engine *curr_sdma = &dd->per_sdma[index]; 455 456 if (curr_sdma != sde) 457 curr_sdma->progress_check_head = 458 curr_sdma->descq_head; 459 } 460 dd_dev_err(sde->dd, 461 "SDMA engine %d - check scheduled\n", 462 sde->this_idx); 463 mod_timer(&sde->err_progress_check_timer, jiffies + 10); 464 } 465 } 466 467 static void sdma_err_progress_check(struct timer_list *t) 468 { 469 unsigned index; 470 struct sdma_engine *sde = timer_container_of(sde, t, 471 err_progress_check_timer); 472 473 dd_dev_err(sde->dd, "SDE progress check event\n"); 474 for (index = 0; index < sde->dd->num_sdma; index++) { 475 struct sdma_engine *curr_sde = &sde->dd->per_sdma[index]; 476 unsigned long flags; 477 478 /* check progress on each engine except the current one */ 479 if (curr_sde == sde) 480 continue; 481 /* 482 * We must lock interrupts when acquiring sde->lock, 483 * to avoid a deadlock if interrupt triggers and spins on 484 * the same lock on same CPU 485 */ 486 spin_lock_irqsave(&curr_sde->tail_lock, flags); 487 write_seqlock(&curr_sde->head_lock); 488 489 /* skip non-running queues */ 490 if (curr_sde->state.current_state != sdma_state_s99_running) { 491 write_sequnlock(&curr_sde->head_lock); 492 spin_unlock_irqrestore(&curr_sde->tail_lock, flags); 493 continue; 494 } 495 496 if ((curr_sde->descq_head != curr_sde->descq_tail) && 497 (curr_sde->descq_head == 498 curr_sde->progress_check_head)) 499 __sdma_process_event(curr_sde, 500 sdma_event_e90_sw_halted); 501 write_sequnlock(&curr_sde->head_lock); 502 spin_unlock_irqrestore(&curr_sde->tail_lock, flags); 503 } 504 schedule_work(&sde->err_halt_worker); 505 } 506 507 static void sdma_hw_clean_up_task(struct tasklet_struct *t) 508 { 509 struct sdma_engine *sde = from_tasklet(sde, t, 510 sdma_hw_clean_up_task); 511 u64 statuscsr; 512 513 while (1) { 514 #ifdef CONFIG_SDMA_VERBOSITY 515 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", 516 sde->this_idx, slashstrip(__FILE__), __LINE__, 517 __func__); 518 #endif 519 statuscsr = read_sde_csr(sde, SD(STATUS)); 520 statuscsr &= SD(STATUS_ENG_CLEANED_UP_SMASK); 521 if (statuscsr) 522 break; 523 udelay(10); 524 } 525 526 sdma_process_event(sde, sdma_event_e25_hw_clean_up_done); 527 } 528 529 static inline struct sdma_txreq *get_txhead(struct sdma_engine *sde) 530 { 531 return sde->tx_ring[sde->tx_head & sde->sdma_mask]; 532 } 533 534 /* 535 * flush ring for recovery 536 */ 537 static void sdma_flush_descq(struct sdma_engine *sde) 538 { 539 u16 head, tail; 540 int progress = 0; 541 struct sdma_txreq *txp = get_txhead(sde); 542 543 /* The reason for some of the complexity of this code is that 544 * not all descriptors have corresponding txps. So, we have to 545 * be able to skip over descs until we wander into the range of 546 * the next txp on the list. 547 */ 548 head = sde->descq_head & sde->sdma_mask; 549 tail = sde->descq_tail & sde->sdma_mask; 550 while (head != tail) { 551 /* advance head, wrap if needed */ 552 head = ++sde->descq_head & sde->sdma_mask; 553 /* if now past this txp's descs, do the callback */ 554 if (txp && txp->next_descq_idx == head) { 555 /* remove from list */ 556 sde->tx_ring[sde->tx_head++ & sde->sdma_mask] = NULL; 557 complete_tx(sde, txp, SDMA_TXREQ_S_ABORTED); 558 trace_hfi1_sdma_progress(sde, head, tail, txp); 559 txp = get_txhead(sde); 560 } 561 progress++; 562 } 563 if (progress) 564 sdma_desc_avail(sde, sdma_descq_freecnt(sde)); 565 } 566 567 static void sdma_sw_clean_up_task(struct tasklet_struct *t) 568 { 569 struct sdma_engine *sde = from_tasklet(sde, t, sdma_sw_clean_up_task); 570 unsigned long flags; 571 572 spin_lock_irqsave(&sde->tail_lock, flags); 573 write_seqlock(&sde->head_lock); 574 575 /* 576 * At this point, the following should always be true: 577 * - We are halted, so no more descriptors are getting retired. 578 * - We are not running, so no one is submitting new work. 579 * - Only we can send the e40_sw_cleaned, so we can't start 580 * running again until we say so. So, the active list and 581 * descq are ours to play with. 582 */ 583 584 /* 585 * In the error clean up sequence, software clean must be called 586 * before the hardware clean so we can use the hardware head in 587 * the progress routine. A hardware clean or SPC unfreeze will 588 * reset the hardware head. 589 * 590 * Process all retired requests. The progress routine will use the 591 * latest physical hardware head - we are not running so speed does 592 * not matter. 593 */ 594 sdma_make_progress(sde, 0); 595 596 sdma_flush(sde); 597 598 /* 599 * Reset our notion of head and tail. 600 * Note that the HW registers have been reset via an earlier 601 * clean up. 602 */ 603 sde->descq_tail = 0; 604 sde->descq_head = 0; 605 sde->desc_avail = sdma_descq_freecnt(sde); 606 *sde->head_dma = 0; 607 608 __sdma_process_event(sde, sdma_event_e40_sw_cleaned); 609 610 write_sequnlock(&sde->head_lock); 611 spin_unlock_irqrestore(&sde->tail_lock, flags); 612 } 613 614 static void sdma_sw_tear_down(struct sdma_engine *sde) 615 { 616 struct sdma_state *ss = &sde->state; 617 618 /* Releasing this reference means the state machine has stopped. */ 619 sdma_put(ss); 620 621 /* stop waiting for all unfreeze events to complete */ 622 atomic_set(&sde->dd->sdma_unfreeze_count, -1); 623 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq); 624 } 625 626 static void sdma_start_hw_clean_up(struct sdma_engine *sde) 627 { 628 tasklet_hi_schedule(&sde->sdma_hw_clean_up_task); 629 } 630 631 static void sdma_set_state(struct sdma_engine *sde, 632 enum sdma_states next_state) 633 { 634 struct sdma_state *ss = &sde->state; 635 const struct sdma_set_state_action *action = sdma_action_table; 636 unsigned op = 0; 637 638 trace_hfi1_sdma_state( 639 sde, 640 sdma_state_names[ss->current_state], 641 sdma_state_names[next_state]); 642 643 /* debugging bookkeeping */ 644 ss->previous_state = ss->current_state; 645 ss->previous_op = ss->current_op; 646 ss->current_state = next_state; 647 648 if (ss->previous_state != sdma_state_s99_running && 649 next_state == sdma_state_s99_running) 650 sdma_flush(sde); 651 652 if (action[next_state].op_enable) 653 op |= SDMA_SENDCTRL_OP_ENABLE; 654 655 if (action[next_state].op_intenable) 656 op |= SDMA_SENDCTRL_OP_INTENABLE; 657 658 if (action[next_state].op_halt) 659 op |= SDMA_SENDCTRL_OP_HALT; 660 661 if (action[next_state].op_cleanup) 662 op |= SDMA_SENDCTRL_OP_CLEANUP; 663 664 if (action[next_state].go_s99_running_tofalse) 665 ss->go_s99_running = 0; 666 667 if (action[next_state].go_s99_running_totrue) 668 ss->go_s99_running = 1; 669 670 ss->current_op = op; 671 sdma_sendctrl(sde, ss->current_op); 672 } 673 674 /** 675 * sdma_get_descq_cnt() - called when device probed 676 * 677 * Return a validated descq count. 678 * 679 * This is currently only used in the verbs initialization to build the tx 680 * list. 681 * 682 * This will probably be deleted in favor of a more scalable approach to 683 * alloc tx's. 684 * 685 */ 686 u16 sdma_get_descq_cnt(void) 687 { 688 u16 count = sdma_descq_cnt; 689 690 if (!count) 691 return SDMA_DESCQ_CNT; 692 /* count must be a power of 2 greater than 64 and less than 693 * 32768. Otherwise return default. 694 */ 695 if (!is_power_of_2(count)) 696 return SDMA_DESCQ_CNT; 697 if (count < 64 || count > 32768) 698 return SDMA_DESCQ_CNT; 699 return count; 700 } 701 702 /** 703 * sdma_engine_get_vl() - return vl for a given sdma engine 704 * @sde: sdma engine 705 * 706 * This function returns the vl mapped to a given engine, or an error if 707 * the mapping can't be found. The mapping fields are protected by RCU. 708 */ 709 int sdma_engine_get_vl(struct sdma_engine *sde) 710 { 711 struct hfi1_devdata *dd = sde->dd; 712 struct sdma_vl_map *m; 713 u8 vl; 714 715 if (sde->this_idx >= TXE_NUM_SDMA_ENGINES) 716 return -EINVAL; 717 718 rcu_read_lock(); 719 m = rcu_dereference(dd->sdma_map); 720 if (unlikely(!m)) { 721 rcu_read_unlock(); 722 return -EINVAL; 723 } 724 vl = m->engine_to_vl[sde->this_idx]; 725 rcu_read_unlock(); 726 727 return vl; 728 } 729 730 /** 731 * sdma_select_engine_vl() - select sdma engine 732 * @dd: devdata 733 * @selector: a spreading factor 734 * @vl: this vl 735 * 736 * 737 * This function returns an engine based on the selector and a vl. The 738 * mapping fields are protected by RCU. 739 */ 740 struct sdma_engine *sdma_select_engine_vl( 741 struct hfi1_devdata *dd, 742 u32 selector, 743 u8 vl) 744 { 745 struct sdma_vl_map *m; 746 struct sdma_map_elem *e; 747 struct sdma_engine *rval; 748 749 /* NOTE This should only happen if SC->VL changed after the initial 750 * checks on the QP/AH 751 * Default will return engine 0 below 752 */ 753 if (vl >= num_vls) { 754 rval = NULL; 755 goto done; 756 } 757 758 rcu_read_lock(); 759 m = rcu_dereference(dd->sdma_map); 760 if (unlikely(!m)) { 761 rcu_read_unlock(); 762 return &dd->per_sdma[0]; 763 } 764 e = m->map[vl & m->mask]; 765 rval = e->sde[selector & e->mask]; 766 rcu_read_unlock(); 767 768 done: 769 rval = !rval ? &dd->per_sdma[0] : rval; 770 trace_hfi1_sdma_engine_select(dd, selector, vl, rval->this_idx); 771 return rval; 772 } 773 774 /** 775 * sdma_select_engine_sc() - select sdma engine 776 * @dd: devdata 777 * @selector: a spreading factor 778 * @sc5: the 5 bit sc 779 * 780 * 781 * This function returns an engine based on the selector and an sc. 782 */ 783 struct sdma_engine *sdma_select_engine_sc( 784 struct hfi1_devdata *dd, 785 u32 selector, 786 u8 sc5) 787 { 788 u8 vl = sc_to_vlt(dd, sc5); 789 790 return sdma_select_engine_vl(dd, selector, vl); 791 } 792 793 struct sdma_rht_map_elem { 794 u32 mask; 795 u8 ctr; 796 struct sdma_engine *sde[]; 797 }; 798 799 struct sdma_rht_node { 800 unsigned long cpu_id; 801 struct sdma_rht_map_elem *map[HFI1_MAX_VLS_SUPPORTED]; 802 struct rhash_head node; 803 }; 804 805 #define NR_CPUS_HINT 192 806 807 static const struct rhashtable_params sdma_rht_params = { 808 .nelem_hint = NR_CPUS_HINT, 809 .head_offset = offsetof(struct sdma_rht_node, node), 810 .key_offset = offsetof(struct sdma_rht_node, cpu_id), 811 .key_len = sizeof_field(struct sdma_rht_node, cpu_id), 812 .max_size = NR_CPUS, 813 .min_size = 8, 814 .automatic_shrinking = true, 815 }; 816 817 /* 818 * sdma_select_user_engine() - select sdma engine based on user setup 819 * @dd: devdata 820 * @selector: a spreading factor 821 * @vl: this vl 822 * 823 * This function returns an sdma engine for a user sdma request. 824 * User defined sdma engine affinity setting is honored when applicable, 825 * otherwise system default sdma engine mapping is used. To ensure correct 826 * ordering, the mapping from <selector, vl> to sde must remain unchanged. 827 */ 828 struct sdma_engine *sdma_select_user_engine(struct hfi1_devdata *dd, 829 u32 selector, u8 vl) 830 { 831 struct sdma_rht_node *rht_node; 832 struct sdma_engine *sde = NULL; 833 unsigned long cpu_id; 834 835 /* 836 * To ensure that always the same sdma engine(s) will be 837 * selected make sure the process is pinned to this CPU only. 838 */ 839 if (current->nr_cpus_allowed != 1) 840 goto out; 841 842 rcu_read_lock(); 843 cpu_id = smp_processor_id(); 844 rht_node = rhashtable_lookup(dd->sdma_rht, &cpu_id, 845 sdma_rht_params); 846 847 if (rht_node && rht_node->map[vl]) { 848 struct sdma_rht_map_elem *map = rht_node->map[vl]; 849 850 sde = map->sde[selector & map->mask]; 851 } 852 rcu_read_unlock(); 853 854 if (sde) 855 return sde; 856 857 out: 858 return sdma_select_engine_vl(dd, selector, vl); 859 } 860 861 static void sdma_populate_sde_map(struct sdma_rht_map_elem *map) 862 { 863 int i; 864 865 for (i = 0; i < roundup_pow_of_two(map->ctr ? : 1) - map->ctr; i++) 866 map->sde[map->ctr + i] = map->sde[i]; 867 } 868 869 static void sdma_cleanup_sde_map(struct sdma_rht_map_elem *map, 870 struct sdma_engine *sde) 871 { 872 unsigned int i, pow; 873 874 /* only need to check the first ctr entries for a match */ 875 for (i = 0; i < map->ctr; i++) { 876 if (map->sde[i] == sde) { 877 memmove(&map->sde[i], &map->sde[i + 1], 878 (map->ctr - i - 1) * sizeof(map->sde[0])); 879 map->ctr--; 880 pow = roundup_pow_of_two(map->ctr ? : 1); 881 map->mask = pow - 1; 882 sdma_populate_sde_map(map); 883 break; 884 } 885 } 886 } 887 888 /* 889 * Prevents concurrent reads and writes of the sdma engine cpu_mask 890 */ 891 static DEFINE_MUTEX(process_to_sde_mutex); 892 893 ssize_t sdma_set_cpu_to_sde_map(struct sdma_engine *sde, const char *buf, 894 size_t count) 895 { 896 struct hfi1_devdata *dd = sde->dd; 897 cpumask_var_t mask, new_mask; 898 unsigned long cpu; 899 int ret, vl, sz; 900 struct sdma_rht_node *rht_node; 901 902 vl = sdma_engine_get_vl(sde); 903 if (unlikely(vl < 0 || vl >= ARRAY_SIZE(rht_node->map))) 904 return -EINVAL; 905 906 ret = zalloc_cpumask_var(&mask, GFP_KERNEL); 907 if (!ret) 908 return -ENOMEM; 909 910 ret = zalloc_cpumask_var(&new_mask, GFP_KERNEL); 911 if (!ret) { 912 free_cpumask_var(mask); 913 return -ENOMEM; 914 } 915 ret = cpulist_parse(buf, mask); 916 if (ret) 917 goto out_free; 918 919 if (!cpumask_subset(mask, cpu_online_mask)) { 920 dd_dev_warn(sde->dd, "Invalid CPU mask\n"); 921 ret = -EINVAL; 922 goto out_free; 923 } 924 925 sz = sizeof(struct sdma_rht_map_elem) + 926 (TXE_NUM_SDMA_ENGINES * sizeof(struct sdma_engine *)); 927 928 mutex_lock(&process_to_sde_mutex); 929 930 for_each_cpu(cpu, mask) { 931 /* Check if we have this already mapped */ 932 if (cpumask_test_cpu(cpu, &sde->cpu_mask)) { 933 cpumask_set_cpu(cpu, new_mask); 934 continue; 935 } 936 937 rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpu, 938 sdma_rht_params); 939 if (!rht_node) { 940 rht_node = kzalloc(sizeof(*rht_node), GFP_KERNEL); 941 if (!rht_node) { 942 ret = -ENOMEM; 943 goto out; 944 } 945 946 rht_node->map[vl] = kzalloc(sz, GFP_KERNEL); 947 if (!rht_node->map[vl]) { 948 kfree(rht_node); 949 ret = -ENOMEM; 950 goto out; 951 } 952 rht_node->cpu_id = cpu; 953 rht_node->map[vl]->mask = 0; 954 rht_node->map[vl]->ctr = 1; 955 rht_node->map[vl]->sde[0] = sde; 956 957 ret = rhashtable_insert_fast(dd->sdma_rht, 958 &rht_node->node, 959 sdma_rht_params); 960 if (ret) { 961 kfree(rht_node->map[vl]); 962 kfree(rht_node); 963 dd_dev_err(sde->dd, "Failed to set process to sde affinity for cpu %lu\n", 964 cpu); 965 goto out; 966 } 967 968 } else { 969 int ctr, pow; 970 971 /* Add new user mappings */ 972 if (!rht_node->map[vl]) 973 rht_node->map[vl] = kzalloc(sz, GFP_KERNEL); 974 975 if (!rht_node->map[vl]) { 976 ret = -ENOMEM; 977 goto out; 978 } 979 980 rht_node->map[vl]->ctr++; 981 ctr = rht_node->map[vl]->ctr; 982 rht_node->map[vl]->sde[ctr - 1] = sde; 983 pow = roundup_pow_of_two(ctr); 984 rht_node->map[vl]->mask = pow - 1; 985 986 /* Populate the sde map table */ 987 sdma_populate_sde_map(rht_node->map[vl]); 988 } 989 cpumask_set_cpu(cpu, new_mask); 990 } 991 992 /* Clean up old mappings */ 993 for_each_cpu(cpu, cpu_online_mask) { 994 struct sdma_rht_node *rht_node; 995 996 /* Don't cleanup sdes that are set in the new mask */ 997 if (cpumask_test_cpu(cpu, mask)) 998 continue; 999 1000 rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpu, 1001 sdma_rht_params); 1002 if (rht_node) { 1003 bool empty = true; 1004 int i; 1005 1006 /* Remove mappings for old sde */ 1007 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) 1008 if (rht_node->map[i]) 1009 sdma_cleanup_sde_map(rht_node->map[i], 1010 sde); 1011 1012 /* Free empty hash table entries */ 1013 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) { 1014 if (!rht_node->map[i]) 1015 continue; 1016 1017 if (rht_node->map[i]->ctr) { 1018 empty = false; 1019 break; 1020 } 1021 } 1022 1023 if (empty) { 1024 ret = rhashtable_remove_fast(dd->sdma_rht, 1025 &rht_node->node, 1026 sdma_rht_params); 1027 WARN_ON(ret); 1028 1029 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) 1030 kfree(rht_node->map[i]); 1031 1032 kfree(rht_node); 1033 } 1034 } 1035 } 1036 1037 cpumask_copy(&sde->cpu_mask, new_mask); 1038 out: 1039 mutex_unlock(&process_to_sde_mutex); 1040 out_free: 1041 free_cpumask_var(mask); 1042 free_cpumask_var(new_mask); 1043 return ret ? : strnlen(buf, PAGE_SIZE); 1044 } 1045 1046 ssize_t sdma_get_cpu_to_sde_map(struct sdma_engine *sde, char *buf) 1047 { 1048 mutex_lock(&process_to_sde_mutex); 1049 if (cpumask_empty(&sde->cpu_mask)) 1050 snprintf(buf, PAGE_SIZE, "%s\n", "empty"); 1051 else 1052 cpumap_print_to_pagebuf(true, buf, &sde->cpu_mask); 1053 mutex_unlock(&process_to_sde_mutex); 1054 return strnlen(buf, PAGE_SIZE); 1055 } 1056 1057 static void sdma_rht_free(void *ptr, void *arg) 1058 { 1059 struct sdma_rht_node *rht_node = ptr; 1060 int i; 1061 1062 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) 1063 kfree(rht_node->map[i]); 1064 1065 kfree(rht_node); 1066 } 1067 1068 /** 1069 * sdma_seqfile_dump_cpu_list() - debugfs dump the cpu to sdma mappings 1070 * @s: seq file 1071 * @dd: hfi1_devdata 1072 * @cpuid: cpu id 1073 * 1074 * This routine dumps the process to sde mappings per cpu 1075 */ 1076 void sdma_seqfile_dump_cpu_list(struct seq_file *s, 1077 struct hfi1_devdata *dd, 1078 unsigned long cpuid) 1079 { 1080 struct sdma_rht_node *rht_node; 1081 int i, j; 1082 1083 rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpuid, 1084 sdma_rht_params); 1085 if (!rht_node) 1086 return; 1087 1088 seq_printf(s, "cpu%3lu: ", cpuid); 1089 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) { 1090 if (!rht_node->map[i] || !rht_node->map[i]->ctr) 1091 continue; 1092 1093 seq_printf(s, " vl%d: [", i); 1094 1095 for (j = 0; j < rht_node->map[i]->ctr; j++) { 1096 if (!rht_node->map[i]->sde[j]) 1097 continue; 1098 1099 if (j > 0) 1100 seq_puts(s, ","); 1101 1102 seq_printf(s, " sdma%2d", 1103 rht_node->map[i]->sde[j]->this_idx); 1104 } 1105 seq_puts(s, " ]"); 1106 } 1107 1108 seq_puts(s, "\n"); 1109 } 1110 1111 /* 1112 * Free the indicated map struct 1113 */ 1114 static void sdma_map_free(struct sdma_vl_map *m) 1115 { 1116 int i; 1117 1118 for (i = 0; m && i < m->actual_vls; i++) 1119 kfree(m->map[i]); 1120 kfree(m); 1121 } 1122 1123 /* 1124 * Handle RCU callback 1125 */ 1126 static void sdma_map_rcu_callback(struct rcu_head *list) 1127 { 1128 struct sdma_vl_map *m = container_of(list, struct sdma_vl_map, list); 1129 1130 sdma_map_free(m); 1131 } 1132 1133 /** 1134 * sdma_map_init - called when # vls change 1135 * @dd: hfi1_devdata 1136 * @port: port number 1137 * @num_vls: number of vls 1138 * @vl_engines: per vl engine mapping (optional) 1139 * 1140 * This routine changes the mapping based on the number of vls. 1141 * 1142 * vl_engines is used to specify a non-uniform vl/engine loading. NULL 1143 * implies auto computing the loading and giving each VLs a uniform 1144 * distribution of engines per VL. 1145 * 1146 * The auto algorithm computes the sde_per_vl and the number of extra 1147 * engines. Any extra engines are added from the last VL on down. 1148 * 1149 * rcu locking is used here to control access to the mapping fields. 1150 * 1151 * If either the num_vls or num_sdma are non-power of 2, the array sizes 1152 * in the struct sdma_vl_map and the struct sdma_map_elem are rounded 1153 * up to the next highest power of 2 and the first entry is reused 1154 * in a round robin fashion. 1155 * 1156 * If an error occurs the map change is not done and the mapping is 1157 * not changed. 1158 * 1159 */ 1160 int sdma_map_init(struct hfi1_devdata *dd, u8 port, u8 num_vls, u8 *vl_engines) 1161 { 1162 int i, j; 1163 int extra, sde_per_vl; 1164 int engine = 0; 1165 u8 lvl_engines[OPA_MAX_VLS]; 1166 struct sdma_vl_map *oldmap, *newmap; 1167 1168 if (!(dd->flags & HFI1_HAS_SEND_DMA)) 1169 return 0; 1170 1171 if (!vl_engines) { 1172 /* truncate divide */ 1173 sde_per_vl = dd->num_sdma / num_vls; 1174 /* extras */ 1175 extra = dd->num_sdma % num_vls; 1176 vl_engines = lvl_engines; 1177 /* add extras from last vl down */ 1178 for (i = num_vls - 1; i >= 0; i--, extra--) 1179 vl_engines[i] = sde_per_vl + (extra > 0 ? 1 : 0); 1180 } 1181 /* build new map */ 1182 newmap = kzalloc( 1183 sizeof(struct sdma_vl_map) + 1184 roundup_pow_of_two(num_vls) * 1185 sizeof(struct sdma_map_elem *), 1186 GFP_KERNEL); 1187 if (!newmap) 1188 goto bail; 1189 newmap->actual_vls = num_vls; 1190 newmap->vls = roundup_pow_of_two(num_vls); 1191 newmap->mask = (1 << ilog2(newmap->vls)) - 1; 1192 /* initialize back-map */ 1193 for (i = 0; i < TXE_NUM_SDMA_ENGINES; i++) 1194 newmap->engine_to_vl[i] = -1; 1195 for (i = 0; i < newmap->vls; i++) { 1196 /* save for wrap around */ 1197 int first_engine = engine; 1198 1199 if (i < newmap->actual_vls) { 1200 int sz = roundup_pow_of_two(vl_engines[i]); 1201 1202 /* only allocate once */ 1203 newmap->map[i] = kzalloc( 1204 sizeof(struct sdma_map_elem) + 1205 sz * sizeof(struct sdma_engine *), 1206 GFP_KERNEL); 1207 if (!newmap->map[i]) 1208 goto bail; 1209 newmap->map[i]->mask = (1 << ilog2(sz)) - 1; 1210 /* assign engines */ 1211 for (j = 0; j < sz; j++) { 1212 newmap->map[i]->sde[j] = 1213 &dd->per_sdma[engine]; 1214 if (++engine >= first_engine + vl_engines[i]) 1215 /* wrap back to first engine */ 1216 engine = first_engine; 1217 } 1218 /* assign back-map */ 1219 for (j = 0; j < vl_engines[i]; j++) 1220 newmap->engine_to_vl[first_engine + j] = i; 1221 } else { 1222 /* just re-use entry without allocating */ 1223 newmap->map[i] = newmap->map[i % num_vls]; 1224 } 1225 engine = first_engine + vl_engines[i]; 1226 } 1227 /* newmap in hand, save old map */ 1228 spin_lock_irq(&dd->sde_map_lock); 1229 oldmap = rcu_dereference_protected(dd->sdma_map, 1230 lockdep_is_held(&dd->sde_map_lock)); 1231 1232 /* publish newmap */ 1233 rcu_assign_pointer(dd->sdma_map, newmap); 1234 1235 spin_unlock_irq(&dd->sde_map_lock); 1236 /* success, free any old map after grace period */ 1237 if (oldmap) 1238 call_rcu(&oldmap->list, sdma_map_rcu_callback); 1239 return 0; 1240 bail: 1241 /* free any partial allocation */ 1242 sdma_map_free(newmap); 1243 return -ENOMEM; 1244 } 1245 1246 /** 1247 * sdma_clean - Clean up allocated memory 1248 * @dd: struct hfi1_devdata 1249 * @num_engines: num sdma engines 1250 * 1251 * This routine can be called regardless of the success of 1252 * sdma_init() 1253 */ 1254 void sdma_clean(struct hfi1_devdata *dd, size_t num_engines) 1255 { 1256 size_t i; 1257 struct sdma_engine *sde; 1258 1259 if (dd->sdma_pad_dma) { 1260 dma_free_coherent(&dd->pcidev->dev, SDMA_PAD, 1261 (void *)dd->sdma_pad_dma, 1262 dd->sdma_pad_phys); 1263 dd->sdma_pad_dma = NULL; 1264 dd->sdma_pad_phys = 0; 1265 } 1266 if (dd->sdma_heads_dma) { 1267 dma_free_coherent(&dd->pcidev->dev, dd->sdma_heads_size, 1268 (void *)dd->sdma_heads_dma, 1269 dd->sdma_heads_phys); 1270 dd->sdma_heads_dma = NULL; 1271 dd->sdma_heads_phys = 0; 1272 } 1273 for (i = 0; dd->per_sdma && i < num_engines; ++i) { 1274 sde = &dd->per_sdma[i]; 1275 1276 sde->head_dma = NULL; 1277 sde->head_phys = 0; 1278 1279 if (sde->descq) { 1280 dma_free_coherent( 1281 &dd->pcidev->dev, 1282 sde->descq_cnt * sizeof(u64[2]), 1283 sde->descq, 1284 sde->descq_phys 1285 ); 1286 sde->descq = NULL; 1287 sde->descq_phys = 0; 1288 } 1289 kvfree(sde->tx_ring); 1290 sde->tx_ring = NULL; 1291 } 1292 if (rcu_access_pointer(dd->sdma_map)) { 1293 spin_lock_irq(&dd->sde_map_lock); 1294 sdma_map_free(rcu_access_pointer(dd->sdma_map)); 1295 RCU_INIT_POINTER(dd->sdma_map, NULL); 1296 spin_unlock_irq(&dd->sde_map_lock); 1297 synchronize_rcu(); 1298 } 1299 kfree(dd->per_sdma); 1300 dd->per_sdma = NULL; 1301 1302 if (dd->sdma_rht) { 1303 rhashtable_free_and_destroy(dd->sdma_rht, sdma_rht_free, NULL); 1304 kfree(dd->sdma_rht); 1305 dd->sdma_rht = NULL; 1306 } 1307 } 1308 1309 /** 1310 * sdma_init() - called when device probed 1311 * @dd: hfi1_devdata 1312 * @port: port number (currently only zero) 1313 * 1314 * Initializes each sde and its csrs. 1315 * Interrupts are not required to be enabled. 1316 * 1317 * Returns: 1318 * 0 - success, -errno on failure 1319 */ 1320 int sdma_init(struct hfi1_devdata *dd, u8 port) 1321 { 1322 unsigned this_idx; 1323 struct sdma_engine *sde; 1324 struct rhashtable *tmp_sdma_rht; 1325 u16 descq_cnt; 1326 void *curr_head; 1327 struct hfi1_pportdata *ppd = dd->pport + port; 1328 u32 per_sdma_credits; 1329 uint idle_cnt = sdma_idle_cnt; 1330 size_t num_engines = chip_sdma_engines(dd); 1331 int ret = -ENOMEM; 1332 1333 if (!HFI1_CAP_IS_KSET(SDMA)) { 1334 HFI1_CAP_CLEAR(SDMA_AHG); 1335 return 0; 1336 } 1337 if (mod_num_sdma && 1338 /* can't exceed chip support */ 1339 mod_num_sdma <= chip_sdma_engines(dd) && 1340 /* count must be >= vls */ 1341 mod_num_sdma >= num_vls) 1342 num_engines = mod_num_sdma; 1343 1344 dd_dev_info(dd, "SDMA mod_num_sdma: %u\n", mod_num_sdma); 1345 dd_dev_info(dd, "SDMA chip_sdma_engines: %u\n", chip_sdma_engines(dd)); 1346 dd_dev_info(dd, "SDMA chip_sdma_mem_size: %u\n", 1347 chip_sdma_mem_size(dd)); 1348 1349 per_sdma_credits = 1350 chip_sdma_mem_size(dd) / (num_engines * SDMA_BLOCK_SIZE); 1351 1352 /* set up freeze waitqueue */ 1353 init_waitqueue_head(&dd->sdma_unfreeze_wq); 1354 atomic_set(&dd->sdma_unfreeze_count, 0); 1355 1356 descq_cnt = sdma_get_descq_cnt(); 1357 dd_dev_info(dd, "SDMA engines %zu descq_cnt %u\n", 1358 num_engines, descq_cnt); 1359 1360 /* alloc memory for array of send engines */ 1361 dd->per_sdma = kcalloc_node(num_engines, sizeof(*dd->per_sdma), 1362 GFP_KERNEL, dd->node); 1363 if (!dd->per_sdma) 1364 return ret; 1365 1366 idle_cnt = ns_to_cclock(dd, idle_cnt); 1367 if (idle_cnt) 1368 dd->default_desc1 = 1369 SDMA_DESC1_HEAD_TO_HOST_FLAG; 1370 else 1371 dd->default_desc1 = 1372 SDMA_DESC1_INT_REQ_FLAG; 1373 1374 if (!sdma_desct_intr) 1375 sdma_desct_intr = SDMA_DESC_INTR; 1376 1377 /* Allocate memory for SendDMA descriptor FIFOs */ 1378 for (this_idx = 0; this_idx < num_engines; ++this_idx) { 1379 sde = &dd->per_sdma[this_idx]; 1380 sde->dd = dd; 1381 sde->ppd = ppd; 1382 sde->this_idx = this_idx; 1383 sde->descq_cnt = descq_cnt; 1384 sde->desc_avail = sdma_descq_freecnt(sde); 1385 sde->sdma_shift = ilog2(descq_cnt); 1386 sde->sdma_mask = (1 << sde->sdma_shift) - 1; 1387 1388 /* Create a mask specifically for each interrupt source */ 1389 sde->int_mask = (u64)1 << (0 * TXE_NUM_SDMA_ENGINES + 1390 this_idx); 1391 sde->progress_mask = (u64)1 << (1 * TXE_NUM_SDMA_ENGINES + 1392 this_idx); 1393 sde->idle_mask = (u64)1 << (2 * TXE_NUM_SDMA_ENGINES + 1394 this_idx); 1395 /* Create a combined mask to cover all 3 interrupt sources */ 1396 sde->imask = sde->int_mask | sde->progress_mask | 1397 sde->idle_mask; 1398 1399 spin_lock_init(&sde->tail_lock); 1400 seqlock_init(&sde->head_lock); 1401 spin_lock_init(&sde->senddmactrl_lock); 1402 spin_lock_init(&sde->flushlist_lock); 1403 seqlock_init(&sde->waitlock); 1404 /* insure there is always a zero bit */ 1405 sde->ahg_bits = 0xfffffffe00000000ULL; 1406 1407 sdma_set_state(sde, sdma_state_s00_hw_down); 1408 1409 /* set up reference counting */ 1410 kref_init(&sde->state.kref); 1411 init_completion(&sde->state.comp); 1412 1413 INIT_LIST_HEAD(&sde->flushlist); 1414 INIT_LIST_HEAD(&sde->dmawait); 1415 1416 sde->tail_csr = 1417 get_kctxt_csr_addr(dd, this_idx, SD(TAIL)); 1418 1419 tasklet_setup(&sde->sdma_hw_clean_up_task, 1420 sdma_hw_clean_up_task); 1421 tasklet_setup(&sde->sdma_sw_clean_up_task, 1422 sdma_sw_clean_up_task); 1423 INIT_WORK(&sde->err_halt_worker, sdma_err_halt_wait); 1424 INIT_WORK(&sde->flush_worker, sdma_field_flush); 1425 1426 sde->progress_check_head = 0; 1427 1428 timer_setup(&sde->err_progress_check_timer, 1429 sdma_err_progress_check, 0); 1430 1431 sde->descq = dma_alloc_coherent(&dd->pcidev->dev, 1432 descq_cnt * sizeof(u64[2]), 1433 &sde->descq_phys, GFP_KERNEL); 1434 if (!sde->descq) 1435 goto bail; 1436 sde->tx_ring = 1437 kvzalloc_node(array_size(descq_cnt, 1438 sizeof(struct sdma_txreq *)), 1439 GFP_KERNEL, dd->node); 1440 if (!sde->tx_ring) 1441 goto bail; 1442 } 1443 1444 dd->sdma_heads_size = L1_CACHE_BYTES * num_engines; 1445 /* Allocate memory for DMA of head registers to memory */ 1446 dd->sdma_heads_dma = dma_alloc_coherent(&dd->pcidev->dev, 1447 dd->sdma_heads_size, 1448 &dd->sdma_heads_phys, 1449 GFP_KERNEL); 1450 if (!dd->sdma_heads_dma) { 1451 dd_dev_err(dd, "failed to allocate SendDMA head memory\n"); 1452 goto bail; 1453 } 1454 1455 /* Allocate memory for pad */ 1456 dd->sdma_pad_dma = dma_alloc_coherent(&dd->pcidev->dev, SDMA_PAD, 1457 &dd->sdma_pad_phys, GFP_KERNEL); 1458 if (!dd->sdma_pad_dma) { 1459 dd_dev_err(dd, "failed to allocate SendDMA pad memory\n"); 1460 goto bail; 1461 } 1462 1463 /* assign each engine to different cacheline and init registers */ 1464 curr_head = (void *)dd->sdma_heads_dma; 1465 for (this_idx = 0; this_idx < num_engines; ++this_idx) { 1466 unsigned long phys_offset; 1467 1468 sde = &dd->per_sdma[this_idx]; 1469 1470 sde->head_dma = curr_head; 1471 curr_head += L1_CACHE_BYTES; 1472 phys_offset = (unsigned long)sde->head_dma - 1473 (unsigned long)dd->sdma_heads_dma; 1474 sde->head_phys = dd->sdma_heads_phys + phys_offset; 1475 init_sdma_regs(sde, per_sdma_credits, idle_cnt); 1476 } 1477 dd->flags |= HFI1_HAS_SEND_DMA; 1478 dd->flags |= idle_cnt ? HFI1_HAS_SDMA_TIMEOUT : 0; 1479 dd->num_sdma = num_engines; 1480 ret = sdma_map_init(dd, port, ppd->vls_operational, NULL); 1481 if (ret < 0) 1482 goto bail; 1483 1484 tmp_sdma_rht = kzalloc(sizeof(*tmp_sdma_rht), GFP_KERNEL); 1485 if (!tmp_sdma_rht) { 1486 ret = -ENOMEM; 1487 goto bail; 1488 } 1489 1490 ret = rhashtable_init(tmp_sdma_rht, &sdma_rht_params); 1491 if (ret < 0) { 1492 kfree(tmp_sdma_rht); 1493 goto bail; 1494 } 1495 1496 dd->sdma_rht = tmp_sdma_rht; 1497 1498 dd_dev_info(dd, "SDMA num_sdma: %u\n", dd->num_sdma); 1499 return 0; 1500 1501 bail: 1502 sdma_clean(dd, num_engines); 1503 return ret; 1504 } 1505 1506 /** 1507 * sdma_all_running() - called when the link goes up 1508 * @dd: hfi1_devdata 1509 * 1510 * This routine moves all engines to the running state. 1511 */ 1512 void sdma_all_running(struct hfi1_devdata *dd) 1513 { 1514 struct sdma_engine *sde; 1515 unsigned int i; 1516 1517 /* move all engines to running */ 1518 for (i = 0; i < dd->num_sdma; ++i) { 1519 sde = &dd->per_sdma[i]; 1520 sdma_process_event(sde, sdma_event_e30_go_running); 1521 } 1522 } 1523 1524 /** 1525 * sdma_start() - called to kick off state processing for all engines 1526 * @dd: hfi1_devdata 1527 * 1528 * This routine is for kicking off the state processing for all required 1529 * sdma engines. Interrupts need to be working at this point. 1530 * 1531 */ 1532 void sdma_start(struct hfi1_devdata *dd) 1533 { 1534 unsigned i; 1535 struct sdma_engine *sde; 1536 1537 /* kick off the engines state processing */ 1538 for (i = 0; i < dd->num_sdma; ++i) { 1539 sde = &dd->per_sdma[i]; 1540 sdma_process_event(sde, sdma_event_e10_go_hw_start); 1541 } 1542 } 1543 1544 /** 1545 * sdma_exit() - used when module is removed 1546 * @dd: hfi1_devdata 1547 */ 1548 void sdma_exit(struct hfi1_devdata *dd) 1549 { 1550 unsigned this_idx; 1551 struct sdma_engine *sde; 1552 1553 for (this_idx = 0; dd->per_sdma && this_idx < dd->num_sdma; 1554 ++this_idx) { 1555 sde = &dd->per_sdma[this_idx]; 1556 if (!list_empty(&sde->dmawait)) 1557 dd_dev_err(dd, "sde %u: dmawait list not empty!\n", 1558 sde->this_idx); 1559 sdma_process_event(sde, sdma_event_e00_go_hw_down); 1560 1561 timer_delete_sync(&sde->err_progress_check_timer); 1562 1563 /* 1564 * This waits for the state machine to exit so it is not 1565 * necessary to kill the sdma_sw_clean_up_task to make sure 1566 * it is not running. 1567 */ 1568 sdma_finalput(&sde->state); 1569 } 1570 } 1571 1572 /* 1573 * unmap the indicated descriptor 1574 */ 1575 static inline void sdma_unmap_desc( 1576 struct hfi1_devdata *dd, 1577 struct sdma_desc *descp) 1578 { 1579 switch (sdma_mapping_type(descp)) { 1580 case SDMA_MAP_SINGLE: 1581 dma_unmap_single(&dd->pcidev->dev, sdma_mapping_addr(descp), 1582 sdma_mapping_len(descp), DMA_TO_DEVICE); 1583 break; 1584 case SDMA_MAP_PAGE: 1585 dma_unmap_page(&dd->pcidev->dev, sdma_mapping_addr(descp), 1586 sdma_mapping_len(descp), DMA_TO_DEVICE); 1587 break; 1588 } 1589 1590 if (descp->pinning_ctx && descp->ctx_put) 1591 descp->ctx_put(descp->pinning_ctx); 1592 descp->pinning_ctx = NULL; 1593 } 1594 1595 /* 1596 * return the mode as indicated by the first 1597 * descriptor in the tx. 1598 */ 1599 static inline u8 ahg_mode(struct sdma_txreq *tx) 1600 { 1601 return (tx->descp[0].qw[1] & SDMA_DESC1_HEADER_MODE_SMASK) 1602 >> SDMA_DESC1_HEADER_MODE_SHIFT; 1603 } 1604 1605 /** 1606 * __sdma_txclean() - clean tx of mappings, descp *kmalloc's 1607 * @dd: hfi1_devdata for unmapping 1608 * @tx: tx request to clean 1609 * 1610 * This is used in the progress routine to clean the tx or 1611 * by the ULP to toss an in-process tx build. 1612 * 1613 * The code can be called multiple times without issue. 1614 * 1615 */ 1616 void __sdma_txclean( 1617 struct hfi1_devdata *dd, 1618 struct sdma_txreq *tx) 1619 { 1620 u16 i; 1621 1622 if (tx->num_desc) { 1623 u8 skip = 0, mode = ahg_mode(tx); 1624 1625 /* unmap first */ 1626 sdma_unmap_desc(dd, &tx->descp[0]); 1627 /* determine number of AHG descriptors to skip */ 1628 if (mode > SDMA_AHG_APPLY_UPDATE1) 1629 skip = mode >> 1; 1630 for (i = 1 + skip; i < tx->num_desc; i++) 1631 sdma_unmap_desc(dd, &tx->descp[i]); 1632 tx->num_desc = 0; 1633 } 1634 kfree(tx->coalesce_buf); 1635 tx->coalesce_buf = NULL; 1636 /* kmalloc'ed descp */ 1637 if (unlikely(tx->desc_limit > ARRAY_SIZE(tx->descs))) { 1638 tx->desc_limit = ARRAY_SIZE(tx->descs); 1639 kfree(tx->descp); 1640 } 1641 } 1642 1643 static inline u16 sdma_gethead(struct sdma_engine *sde) 1644 { 1645 struct hfi1_devdata *dd = sde->dd; 1646 int use_dmahead; 1647 u16 hwhead; 1648 1649 #ifdef CONFIG_SDMA_VERBOSITY 1650 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", 1651 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__); 1652 #endif 1653 1654 retry: 1655 use_dmahead = HFI1_CAP_IS_KSET(USE_SDMA_HEAD) && __sdma_running(sde) && 1656 (dd->flags & HFI1_HAS_SDMA_TIMEOUT); 1657 hwhead = use_dmahead ? 1658 (u16)le64_to_cpu(*sde->head_dma) : 1659 (u16)read_sde_csr(sde, SD(HEAD)); 1660 1661 if (unlikely(HFI1_CAP_IS_KSET(SDMA_HEAD_CHECK))) { 1662 u16 cnt; 1663 u16 swtail; 1664 u16 swhead; 1665 int sane; 1666 1667 swhead = sde->descq_head & sde->sdma_mask; 1668 /* this code is really bad for cache line trading */ 1669 swtail = READ_ONCE(sde->descq_tail) & sde->sdma_mask; 1670 cnt = sde->descq_cnt; 1671 1672 if (swhead < swtail) 1673 /* not wrapped */ 1674 sane = (hwhead >= swhead) & (hwhead <= swtail); 1675 else if (swhead > swtail) 1676 /* wrapped around */ 1677 sane = ((hwhead >= swhead) && (hwhead < cnt)) || 1678 (hwhead <= swtail); 1679 else 1680 /* empty */ 1681 sane = (hwhead == swhead); 1682 1683 if (unlikely(!sane)) { 1684 dd_dev_err(dd, "SDMA(%u) bad head (%s) hwhd=%u swhd=%u swtl=%u cnt=%u\n", 1685 sde->this_idx, 1686 use_dmahead ? "dma" : "kreg", 1687 hwhead, swhead, swtail, cnt); 1688 if (use_dmahead) { 1689 /* try one more time, using csr */ 1690 use_dmahead = 0; 1691 goto retry; 1692 } 1693 /* proceed as if no progress */ 1694 hwhead = swhead; 1695 } 1696 } 1697 return hwhead; 1698 } 1699 1700 /* 1701 * This is called when there are send DMA descriptors that might be 1702 * available. 1703 * 1704 * This is called with head_lock held. 1705 */ 1706 static void sdma_desc_avail(struct sdma_engine *sde, uint avail) 1707 { 1708 struct iowait *wait, *nw, *twait; 1709 struct iowait *waits[SDMA_WAIT_BATCH_SIZE]; 1710 uint i, n = 0, seq, tidx = 0; 1711 1712 #ifdef CONFIG_SDMA_VERBOSITY 1713 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx, 1714 slashstrip(__FILE__), __LINE__, __func__); 1715 dd_dev_err(sde->dd, "avail: %u\n", avail); 1716 #endif 1717 1718 do { 1719 seq = read_seqbegin(&sde->waitlock); 1720 if (!list_empty(&sde->dmawait)) { 1721 /* at least one item */ 1722 write_seqlock(&sde->waitlock); 1723 /* Harvest waiters wanting DMA descriptors */ 1724 list_for_each_entry_safe( 1725 wait, 1726 nw, 1727 &sde->dmawait, 1728 list) { 1729 u32 num_desc; 1730 1731 if (!wait->wakeup) 1732 continue; 1733 if (n == ARRAY_SIZE(waits)) 1734 break; 1735 iowait_init_priority(wait); 1736 num_desc = iowait_get_all_desc(wait); 1737 if (num_desc > avail) 1738 break; 1739 avail -= num_desc; 1740 /* Find the top-priority wait memeber */ 1741 if (n) { 1742 twait = waits[tidx]; 1743 tidx = 1744 iowait_priority_update_top(wait, 1745 twait, 1746 n, 1747 tidx); 1748 } 1749 list_del_init(&wait->list); 1750 waits[n++] = wait; 1751 } 1752 write_sequnlock(&sde->waitlock); 1753 break; 1754 } 1755 } while (read_seqretry(&sde->waitlock, seq)); 1756 1757 /* Schedule the top-priority entry first */ 1758 if (n) 1759 waits[tidx]->wakeup(waits[tidx], SDMA_AVAIL_REASON); 1760 1761 for (i = 0; i < n; i++) 1762 if (i != tidx) 1763 waits[i]->wakeup(waits[i], SDMA_AVAIL_REASON); 1764 } 1765 1766 /* head_lock must be held */ 1767 static void sdma_make_progress(struct sdma_engine *sde, u64 status) 1768 { 1769 struct sdma_txreq *txp = NULL; 1770 int progress = 0; 1771 u16 hwhead, swhead; 1772 int idle_check_done = 0; 1773 1774 hwhead = sdma_gethead(sde); 1775 1776 /* The reason for some of the complexity of this code is that 1777 * not all descriptors have corresponding txps. So, we have to 1778 * be able to skip over descs until we wander into the range of 1779 * the next txp on the list. 1780 */ 1781 1782 retry: 1783 txp = get_txhead(sde); 1784 swhead = sde->descq_head & sde->sdma_mask; 1785 trace_hfi1_sdma_progress(sde, hwhead, swhead, txp); 1786 while (swhead != hwhead) { 1787 /* advance head, wrap if needed */ 1788 swhead = ++sde->descq_head & sde->sdma_mask; 1789 1790 /* if now past this txp's descs, do the callback */ 1791 if (txp && txp->next_descq_idx == swhead) { 1792 /* remove from list */ 1793 sde->tx_ring[sde->tx_head++ & sde->sdma_mask] = NULL; 1794 complete_tx(sde, txp, SDMA_TXREQ_S_OK); 1795 /* see if there is another txp */ 1796 txp = get_txhead(sde); 1797 } 1798 trace_hfi1_sdma_progress(sde, hwhead, swhead, txp); 1799 progress++; 1800 } 1801 1802 /* 1803 * The SDMA idle interrupt is not guaranteed to be ordered with respect 1804 * to updates to the dma_head location in host memory. The head 1805 * value read might not be fully up to date. If there are pending 1806 * descriptors and the SDMA idle interrupt fired then read from the 1807 * CSR SDMA head instead to get the latest value from the hardware. 1808 * The hardware SDMA head should be read at most once in this invocation 1809 * of sdma_make_progress(..) which is ensured by idle_check_done flag 1810 */ 1811 if ((status & sde->idle_mask) && !idle_check_done) { 1812 u16 swtail; 1813 1814 swtail = READ_ONCE(sde->descq_tail) & sde->sdma_mask; 1815 if (swtail != hwhead) { 1816 hwhead = (u16)read_sde_csr(sde, SD(HEAD)); 1817 idle_check_done = 1; 1818 goto retry; 1819 } 1820 } 1821 1822 sde->last_status = status; 1823 if (progress) 1824 sdma_desc_avail(sde, sdma_descq_freecnt(sde)); 1825 } 1826 1827 /* 1828 * sdma_engine_interrupt() - interrupt handler for engine 1829 * @sde: sdma engine 1830 * @status: sdma interrupt reason 1831 * 1832 * Status is a mask of the 3 possible interrupts for this engine. It will 1833 * contain bits _only_ for this SDMA engine. It will contain at least one 1834 * bit, it may contain more. 1835 */ 1836 void sdma_engine_interrupt(struct sdma_engine *sde, u64 status) 1837 { 1838 trace_hfi1_sdma_engine_interrupt(sde, status); 1839 write_seqlock(&sde->head_lock); 1840 sdma_set_desc_cnt(sde, sdma_desct_intr); 1841 if (status & sde->idle_mask) 1842 sde->idle_int_cnt++; 1843 else if (status & sde->progress_mask) 1844 sde->progress_int_cnt++; 1845 else if (status & sde->int_mask) 1846 sde->sdma_int_cnt++; 1847 sdma_make_progress(sde, status); 1848 write_sequnlock(&sde->head_lock); 1849 } 1850 1851 /** 1852 * sdma_engine_error() - error handler for engine 1853 * @sde: sdma engine 1854 * @status: sdma interrupt reason 1855 */ 1856 void sdma_engine_error(struct sdma_engine *sde, u64 status) 1857 { 1858 unsigned long flags; 1859 1860 #ifdef CONFIG_SDMA_VERBOSITY 1861 dd_dev_err(sde->dd, "CONFIG SDMA(%u) error status 0x%llx state %s\n", 1862 sde->this_idx, 1863 (unsigned long long)status, 1864 sdma_state_names[sde->state.current_state]); 1865 #endif 1866 spin_lock_irqsave(&sde->tail_lock, flags); 1867 write_seqlock(&sde->head_lock); 1868 if (status & ALL_SDMA_ENG_HALT_ERRS) 1869 __sdma_process_event(sde, sdma_event_e60_hw_halted); 1870 if (status & ~SD(ENG_ERR_STATUS_SDMA_HALT_ERR_SMASK)) { 1871 dd_dev_err(sde->dd, 1872 "SDMA (%u) engine error: 0x%llx state %s\n", 1873 sde->this_idx, 1874 (unsigned long long)status, 1875 sdma_state_names[sde->state.current_state]); 1876 dump_sdma_state(sde); 1877 } 1878 write_sequnlock(&sde->head_lock); 1879 spin_unlock_irqrestore(&sde->tail_lock, flags); 1880 } 1881 1882 static void sdma_sendctrl(struct sdma_engine *sde, unsigned op) 1883 { 1884 u64 set_senddmactrl = 0; 1885 u64 clr_senddmactrl = 0; 1886 unsigned long flags; 1887 1888 #ifdef CONFIG_SDMA_VERBOSITY 1889 dd_dev_err(sde->dd, "CONFIG SDMA(%u) senddmactrl E=%d I=%d H=%d C=%d\n", 1890 sde->this_idx, 1891 (op & SDMA_SENDCTRL_OP_ENABLE) ? 1 : 0, 1892 (op & SDMA_SENDCTRL_OP_INTENABLE) ? 1 : 0, 1893 (op & SDMA_SENDCTRL_OP_HALT) ? 1 : 0, 1894 (op & SDMA_SENDCTRL_OP_CLEANUP) ? 1 : 0); 1895 #endif 1896 1897 if (op & SDMA_SENDCTRL_OP_ENABLE) 1898 set_senddmactrl |= SD(CTRL_SDMA_ENABLE_SMASK); 1899 else 1900 clr_senddmactrl |= SD(CTRL_SDMA_ENABLE_SMASK); 1901 1902 if (op & SDMA_SENDCTRL_OP_INTENABLE) 1903 set_senddmactrl |= SD(CTRL_SDMA_INT_ENABLE_SMASK); 1904 else 1905 clr_senddmactrl |= SD(CTRL_SDMA_INT_ENABLE_SMASK); 1906 1907 if (op & SDMA_SENDCTRL_OP_HALT) 1908 set_senddmactrl |= SD(CTRL_SDMA_HALT_SMASK); 1909 else 1910 clr_senddmactrl |= SD(CTRL_SDMA_HALT_SMASK); 1911 1912 spin_lock_irqsave(&sde->senddmactrl_lock, flags); 1913 1914 sde->p_senddmactrl |= set_senddmactrl; 1915 sde->p_senddmactrl &= ~clr_senddmactrl; 1916 1917 if (op & SDMA_SENDCTRL_OP_CLEANUP) 1918 write_sde_csr(sde, SD(CTRL), 1919 sde->p_senddmactrl | 1920 SD(CTRL_SDMA_CLEANUP_SMASK)); 1921 else 1922 write_sde_csr(sde, SD(CTRL), sde->p_senddmactrl); 1923 1924 spin_unlock_irqrestore(&sde->senddmactrl_lock, flags); 1925 1926 #ifdef CONFIG_SDMA_VERBOSITY 1927 sdma_dumpstate(sde); 1928 #endif 1929 } 1930 1931 static void sdma_setlengen(struct sdma_engine *sde) 1932 { 1933 #ifdef CONFIG_SDMA_VERBOSITY 1934 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", 1935 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__); 1936 #endif 1937 1938 /* 1939 * Set SendDmaLenGen and clear-then-set the MSB of the generation 1940 * count to enable generation checking and load the internal 1941 * generation counter. 1942 */ 1943 write_sde_csr(sde, SD(LEN_GEN), 1944 (sde->descq_cnt / 64) << SD(LEN_GEN_LENGTH_SHIFT)); 1945 write_sde_csr(sde, SD(LEN_GEN), 1946 ((sde->descq_cnt / 64) << SD(LEN_GEN_LENGTH_SHIFT)) | 1947 (4ULL << SD(LEN_GEN_GENERATION_SHIFT))); 1948 } 1949 1950 static inline void sdma_update_tail(struct sdma_engine *sde, u16 tail) 1951 { 1952 /* Commit writes to memory and advance the tail on the chip */ 1953 smp_wmb(); /* see get_txhead() */ 1954 writeq(tail, sde->tail_csr); 1955 } 1956 1957 /* 1958 * This is called when changing to state s10_hw_start_up_halt_wait as 1959 * a result of send buffer errors or send DMA descriptor errors. 1960 */ 1961 static void sdma_hw_start_up(struct sdma_engine *sde) 1962 { 1963 u64 reg; 1964 1965 #ifdef CONFIG_SDMA_VERBOSITY 1966 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", 1967 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__); 1968 #endif 1969 1970 sdma_setlengen(sde); 1971 sdma_update_tail(sde, 0); /* Set SendDmaTail */ 1972 *sde->head_dma = 0; 1973 1974 reg = SD(ENG_ERR_CLEAR_SDMA_HEADER_REQUEST_FIFO_UNC_ERR_MASK) << 1975 SD(ENG_ERR_CLEAR_SDMA_HEADER_REQUEST_FIFO_UNC_ERR_SHIFT); 1976 write_sde_csr(sde, SD(ENG_ERR_CLEAR), reg); 1977 } 1978 1979 /* 1980 * set_sdma_integrity 1981 * 1982 * Set the SEND_DMA_CHECK_ENABLE register for send DMA engine 'sde'. 1983 */ 1984 static void set_sdma_integrity(struct sdma_engine *sde) 1985 { 1986 struct hfi1_devdata *dd = sde->dd; 1987 1988 write_sde_csr(sde, SD(CHECK_ENABLE), 1989 hfi1_pkt_base_sdma_integrity(dd)); 1990 } 1991 1992 static void init_sdma_regs( 1993 struct sdma_engine *sde, 1994 u32 credits, 1995 uint idle_cnt) 1996 { 1997 u8 opval, opmask; 1998 #ifdef CONFIG_SDMA_VERBOSITY 1999 struct hfi1_devdata *dd = sde->dd; 2000 2001 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", 2002 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__); 2003 #endif 2004 2005 write_sde_csr(sde, SD(BASE_ADDR), sde->descq_phys); 2006 sdma_setlengen(sde); 2007 sdma_update_tail(sde, 0); /* Set SendDmaTail */ 2008 write_sde_csr(sde, SD(RELOAD_CNT), idle_cnt); 2009 write_sde_csr(sde, SD(DESC_CNT), 0); 2010 write_sde_csr(sde, SD(HEAD_ADDR), sde->head_phys); 2011 write_sde_csr(sde, SD(MEMORY), 2012 ((u64)credits << SD(MEMORY_SDMA_MEMORY_CNT_SHIFT)) | 2013 ((u64)(credits * sde->this_idx) << 2014 SD(MEMORY_SDMA_MEMORY_INDEX_SHIFT))); 2015 write_sde_csr(sde, SD(ENG_ERR_MASK), ~0ull); 2016 set_sdma_integrity(sde); 2017 opmask = OPCODE_CHECK_MASK_DISABLED; 2018 opval = OPCODE_CHECK_VAL_DISABLED; 2019 write_sde_csr(sde, SD(CHECK_OPCODE), 2020 (opmask << SEND_CTXT_CHECK_OPCODE_MASK_SHIFT) | 2021 (opval << SEND_CTXT_CHECK_OPCODE_VALUE_SHIFT)); 2022 } 2023 2024 #ifdef CONFIG_SDMA_VERBOSITY 2025 2026 #define sdma_dumpstate_helper0(reg) do { \ 2027 csr = read_csr(sde->dd, reg); \ 2028 dd_dev_err(sde->dd, "%36s 0x%016llx\n", #reg, csr); \ 2029 } while (0) 2030 2031 #define sdma_dumpstate_helper(reg) do { \ 2032 csr = read_sde_csr(sde, reg); \ 2033 dd_dev_err(sde->dd, "%36s[%02u] 0x%016llx\n", \ 2034 #reg, sde->this_idx, csr); \ 2035 } while (0) 2036 2037 #define sdma_dumpstate_helper2(reg) do { \ 2038 csr = read_csr(sde->dd, reg + (8 * i)); \ 2039 dd_dev_err(sde->dd, "%33s_%02u 0x%016llx\n", \ 2040 #reg, i, csr); \ 2041 } while (0) 2042 2043 void sdma_dumpstate(struct sdma_engine *sde) 2044 { 2045 u64 csr; 2046 unsigned i; 2047 2048 sdma_dumpstate_helper(SD(CTRL)); 2049 sdma_dumpstate_helper(SD(STATUS)); 2050 sdma_dumpstate_helper0(SD(ERR_STATUS)); 2051 sdma_dumpstate_helper0(SD(ERR_MASK)); 2052 sdma_dumpstate_helper(SD(ENG_ERR_STATUS)); 2053 sdma_dumpstate_helper(SD(ENG_ERR_MASK)); 2054 2055 for (i = 0; i < CCE_NUM_INT_CSRS; ++i) { 2056 sdma_dumpstate_helper2(CCE_INT_STATUS); 2057 sdma_dumpstate_helper2(CCE_INT_MASK); 2058 sdma_dumpstate_helper2(CCE_INT_BLOCKED); 2059 } 2060 2061 sdma_dumpstate_helper(SD(TAIL)); 2062 sdma_dumpstate_helper(SD(HEAD)); 2063 sdma_dumpstate_helper(SD(PRIORITY_THLD)); 2064 sdma_dumpstate_helper(SD(IDLE_CNT)); 2065 sdma_dumpstate_helper(SD(RELOAD_CNT)); 2066 sdma_dumpstate_helper(SD(DESC_CNT)); 2067 sdma_dumpstate_helper(SD(DESC_FETCHED_CNT)); 2068 sdma_dumpstate_helper(SD(MEMORY)); 2069 sdma_dumpstate_helper0(SD(ENGINES)); 2070 sdma_dumpstate_helper0(SD(MEM_SIZE)); 2071 /* sdma_dumpstate_helper(SEND_EGRESS_SEND_DMA_STATUS); */ 2072 sdma_dumpstate_helper(SD(BASE_ADDR)); 2073 sdma_dumpstate_helper(SD(LEN_GEN)); 2074 sdma_dumpstate_helper(SD(HEAD_ADDR)); 2075 sdma_dumpstate_helper(SD(CHECK_ENABLE)); 2076 sdma_dumpstate_helper(SD(CHECK_VL)); 2077 sdma_dumpstate_helper(SD(CHECK_JOB_KEY)); 2078 sdma_dumpstate_helper(SD(CHECK_PARTITION_KEY)); 2079 sdma_dumpstate_helper(SD(CHECK_SLID)); 2080 sdma_dumpstate_helper(SD(CHECK_OPCODE)); 2081 } 2082 #endif 2083 2084 static void dump_sdma_state(struct sdma_engine *sde) 2085 { 2086 struct hw_sdma_desc *descqp; 2087 u64 desc[2]; 2088 u64 addr; 2089 u8 gen; 2090 u16 len; 2091 u16 head, tail, cnt; 2092 2093 head = sde->descq_head & sde->sdma_mask; 2094 tail = sde->descq_tail & sde->sdma_mask; 2095 cnt = sdma_descq_freecnt(sde); 2096 2097 dd_dev_err(sde->dd, 2098 "SDMA (%u) descq_head: %u descq_tail: %u freecnt: %u FLE %d\n", 2099 sde->this_idx, head, tail, cnt, 2100 !list_empty(&sde->flushlist)); 2101 2102 /* print info for each entry in the descriptor queue */ 2103 while (head != tail) { 2104 char flags[6] = { 'x', 'x', 'x', 'x', 0 }; 2105 2106 descqp = &sde->descq[head]; 2107 desc[0] = le64_to_cpu(descqp->qw[0]); 2108 desc[1] = le64_to_cpu(descqp->qw[1]); 2109 flags[0] = (desc[1] & SDMA_DESC1_INT_REQ_FLAG) ? 'I' : '-'; 2110 flags[1] = (desc[1] & SDMA_DESC1_HEAD_TO_HOST_FLAG) ? 2111 'H' : '-'; 2112 flags[2] = (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) ? 'F' : '-'; 2113 flags[3] = (desc[0] & SDMA_DESC0_LAST_DESC_FLAG) ? 'L' : '-'; 2114 addr = (desc[0] >> SDMA_DESC0_PHY_ADDR_SHIFT) 2115 & SDMA_DESC0_PHY_ADDR_MASK; 2116 gen = (desc[1] >> SDMA_DESC1_GENERATION_SHIFT) 2117 & SDMA_DESC1_GENERATION_MASK; 2118 len = (desc[0] >> SDMA_DESC0_BYTE_COUNT_SHIFT) 2119 & SDMA_DESC0_BYTE_COUNT_MASK; 2120 dd_dev_err(sde->dd, 2121 "SDMA sdmadesc[%u]: flags:%s addr:0x%016llx gen:%u len:%u bytes\n", 2122 head, flags, addr, gen, len); 2123 dd_dev_err(sde->dd, 2124 "\tdesc0:0x%016llx desc1 0x%016llx\n", 2125 desc[0], desc[1]); 2126 if (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) 2127 dd_dev_err(sde->dd, 2128 "\taidx: %u amode: %u alen: %u\n", 2129 (u8)((desc[1] & 2130 SDMA_DESC1_HEADER_INDEX_SMASK) >> 2131 SDMA_DESC1_HEADER_INDEX_SHIFT), 2132 (u8)((desc[1] & 2133 SDMA_DESC1_HEADER_MODE_SMASK) >> 2134 SDMA_DESC1_HEADER_MODE_SHIFT), 2135 (u8)((desc[1] & 2136 SDMA_DESC1_HEADER_DWS_SMASK) >> 2137 SDMA_DESC1_HEADER_DWS_SHIFT)); 2138 head++; 2139 head &= sde->sdma_mask; 2140 } 2141 } 2142 2143 #define SDE_FMT \ 2144 "SDE %u CPU %d STE %s C 0x%llx S 0x%016llx E 0x%llx T(HW) 0x%llx T(SW) 0x%x H(HW) 0x%llx H(SW) 0x%x H(D) 0x%llx DM 0x%llx GL 0x%llx R 0x%llx LIS 0x%llx AHGI 0x%llx TXT %u TXH %u DT %u DH %u FLNE %d DQF %u SLC 0x%llx\n" 2145 /** 2146 * sdma_seqfile_dump_sde() - debugfs dump of sde 2147 * @s: seq file 2148 * @sde: send dma engine to dump 2149 * 2150 * This routine dumps the sde to the indicated seq file. 2151 */ 2152 void sdma_seqfile_dump_sde(struct seq_file *s, struct sdma_engine *sde) 2153 { 2154 u16 head, tail; 2155 struct hw_sdma_desc *descqp; 2156 u64 desc[2]; 2157 u64 addr; 2158 u8 gen; 2159 u16 len; 2160 2161 head = sde->descq_head & sde->sdma_mask; 2162 tail = READ_ONCE(sde->descq_tail) & sde->sdma_mask; 2163 seq_printf(s, SDE_FMT, sde->this_idx, 2164 sde->cpu, 2165 sdma_state_name(sde->state.current_state), 2166 (unsigned long long)read_sde_csr(sde, SD(CTRL)), 2167 (unsigned long long)read_sde_csr(sde, SD(STATUS)), 2168 (unsigned long long)read_sde_csr(sde, SD(ENG_ERR_STATUS)), 2169 (unsigned long long)read_sde_csr(sde, SD(TAIL)), tail, 2170 (unsigned long long)read_sde_csr(sde, SD(HEAD)), head, 2171 (unsigned long long)le64_to_cpu(*sde->head_dma), 2172 (unsigned long long)read_sde_csr(sde, SD(MEMORY)), 2173 (unsigned long long)read_sde_csr(sde, SD(LEN_GEN)), 2174 (unsigned long long)read_sde_csr(sde, SD(RELOAD_CNT)), 2175 (unsigned long long)sde->last_status, 2176 (unsigned long long)sde->ahg_bits, 2177 sde->tx_tail, 2178 sde->tx_head, 2179 sde->descq_tail, 2180 sde->descq_head, 2181 !list_empty(&sde->flushlist), 2182 sde->descq_full_count, 2183 (unsigned long long)read_sde_csr(sde, SEND_DMA_CHECK_SLID)); 2184 2185 /* print info for each entry in the descriptor queue */ 2186 while (head != tail) { 2187 char flags[6] = { 'x', 'x', 'x', 'x', 0 }; 2188 2189 descqp = &sde->descq[head]; 2190 desc[0] = le64_to_cpu(descqp->qw[0]); 2191 desc[1] = le64_to_cpu(descqp->qw[1]); 2192 flags[0] = (desc[1] & SDMA_DESC1_INT_REQ_FLAG) ? 'I' : '-'; 2193 flags[1] = (desc[1] & SDMA_DESC1_HEAD_TO_HOST_FLAG) ? 2194 'H' : '-'; 2195 flags[2] = (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) ? 'F' : '-'; 2196 flags[3] = (desc[0] & SDMA_DESC0_LAST_DESC_FLAG) ? 'L' : '-'; 2197 addr = (desc[0] >> SDMA_DESC0_PHY_ADDR_SHIFT) 2198 & SDMA_DESC0_PHY_ADDR_MASK; 2199 gen = (desc[1] >> SDMA_DESC1_GENERATION_SHIFT) 2200 & SDMA_DESC1_GENERATION_MASK; 2201 len = (desc[0] >> SDMA_DESC0_BYTE_COUNT_SHIFT) 2202 & SDMA_DESC0_BYTE_COUNT_MASK; 2203 seq_printf(s, 2204 "\tdesc[%u]: flags:%s addr:0x%016llx gen:%u len:%u bytes\n", 2205 head, flags, addr, gen, len); 2206 if (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) 2207 seq_printf(s, "\t\tahgidx: %u ahgmode: %u\n", 2208 (u8)((desc[1] & 2209 SDMA_DESC1_HEADER_INDEX_SMASK) >> 2210 SDMA_DESC1_HEADER_INDEX_SHIFT), 2211 (u8)((desc[1] & 2212 SDMA_DESC1_HEADER_MODE_SMASK) >> 2213 SDMA_DESC1_HEADER_MODE_SHIFT)); 2214 head = (head + 1) & sde->sdma_mask; 2215 } 2216 } 2217 2218 /* 2219 * add the generation number into 2220 * the qw1 and return 2221 */ 2222 static inline u64 add_gen(struct sdma_engine *sde, u64 qw1) 2223 { 2224 u8 generation = (sde->descq_tail >> sde->sdma_shift) & 3; 2225 2226 qw1 &= ~SDMA_DESC1_GENERATION_SMASK; 2227 qw1 |= ((u64)generation & SDMA_DESC1_GENERATION_MASK) 2228 << SDMA_DESC1_GENERATION_SHIFT; 2229 return qw1; 2230 } 2231 2232 /* 2233 * This routine submits the indicated tx 2234 * 2235 * Space has already been guaranteed and 2236 * tail side of ring is locked. 2237 * 2238 * The hardware tail update is done 2239 * in the caller and that is facilitated 2240 * by returning the new tail. 2241 * 2242 * There is special case logic for ahg 2243 * to not add the generation number for 2244 * up to 2 descriptors that follow the 2245 * first descriptor. 2246 * 2247 */ 2248 static inline u16 submit_tx(struct sdma_engine *sde, struct sdma_txreq *tx) 2249 { 2250 int i; 2251 u16 tail; 2252 struct sdma_desc *descp = tx->descp; 2253 u8 skip = 0, mode = ahg_mode(tx); 2254 2255 tail = sde->descq_tail & sde->sdma_mask; 2256 sde->descq[tail].qw[0] = cpu_to_le64(descp->qw[0]); 2257 sde->descq[tail].qw[1] = cpu_to_le64(add_gen(sde, descp->qw[1])); 2258 trace_hfi1_sdma_descriptor(sde, descp->qw[0], descp->qw[1], 2259 tail, &sde->descq[tail]); 2260 tail = ++sde->descq_tail & sde->sdma_mask; 2261 descp++; 2262 if (mode > SDMA_AHG_APPLY_UPDATE1) 2263 skip = mode >> 1; 2264 for (i = 1; i < tx->num_desc; i++, descp++) { 2265 u64 qw1; 2266 2267 sde->descq[tail].qw[0] = cpu_to_le64(descp->qw[0]); 2268 if (skip) { 2269 /* edits don't have generation */ 2270 qw1 = descp->qw[1]; 2271 skip--; 2272 } else { 2273 /* replace generation with real one for non-edits */ 2274 qw1 = add_gen(sde, descp->qw[1]); 2275 } 2276 sde->descq[tail].qw[1] = cpu_to_le64(qw1); 2277 trace_hfi1_sdma_descriptor(sde, descp->qw[0], qw1, 2278 tail, &sde->descq[tail]); 2279 tail = ++sde->descq_tail & sde->sdma_mask; 2280 } 2281 tx->next_descq_idx = tail; 2282 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER 2283 tx->sn = sde->tail_sn++; 2284 trace_hfi1_sdma_in_sn(sde, tx->sn); 2285 WARN_ON_ONCE(sde->tx_ring[sde->tx_tail & sde->sdma_mask]); 2286 #endif 2287 sde->tx_ring[sde->tx_tail++ & sde->sdma_mask] = tx; 2288 sde->desc_avail -= tx->num_desc; 2289 return tail; 2290 } 2291 2292 /* 2293 * Check for progress 2294 */ 2295 static int sdma_check_progress( 2296 struct sdma_engine *sde, 2297 struct iowait_work *wait, 2298 struct sdma_txreq *tx, 2299 bool pkts_sent) 2300 { 2301 int ret; 2302 2303 sde->desc_avail = sdma_descq_freecnt(sde); 2304 if (tx->num_desc <= sde->desc_avail) 2305 return -EAGAIN; 2306 /* pulse the head_lock */ 2307 if (wait && iowait_ioww_to_iow(wait)->sleep) { 2308 unsigned seq; 2309 2310 seq = raw_seqcount_begin( 2311 (const seqcount_t *)&sde->head_lock.seqcount); 2312 ret = wait->iow->sleep(sde, wait, tx, seq, pkts_sent); 2313 if (ret == -EAGAIN) 2314 sde->desc_avail = sdma_descq_freecnt(sde); 2315 } else { 2316 ret = -EBUSY; 2317 } 2318 return ret; 2319 } 2320 2321 /** 2322 * sdma_send_txreq() - submit a tx req to ring 2323 * @sde: sdma engine to use 2324 * @wait: SE wait structure to use when full (may be NULL) 2325 * @tx: sdma_txreq to submit 2326 * @pkts_sent: has any packet been sent yet? 2327 * 2328 * The call submits the tx into the ring. If a iowait structure is non-NULL 2329 * the packet will be queued to the list in wait. 2330 * 2331 * Return: 2332 * 0 - Success, -EINVAL - sdma_txreq incomplete, -EBUSY - no space in 2333 * ring (wait == NULL) 2334 * -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state 2335 */ 2336 int sdma_send_txreq(struct sdma_engine *sde, 2337 struct iowait_work *wait, 2338 struct sdma_txreq *tx, 2339 bool pkts_sent) 2340 { 2341 int ret = 0; 2342 u16 tail; 2343 unsigned long flags; 2344 2345 /* user should have supplied entire packet */ 2346 if (unlikely(tx->tlen)) 2347 return -EINVAL; 2348 tx->wait = iowait_ioww_to_iow(wait); 2349 spin_lock_irqsave(&sde->tail_lock, flags); 2350 retry: 2351 if (unlikely(!__sdma_running(sde))) 2352 goto unlock_noconn; 2353 if (unlikely(tx->num_desc > sde->desc_avail)) 2354 goto nodesc; 2355 tail = submit_tx(sde, tx); 2356 if (wait) 2357 iowait_sdma_inc(iowait_ioww_to_iow(wait)); 2358 sdma_update_tail(sde, tail); 2359 unlock: 2360 spin_unlock_irqrestore(&sde->tail_lock, flags); 2361 return ret; 2362 unlock_noconn: 2363 if (wait) 2364 iowait_sdma_inc(iowait_ioww_to_iow(wait)); 2365 tx->next_descq_idx = 0; 2366 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER 2367 tx->sn = sde->tail_sn++; 2368 trace_hfi1_sdma_in_sn(sde, tx->sn); 2369 #endif 2370 spin_lock(&sde->flushlist_lock); 2371 list_add_tail(&tx->list, &sde->flushlist); 2372 spin_unlock(&sde->flushlist_lock); 2373 iowait_inc_wait_count(wait, tx->num_desc); 2374 queue_work_on(sde->cpu, system_highpri_wq, &sde->flush_worker); 2375 ret = -ECOMM; 2376 goto unlock; 2377 nodesc: 2378 ret = sdma_check_progress(sde, wait, tx, pkts_sent); 2379 if (ret == -EAGAIN) { 2380 ret = 0; 2381 goto retry; 2382 } 2383 sde->descq_full_count++; 2384 goto unlock; 2385 } 2386 2387 /** 2388 * sdma_send_txlist() - submit a list of tx req to ring 2389 * @sde: sdma engine to use 2390 * @wait: SE wait structure to use when full (may be NULL) 2391 * @tx_list: list of sdma_txreqs to submit 2392 * @count_out: pointer to a u16 which, after return will contain the total number of 2393 * sdma_txreqs removed from the tx_list. This will include sdma_txreqs 2394 * whose SDMA descriptors are submitted to the ring and the sdma_txreqs 2395 * which are added to SDMA engine flush list if the SDMA engine state is 2396 * not running. 2397 * 2398 * The call submits the list into the ring. 2399 * 2400 * If the iowait structure is non-NULL and not equal to the iowait list 2401 * the unprocessed part of the list will be appended to the list in wait. 2402 * 2403 * In all cases, the tx_list will be updated so the head of the tx_list is 2404 * the list of descriptors that have yet to be transmitted. 2405 * 2406 * The intent of this call is to provide a more efficient 2407 * way of submitting multiple packets to SDMA while holding the tail 2408 * side locking. 2409 * 2410 * Return: 2411 * 0 - Success, 2412 * -EINVAL - sdma_txreq incomplete, -EBUSY - no space in ring (wait == NULL) 2413 * -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state 2414 */ 2415 int sdma_send_txlist(struct sdma_engine *sde, struct iowait_work *wait, 2416 struct list_head *tx_list, u16 *count_out) 2417 { 2418 struct sdma_txreq *tx, *tx_next; 2419 int ret = 0; 2420 unsigned long flags; 2421 u16 tail = INVALID_TAIL; 2422 u32 submit_count = 0, flush_count = 0, total_count; 2423 2424 spin_lock_irqsave(&sde->tail_lock, flags); 2425 retry: 2426 list_for_each_entry_safe(tx, tx_next, tx_list, list) { 2427 tx->wait = iowait_ioww_to_iow(wait); 2428 if (unlikely(!__sdma_running(sde))) 2429 goto unlock_noconn; 2430 if (unlikely(tx->num_desc > sde->desc_avail)) 2431 goto nodesc; 2432 if (unlikely(tx->tlen)) { 2433 ret = -EINVAL; 2434 goto update_tail; 2435 } 2436 list_del_init(&tx->list); 2437 tail = submit_tx(sde, tx); 2438 submit_count++; 2439 if (tail != INVALID_TAIL && 2440 (submit_count & SDMA_TAIL_UPDATE_THRESH) == 0) { 2441 sdma_update_tail(sde, tail); 2442 tail = INVALID_TAIL; 2443 } 2444 } 2445 update_tail: 2446 total_count = submit_count + flush_count; 2447 if (wait) { 2448 iowait_sdma_add(iowait_ioww_to_iow(wait), total_count); 2449 iowait_starve_clear(submit_count > 0, 2450 iowait_ioww_to_iow(wait)); 2451 } 2452 if (tail != INVALID_TAIL) 2453 sdma_update_tail(sde, tail); 2454 spin_unlock_irqrestore(&sde->tail_lock, flags); 2455 *count_out = total_count; 2456 return ret; 2457 unlock_noconn: 2458 spin_lock(&sde->flushlist_lock); 2459 list_for_each_entry_safe(tx, tx_next, tx_list, list) { 2460 tx->wait = iowait_ioww_to_iow(wait); 2461 list_del_init(&tx->list); 2462 tx->next_descq_idx = 0; 2463 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER 2464 tx->sn = sde->tail_sn++; 2465 trace_hfi1_sdma_in_sn(sde, tx->sn); 2466 #endif 2467 list_add_tail(&tx->list, &sde->flushlist); 2468 flush_count++; 2469 iowait_inc_wait_count(wait, tx->num_desc); 2470 } 2471 spin_unlock(&sde->flushlist_lock); 2472 queue_work_on(sde->cpu, system_highpri_wq, &sde->flush_worker); 2473 ret = -ECOMM; 2474 goto update_tail; 2475 nodesc: 2476 ret = sdma_check_progress(sde, wait, tx, submit_count > 0); 2477 if (ret == -EAGAIN) { 2478 ret = 0; 2479 goto retry; 2480 } 2481 sde->descq_full_count++; 2482 goto update_tail; 2483 } 2484 2485 static void sdma_process_event(struct sdma_engine *sde, enum sdma_events event) 2486 { 2487 unsigned long flags; 2488 2489 spin_lock_irqsave(&sde->tail_lock, flags); 2490 write_seqlock(&sde->head_lock); 2491 2492 __sdma_process_event(sde, event); 2493 2494 if (sde->state.current_state == sdma_state_s99_running) 2495 sdma_desc_avail(sde, sdma_descq_freecnt(sde)); 2496 2497 write_sequnlock(&sde->head_lock); 2498 spin_unlock_irqrestore(&sde->tail_lock, flags); 2499 } 2500 2501 static void __sdma_process_event(struct sdma_engine *sde, 2502 enum sdma_events event) 2503 { 2504 struct sdma_state *ss = &sde->state; 2505 int need_progress = 0; 2506 2507 /* CONFIG SDMA temporary */ 2508 #ifdef CONFIG_SDMA_VERBOSITY 2509 dd_dev_err(sde->dd, "CONFIG SDMA(%u) [%s] %s\n", sde->this_idx, 2510 sdma_state_names[ss->current_state], 2511 sdma_event_names[event]); 2512 #endif 2513 2514 switch (ss->current_state) { 2515 case sdma_state_s00_hw_down: 2516 switch (event) { 2517 case sdma_event_e00_go_hw_down: 2518 break; 2519 case sdma_event_e30_go_running: 2520 /* 2521 * If down, but running requested (usually result 2522 * of link up, then we need to start up. 2523 * This can happen when hw down is requested while 2524 * bringing the link up with traffic active on 2525 * 7220, e.g. 2526 */ 2527 ss->go_s99_running = 1; 2528 fallthrough; /* and start dma engine */ 2529 case sdma_event_e10_go_hw_start: 2530 /* This reference means the state machine is started */ 2531 sdma_get(&sde->state); 2532 sdma_set_state(sde, 2533 sdma_state_s10_hw_start_up_halt_wait); 2534 break; 2535 case sdma_event_e15_hw_halt_done: 2536 break; 2537 case sdma_event_e25_hw_clean_up_done: 2538 break; 2539 case sdma_event_e40_sw_cleaned: 2540 sdma_sw_tear_down(sde); 2541 break; 2542 case sdma_event_e50_hw_cleaned: 2543 break; 2544 case sdma_event_e60_hw_halted: 2545 break; 2546 case sdma_event_e70_go_idle: 2547 break; 2548 case sdma_event_e80_hw_freeze: 2549 break; 2550 case sdma_event_e81_hw_frozen: 2551 break; 2552 case sdma_event_e82_hw_unfreeze: 2553 break; 2554 case sdma_event_e85_link_down: 2555 break; 2556 case sdma_event_e90_sw_halted: 2557 break; 2558 } 2559 break; 2560 2561 case sdma_state_s10_hw_start_up_halt_wait: 2562 switch (event) { 2563 case sdma_event_e00_go_hw_down: 2564 sdma_set_state(sde, sdma_state_s00_hw_down); 2565 sdma_sw_tear_down(sde); 2566 break; 2567 case sdma_event_e10_go_hw_start: 2568 break; 2569 case sdma_event_e15_hw_halt_done: 2570 sdma_set_state(sde, 2571 sdma_state_s15_hw_start_up_clean_wait); 2572 sdma_start_hw_clean_up(sde); 2573 break; 2574 case sdma_event_e25_hw_clean_up_done: 2575 break; 2576 case sdma_event_e30_go_running: 2577 ss->go_s99_running = 1; 2578 break; 2579 case sdma_event_e40_sw_cleaned: 2580 break; 2581 case sdma_event_e50_hw_cleaned: 2582 break; 2583 case sdma_event_e60_hw_halted: 2584 schedule_work(&sde->err_halt_worker); 2585 break; 2586 case sdma_event_e70_go_idle: 2587 ss->go_s99_running = 0; 2588 break; 2589 case sdma_event_e80_hw_freeze: 2590 break; 2591 case sdma_event_e81_hw_frozen: 2592 break; 2593 case sdma_event_e82_hw_unfreeze: 2594 break; 2595 case sdma_event_e85_link_down: 2596 break; 2597 case sdma_event_e90_sw_halted: 2598 break; 2599 } 2600 break; 2601 2602 case sdma_state_s15_hw_start_up_clean_wait: 2603 switch (event) { 2604 case sdma_event_e00_go_hw_down: 2605 sdma_set_state(sde, sdma_state_s00_hw_down); 2606 sdma_sw_tear_down(sde); 2607 break; 2608 case sdma_event_e10_go_hw_start: 2609 break; 2610 case sdma_event_e15_hw_halt_done: 2611 break; 2612 case sdma_event_e25_hw_clean_up_done: 2613 sdma_hw_start_up(sde); 2614 sdma_set_state(sde, ss->go_s99_running ? 2615 sdma_state_s99_running : 2616 sdma_state_s20_idle); 2617 break; 2618 case sdma_event_e30_go_running: 2619 ss->go_s99_running = 1; 2620 break; 2621 case sdma_event_e40_sw_cleaned: 2622 break; 2623 case sdma_event_e50_hw_cleaned: 2624 break; 2625 case sdma_event_e60_hw_halted: 2626 break; 2627 case sdma_event_e70_go_idle: 2628 ss->go_s99_running = 0; 2629 break; 2630 case sdma_event_e80_hw_freeze: 2631 break; 2632 case sdma_event_e81_hw_frozen: 2633 break; 2634 case sdma_event_e82_hw_unfreeze: 2635 break; 2636 case sdma_event_e85_link_down: 2637 break; 2638 case sdma_event_e90_sw_halted: 2639 break; 2640 } 2641 break; 2642 2643 case sdma_state_s20_idle: 2644 switch (event) { 2645 case sdma_event_e00_go_hw_down: 2646 sdma_set_state(sde, sdma_state_s00_hw_down); 2647 sdma_sw_tear_down(sde); 2648 break; 2649 case sdma_event_e10_go_hw_start: 2650 break; 2651 case sdma_event_e15_hw_halt_done: 2652 break; 2653 case sdma_event_e25_hw_clean_up_done: 2654 break; 2655 case sdma_event_e30_go_running: 2656 sdma_set_state(sde, sdma_state_s99_running); 2657 ss->go_s99_running = 1; 2658 break; 2659 case sdma_event_e40_sw_cleaned: 2660 break; 2661 case sdma_event_e50_hw_cleaned: 2662 break; 2663 case sdma_event_e60_hw_halted: 2664 sdma_set_state(sde, sdma_state_s50_hw_halt_wait); 2665 schedule_work(&sde->err_halt_worker); 2666 break; 2667 case sdma_event_e70_go_idle: 2668 break; 2669 case sdma_event_e85_link_down: 2670 case sdma_event_e80_hw_freeze: 2671 sdma_set_state(sde, sdma_state_s80_hw_freeze); 2672 atomic_dec(&sde->dd->sdma_unfreeze_count); 2673 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq); 2674 break; 2675 case sdma_event_e81_hw_frozen: 2676 break; 2677 case sdma_event_e82_hw_unfreeze: 2678 break; 2679 case sdma_event_e90_sw_halted: 2680 break; 2681 } 2682 break; 2683 2684 case sdma_state_s30_sw_clean_up_wait: 2685 switch (event) { 2686 case sdma_event_e00_go_hw_down: 2687 sdma_set_state(sde, sdma_state_s00_hw_down); 2688 break; 2689 case sdma_event_e10_go_hw_start: 2690 break; 2691 case sdma_event_e15_hw_halt_done: 2692 break; 2693 case sdma_event_e25_hw_clean_up_done: 2694 break; 2695 case sdma_event_e30_go_running: 2696 ss->go_s99_running = 1; 2697 break; 2698 case sdma_event_e40_sw_cleaned: 2699 sdma_set_state(sde, sdma_state_s40_hw_clean_up_wait); 2700 sdma_start_hw_clean_up(sde); 2701 break; 2702 case sdma_event_e50_hw_cleaned: 2703 break; 2704 case sdma_event_e60_hw_halted: 2705 break; 2706 case sdma_event_e70_go_idle: 2707 ss->go_s99_running = 0; 2708 break; 2709 case sdma_event_e80_hw_freeze: 2710 break; 2711 case sdma_event_e81_hw_frozen: 2712 break; 2713 case sdma_event_e82_hw_unfreeze: 2714 break; 2715 case sdma_event_e85_link_down: 2716 ss->go_s99_running = 0; 2717 break; 2718 case sdma_event_e90_sw_halted: 2719 break; 2720 } 2721 break; 2722 2723 case sdma_state_s40_hw_clean_up_wait: 2724 switch (event) { 2725 case sdma_event_e00_go_hw_down: 2726 sdma_set_state(sde, sdma_state_s00_hw_down); 2727 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2728 break; 2729 case sdma_event_e10_go_hw_start: 2730 break; 2731 case sdma_event_e15_hw_halt_done: 2732 break; 2733 case sdma_event_e25_hw_clean_up_done: 2734 sdma_hw_start_up(sde); 2735 sdma_set_state(sde, ss->go_s99_running ? 2736 sdma_state_s99_running : 2737 sdma_state_s20_idle); 2738 break; 2739 case sdma_event_e30_go_running: 2740 ss->go_s99_running = 1; 2741 break; 2742 case sdma_event_e40_sw_cleaned: 2743 break; 2744 case sdma_event_e50_hw_cleaned: 2745 break; 2746 case sdma_event_e60_hw_halted: 2747 break; 2748 case sdma_event_e70_go_idle: 2749 ss->go_s99_running = 0; 2750 break; 2751 case sdma_event_e80_hw_freeze: 2752 break; 2753 case sdma_event_e81_hw_frozen: 2754 break; 2755 case sdma_event_e82_hw_unfreeze: 2756 break; 2757 case sdma_event_e85_link_down: 2758 ss->go_s99_running = 0; 2759 break; 2760 case sdma_event_e90_sw_halted: 2761 break; 2762 } 2763 break; 2764 2765 case sdma_state_s50_hw_halt_wait: 2766 switch (event) { 2767 case sdma_event_e00_go_hw_down: 2768 sdma_set_state(sde, sdma_state_s00_hw_down); 2769 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2770 break; 2771 case sdma_event_e10_go_hw_start: 2772 break; 2773 case sdma_event_e15_hw_halt_done: 2774 sdma_set_state(sde, sdma_state_s30_sw_clean_up_wait); 2775 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2776 break; 2777 case sdma_event_e25_hw_clean_up_done: 2778 break; 2779 case sdma_event_e30_go_running: 2780 ss->go_s99_running = 1; 2781 break; 2782 case sdma_event_e40_sw_cleaned: 2783 break; 2784 case sdma_event_e50_hw_cleaned: 2785 break; 2786 case sdma_event_e60_hw_halted: 2787 schedule_work(&sde->err_halt_worker); 2788 break; 2789 case sdma_event_e70_go_idle: 2790 ss->go_s99_running = 0; 2791 break; 2792 case sdma_event_e80_hw_freeze: 2793 break; 2794 case sdma_event_e81_hw_frozen: 2795 break; 2796 case sdma_event_e82_hw_unfreeze: 2797 break; 2798 case sdma_event_e85_link_down: 2799 ss->go_s99_running = 0; 2800 break; 2801 case sdma_event_e90_sw_halted: 2802 break; 2803 } 2804 break; 2805 2806 case sdma_state_s60_idle_halt_wait: 2807 switch (event) { 2808 case sdma_event_e00_go_hw_down: 2809 sdma_set_state(sde, sdma_state_s00_hw_down); 2810 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2811 break; 2812 case sdma_event_e10_go_hw_start: 2813 break; 2814 case sdma_event_e15_hw_halt_done: 2815 sdma_set_state(sde, sdma_state_s30_sw_clean_up_wait); 2816 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2817 break; 2818 case sdma_event_e25_hw_clean_up_done: 2819 break; 2820 case sdma_event_e30_go_running: 2821 ss->go_s99_running = 1; 2822 break; 2823 case sdma_event_e40_sw_cleaned: 2824 break; 2825 case sdma_event_e50_hw_cleaned: 2826 break; 2827 case sdma_event_e60_hw_halted: 2828 schedule_work(&sde->err_halt_worker); 2829 break; 2830 case sdma_event_e70_go_idle: 2831 ss->go_s99_running = 0; 2832 break; 2833 case sdma_event_e80_hw_freeze: 2834 break; 2835 case sdma_event_e81_hw_frozen: 2836 break; 2837 case sdma_event_e82_hw_unfreeze: 2838 break; 2839 case sdma_event_e85_link_down: 2840 break; 2841 case sdma_event_e90_sw_halted: 2842 break; 2843 } 2844 break; 2845 2846 case sdma_state_s80_hw_freeze: 2847 switch (event) { 2848 case sdma_event_e00_go_hw_down: 2849 sdma_set_state(sde, sdma_state_s00_hw_down); 2850 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2851 break; 2852 case sdma_event_e10_go_hw_start: 2853 break; 2854 case sdma_event_e15_hw_halt_done: 2855 break; 2856 case sdma_event_e25_hw_clean_up_done: 2857 break; 2858 case sdma_event_e30_go_running: 2859 ss->go_s99_running = 1; 2860 break; 2861 case sdma_event_e40_sw_cleaned: 2862 break; 2863 case sdma_event_e50_hw_cleaned: 2864 break; 2865 case sdma_event_e60_hw_halted: 2866 break; 2867 case sdma_event_e70_go_idle: 2868 ss->go_s99_running = 0; 2869 break; 2870 case sdma_event_e80_hw_freeze: 2871 break; 2872 case sdma_event_e81_hw_frozen: 2873 sdma_set_state(sde, sdma_state_s82_freeze_sw_clean); 2874 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2875 break; 2876 case sdma_event_e82_hw_unfreeze: 2877 break; 2878 case sdma_event_e85_link_down: 2879 break; 2880 case sdma_event_e90_sw_halted: 2881 break; 2882 } 2883 break; 2884 2885 case sdma_state_s82_freeze_sw_clean: 2886 switch (event) { 2887 case sdma_event_e00_go_hw_down: 2888 sdma_set_state(sde, sdma_state_s00_hw_down); 2889 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2890 break; 2891 case sdma_event_e10_go_hw_start: 2892 break; 2893 case sdma_event_e15_hw_halt_done: 2894 break; 2895 case sdma_event_e25_hw_clean_up_done: 2896 break; 2897 case sdma_event_e30_go_running: 2898 ss->go_s99_running = 1; 2899 break; 2900 case sdma_event_e40_sw_cleaned: 2901 /* notify caller this engine is done cleaning */ 2902 atomic_dec(&sde->dd->sdma_unfreeze_count); 2903 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq); 2904 break; 2905 case sdma_event_e50_hw_cleaned: 2906 break; 2907 case sdma_event_e60_hw_halted: 2908 break; 2909 case sdma_event_e70_go_idle: 2910 ss->go_s99_running = 0; 2911 break; 2912 case sdma_event_e80_hw_freeze: 2913 break; 2914 case sdma_event_e81_hw_frozen: 2915 break; 2916 case sdma_event_e82_hw_unfreeze: 2917 sdma_hw_start_up(sde); 2918 sdma_set_state(sde, ss->go_s99_running ? 2919 sdma_state_s99_running : 2920 sdma_state_s20_idle); 2921 break; 2922 case sdma_event_e85_link_down: 2923 break; 2924 case sdma_event_e90_sw_halted: 2925 break; 2926 } 2927 break; 2928 2929 case sdma_state_s99_running: 2930 switch (event) { 2931 case sdma_event_e00_go_hw_down: 2932 sdma_set_state(sde, sdma_state_s00_hw_down); 2933 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2934 break; 2935 case sdma_event_e10_go_hw_start: 2936 break; 2937 case sdma_event_e15_hw_halt_done: 2938 break; 2939 case sdma_event_e25_hw_clean_up_done: 2940 break; 2941 case sdma_event_e30_go_running: 2942 break; 2943 case sdma_event_e40_sw_cleaned: 2944 break; 2945 case sdma_event_e50_hw_cleaned: 2946 break; 2947 case sdma_event_e60_hw_halted: 2948 need_progress = 1; 2949 sdma_err_progress_check_schedule(sde); 2950 fallthrough; 2951 case sdma_event_e90_sw_halted: 2952 /* 2953 * SW initiated halt does not perform engines 2954 * progress check 2955 */ 2956 sdma_set_state(sde, sdma_state_s50_hw_halt_wait); 2957 schedule_work(&sde->err_halt_worker); 2958 break; 2959 case sdma_event_e70_go_idle: 2960 sdma_set_state(sde, sdma_state_s60_idle_halt_wait); 2961 break; 2962 case sdma_event_e85_link_down: 2963 ss->go_s99_running = 0; 2964 fallthrough; 2965 case sdma_event_e80_hw_freeze: 2966 sdma_set_state(sde, sdma_state_s80_hw_freeze); 2967 atomic_dec(&sde->dd->sdma_unfreeze_count); 2968 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq); 2969 break; 2970 case sdma_event_e81_hw_frozen: 2971 break; 2972 case sdma_event_e82_hw_unfreeze: 2973 break; 2974 } 2975 break; 2976 } 2977 2978 ss->last_event = event; 2979 if (need_progress) 2980 sdma_make_progress(sde, 0); 2981 } 2982 2983 /* 2984 * _extend_sdma_tx_descs() - helper to extend txreq 2985 * 2986 * This is called once the initial nominal allocation 2987 * of descriptors in the sdma_txreq is exhausted. 2988 * 2989 * The code will bump the allocation up to the max 2990 * of MAX_DESC (64) descriptors. There doesn't seem 2991 * much point in an interim step. The last descriptor 2992 * is reserved for coalesce buffer in order to support 2993 * cases where input packet has >MAX_DESC iovecs. 2994 * 2995 */ 2996 static int _extend_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx) 2997 { 2998 int i; 2999 struct sdma_desc *descp; 3000 3001 /* Handle last descriptor */ 3002 if (unlikely((tx->num_desc == (MAX_DESC - 1)))) { 3003 /* if tlen is 0, it is for padding, release last descriptor */ 3004 if (!tx->tlen) { 3005 tx->desc_limit = MAX_DESC; 3006 } else if (!tx->coalesce_buf) { 3007 /* allocate coalesce buffer with space for padding */ 3008 tx->coalesce_buf = kmalloc(tx->tlen + sizeof(u32), 3009 GFP_ATOMIC); 3010 if (!tx->coalesce_buf) 3011 goto enomem; 3012 tx->coalesce_idx = 0; 3013 } 3014 return 0; 3015 } 3016 3017 if (unlikely(tx->num_desc == MAX_DESC)) 3018 goto enomem; 3019 3020 descp = kmalloc_array(MAX_DESC, sizeof(struct sdma_desc), GFP_ATOMIC); 3021 if (!descp) 3022 goto enomem; 3023 tx->descp = descp; 3024 3025 /* reserve last descriptor for coalescing */ 3026 tx->desc_limit = MAX_DESC - 1; 3027 /* copy ones already built */ 3028 for (i = 0; i < tx->num_desc; i++) 3029 tx->descp[i] = tx->descs[i]; 3030 return 0; 3031 enomem: 3032 __sdma_txclean(dd, tx); 3033 return -ENOMEM; 3034 } 3035 3036 /* 3037 * ext_coal_sdma_tx_descs() - extend or coalesce sdma tx descriptors 3038 * 3039 * This is called once the initial nominal allocation of descriptors 3040 * in the sdma_txreq is exhausted. 3041 * 3042 * This function calls _extend_sdma_tx_descs to extend or allocate 3043 * coalesce buffer. If there is a allocated coalesce buffer, it will 3044 * copy the input packet data into the coalesce buffer. It also adds 3045 * coalesce buffer descriptor once when whole packet is received. 3046 * 3047 * Return: 3048 * <0 - error 3049 * 0 - coalescing, don't populate descriptor 3050 * 1 - continue with populating descriptor 3051 */ 3052 int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx, 3053 int type, void *kvaddr, struct page *page, 3054 unsigned long offset, u16 len) 3055 { 3056 int pad_len, rval; 3057 dma_addr_t addr; 3058 3059 rval = _extend_sdma_tx_descs(dd, tx); 3060 if (rval) { 3061 __sdma_txclean(dd, tx); 3062 return rval; 3063 } 3064 3065 /* If coalesce buffer is allocated, copy data into it */ 3066 if (tx->coalesce_buf) { 3067 if (type == SDMA_MAP_NONE) { 3068 __sdma_txclean(dd, tx); 3069 return -EINVAL; 3070 } 3071 3072 if (type == SDMA_MAP_PAGE) { 3073 kvaddr = kmap_local_page(page); 3074 kvaddr += offset; 3075 } else if (WARN_ON(!kvaddr)) { 3076 __sdma_txclean(dd, tx); 3077 return -EINVAL; 3078 } 3079 3080 memcpy(tx->coalesce_buf + tx->coalesce_idx, kvaddr, len); 3081 tx->coalesce_idx += len; 3082 if (type == SDMA_MAP_PAGE) 3083 kunmap_local(kvaddr); 3084 3085 /* If there is more data, return */ 3086 if (tx->tlen - tx->coalesce_idx) 3087 return 0; 3088 3089 /* Whole packet is received; add any padding */ 3090 pad_len = tx->packet_len & (sizeof(u32) - 1); 3091 if (pad_len) { 3092 pad_len = sizeof(u32) - pad_len; 3093 memset(tx->coalesce_buf + tx->coalesce_idx, 0, pad_len); 3094 /* padding is taken care of for coalescing case */ 3095 tx->packet_len += pad_len; 3096 tx->tlen += pad_len; 3097 } 3098 3099 /* dma map the coalesce buffer */ 3100 addr = dma_map_single(&dd->pcidev->dev, 3101 tx->coalesce_buf, 3102 tx->tlen, 3103 DMA_TO_DEVICE); 3104 3105 if (unlikely(dma_mapping_error(&dd->pcidev->dev, addr))) { 3106 __sdma_txclean(dd, tx); 3107 return -ENOSPC; 3108 } 3109 3110 /* Add descriptor for coalesce buffer */ 3111 tx->desc_limit = MAX_DESC; 3112 return _sdma_txadd_daddr(dd, SDMA_MAP_SINGLE, tx, 3113 addr, tx->tlen, NULL, NULL, NULL); 3114 } 3115 3116 return 1; 3117 } 3118 3119 /* Update sdes when the lmc changes */ 3120 void sdma_update_lmc(struct hfi1_devdata *dd, u64 mask, u32 lid) 3121 { 3122 struct sdma_engine *sde; 3123 int i; 3124 u64 sreg; 3125 3126 sreg = ((mask & SD(CHECK_SLID_MASK_MASK)) << 3127 SD(CHECK_SLID_MASK_SHIFT)) | 3128 (((lid & mask) & SD(CHECK_SLID_VALUE_MASK)) << 3129 SD(CHECK_SLID_VALUE_SHIFT)); 3130 3131 for (i = 0; i < dd->num_sdma; i++) { 3132 hfi1_cdbg(LINKVERB, "SendDmaEngine[%d].SLID_CHECK = 0x%x", 3133 i, (u32)sreg); 3134 sde = &dd->per_sdma[i]; 3135 write_sde_csr(sde, SD(CHECK_SLID), sreg); 3136 } 3137 } 3138 3139 /* tx not dword sized - pad */ 3140 int _pad_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx) 3141 { 3142 int rval = 0; 3143 3144 if ((unlikely(tx->num_desc == tx->desc_limit))) { 3145 rval = _extend_sdma_tx_descs(dd, tx); 3146 if (rval) { 3147 __sdma_txclean(dd, tx); 3148 return rval; 3149 } 3150 } 3151 3152 /* finish the one just added */ 3153 make_tx_sdma_desc( 3154 tx, 3155 SDMA_MAP_NONE, 3156 dd->sdma_pad_phys, 3157 sizeof(u32) - (tx->packet_len & (sizeof(u32) - 1)), 3158 NULL, NULL, NULL); 3159 tx->num_desc++; 3160 _sdma_close_tx(dd, tx); 3161 return rval; 3162 } 3163 3164 /* 3165 * Add ahg to the sdma_txreq 3166 * 3167 * The logic will consume up to 3 3168 * descriptors at the beginning of 3169 * sdma_txreq. 3170 */ 3171 void _sdma_txreq_ahgadd( 3172 struct sdma_txreq *tx, 3173 u8 num_ahg, 3174 u8 ahg_entry, 3175 u32 *ahg, 3176 u8 ahg_hlen) 3177 { 3178 u32 i, shift = 0, desc = 0; 3179 u8 mode; 3180 3181 WARN_ON_ONCE(num_ahg > 9 || (ahg_hlen & 3) || ahg_hlen == 4); 3182 /* compute mode */ 3183 if (num_ahg == 1) 3184 mode = SDMA_AHG_APPLY_UPDATE1; 3185 else if (num_ahg <= 5) 3186 mode = SDMA_AHG_APPLY_UPDATE2; 3187 else 3188 mode = SDMA_AHG_APPLY_UPDATE3; 3189 tx->num_desc++; 3190 /* initialize to consumed descriptors to zero */ 3191 switch (mode) { 3192 case SDMA_AHG_APPLY_UPDATE3: 3193 tx->num_desc++; 3194 tx->descs[2].qw[0] = 0; 3195 tx->descs[2].qw[1] = 0; 3196 fallthrough; 3197 case SDMA_AHG_APPLY_UPDATE2: 3198 tx->num_desc++; 3199 tx->descs[1].qw[0] = 0; 3200 tx->descs[1].qw[1] = 0; 3201 break; 3202 } 3203 ahg_hlen >>= 2; 3204 tx->descs[0].qw[1] |= 3205 (((u64)ahg_entry & SDMA_DESC1_HEADER_INDEX_MASK) 3206 << SDMA_DESC1_HEADER_INDEX_SHIFT) | 3207 (((u64)ahg_hlen & SDMA_DESC1_HEADER_DWS_MASK) 3208 << SDMA_DESC1_HEADER_DWS_SHIFT) | 3209 (((u64)mode & SDMA_DESC1_HEADER_MODE_MASK) 3210 << SDMA_DESC1_HEADER_MODE_SHIFT) | 3211 (((u64)ahg[0] & SDMA_DESC1_HEADER_UPDATE1_MASK) 3212 << SDMA_DESC1_HEADER_UPDATE1_SHIFT); 3213 for (i = 0; i < (num_ahg - 1); i++) { 3214 if (!shift && !(i & 2)) 3215 desc++; 3216 tx->descs[desc].qw[!!(i & 2)] |= 3217 (((u64)ahg[i + 1]) 3218 << shift); 3219 shift = (shift + 32) & 63; 3220 } 3221 } 3222 3223 /** 3224 * sdma_ahg_alloc - allocate an AHG entry 3225 * @sde: engine to allocate from 3226 * 3227 * Return: 3228 * 0-31 when successful, -EOPNOTSUPP if AHG is not enabled, 3229 * -ENOSPC if an entry is not available 3230 */ 3231 int sdma_ahg_alloc(struct sdma_engine *sde) 3232 { 3233 int nr; 3234 int oldbit; 3235 3236 if (!sde) { 3237 trace_hfi1_ahg_allocate(sde, -EINVAL); 3238 return -EINVAL; 3239 } 3240 while (1) { 3241 nr = ffz(READ_ONCE(sde->ahg_bits)); 3242 if (nr > 31) { 3243 trace_hfi1_ahg_allocate(sde, -ENOSPC); 3244 return -ENOSPC; 3245 } 3246 oldbit = test_and_set_bit(nr, &sde->ahg_bits); 3247 if (!oldbit) 3248 break; 3249 cpu_relax(); 3250 } 3251 trace_hfi1_ahg_allocate(sde, nr); 3252 return nr; 3253 } 3254 3255 /** 3256 * sdma_ahg_free - free an AHG entry 3257 * @sde: engine to return AHG entry 3258 * @ahg_index: index to free 3259 * 3260 * This routine frees the indicate AHG entry. 3261 */ 3262 void sdma_ahg_free(struct sdma_engine *sde, int ahg_index) 3263 { 3264 if (!sde) 3265 return; 3266 trace_hfi1_ahg_deallocate(sde, ahg_index); 3267 if (ahg_index < 0 || ahg_index > 31) 3268 return; 3269 clear_bit(ahg_index, &sde->ahg_bits); 3270 } 3271 3272 /* 3273 * SPC freeze handling for SDMA engines. Called when the driver knows 3274 * the SPC is going into a freeze but before the freeze is fully 3275 * settled. Generally an error interrupt. 3276 * 3277 * This event will pull the engine out of running so no more entries can be 3278 * added to the engine's queue. 3279 */ 3280 void sdma_freeze_notify(struct hfi1_devdata *dd, int link_down) 3281 { 3282 int i; 3283 enum sdma_events event = link_down ? sdma_event_e85_link_down : 3284 sdma_event_e80_hw_freeze; 3285 3286 /* set up the wait but do not wait here */ 3287 atomic_set(&dd->sdma_unfreeze_count, dd->num_sdma); 3288 3289 /* tell all engines to stop running and wait */ 3290 for (i = 0; i < dd->num_sdma; i++) 3291 sdma_process_event(&dd->per_sdma[i], event); 3292 3293 /* sdma_freeze() will wait for all engines to have stopped */ 3294 } 3295 3296 /* 3297 * SPC freeze handling for SDMA engines. Called when the driver knows 3298 * the SPC is fully frozen. 3299 */ 3300 void sdma_freeze(struct hfi1_devdata *dd) 3301 { 3302 int i; 3303 int ret; 3304 3305 /* 3306 * Make sure all engines have moved out of the running state before 3307 * continuing. 3308 */ 3309 ret = wait_event_interruptible(dd->sdma_unfreeze_wq, 3310 atomic_read(&dd->sdma_unfreeze_count) <= 3311 0); 3312 /* interrupted or count is negative, then unloading - just exit */ 3313 if (ret || atomic_read(&dd->sdma_unfreeze_count) < 0) 3314 return; 3315 3316 /* set up the count for the next wait */ 3317 atomic_set(&dd->sdma_unfreeze_count, dd->num_sdma); 3318 3319 /* tell all engines that the SPC is frozen, they can start cleaning */ 3320 for (i = 0; i < dd->num_sdma; i++) 3321 sdma_process_event(&dd->per_sdma[i], sdma_event_e81_hw_frozen); 3322 3323 /* 3324 * Wait for everyone to finish software clean before exiting. The 3325 * software clean will read engine CSRs, so must be completed before 3326 * the next step, which will clear the engine CSRs. 3327 */ 3328 (void)wait_event_interruptible(dd->sdma_unfreeze_wq, 3329 atomic_read(&dd->sdma_unfreeze_count) <= 0); 3330 /* no need to check results - done no matter what */ 3331 } 3332 3333 /* 3334 * SPC freeze handling for the SDMA engines. Called after the SPC is unfrozen. 3335 * 3336 * The SPC freeze acts like a SDMA halt and a hardware clean combined. All 3337 * that is left is a software clean. We could do it after the SPC is fully 3338 * frozen, but then we'd have to add another state to wait for the unfreeze. 3339 * Instead, just defer the software clean until the unfreeze step. 3340 */ 3341 void sdma_unfreeze(struct hfi1_devdata *dd) 3342 { 3343 int i; 3344 3345 /* tell all engines start freeze clean up */ 3346 for (i = 0; i < dd->num_sdma; i++) 3347 sdma_process_event(&dd->per_sdma[i], 3348 sdma_event_e82_hw_unfreeze); 3349 } 3350 3351 /** 3352 * _sdma_engine_progress_schedule() - schedule progress on engine 3353 * @sde: sdma_engine to schedule progress 3354 * 3355 */ 3356 void _sdma_engine_progress_schedule( 3357 struct sdma_engine *sde) 3358 { 3359 trace_hfi1_sdma_engine_progress(sde, sde->progress_mask); 3360 /* assume we have selected a good cpu */ 3361 write_csr(sde->dd, 3362 CCE_INT_FORCE + (8 * (IS_SDMA_START / 64)), 3363 sde->progress_mask); 3364 } 3365